Bitcoin Core  0.20.99
P2P Digital Currency
txdb.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2020 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <txdb.h>
7 
8 #include <node/ui_interface.h>
9 #include <pow.h>
10 #include <random.h>
11 #include <shutdown.h>
12 #include <uint256.h>
13 #include <util/memory.h>
14 #include <util/system.h>
15 #include <util/translation.h>
16 #include <util/vector.h>
17 
18 #include <stdint.h>
19 
20 static const char DB_COIN = 'C';
21 static const char DB_COINS = 'c';
22 static const char DB_BLOCK_FILES = 'f';
23 static const char DB_BLOCK_INDEX = 'b';
24 
25 static const char DB_BEST_BLOCK = 'B';
26 static const char DB_HEAD_BLOCKS = 'H';
27 static const char DB_FLAG = 'F';
28 static const char DB_REINDEX_FLAG = 'R';
29 static const char DB_LAST_BLOCK = 'l';
30 
31 namespace {
32 
33 struct CoinEntry {
34  COutPoint* outpoint;
35  char key;
36  explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {}
37 
38  SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); }
39 };
40 
41 }
42 
43 CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) :
44  m_db(MakeUnique<CDBWrapper>(ldb_path, nCacheSize, fMemory, fWipe, true)),
45  m_ldb_path(ldb_path),
46  m_is_memory(fMemory) { }
47 
48 void CCoinsViewDB::ResizeCache(size_t new_cache_size)
49 {
50  // Have to do a reset first to get the original `m_db` state to release its
51  // filesystem lock.
52  m_db.reset();
53  m_db = MakeUnique<CDBWrapper>(
54  m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true);
55 }
56 
57 bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
58  return m_db->Read(CoinEntry(&outpoint), coin);
59 }
60 
61 bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
62  return m_db->Exists(CoinEntry(&outpoint));
63 }
64 
66  uint256 hashBestChain;
67  if (!m_db->Read(DB_BEST_BLOCK, hashBestChain))
68  return uint256();
69  return hashBestChain;
70 }
71 
72 std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
73  std::vector<uint256> vhashHeadBlocks;
74  if (!m_db->Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) {
75  return std::vector<uint256>();
76  }
77  return vhashHeadBlocks;
78 }
79 
80 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
81  CDBBatch batch(*m_db);
82  size_t count = 0;
83  size_t changed = 0;
84  size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize);
85  int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
86  assert(!hashBlock.IsNull());
87 
88  uint256 old_tip = GetBestBlock();
89  if (old_tip.IsNull()) {
90  // We may be in the middle of replaying.
91  std::vector<uint256> old_heads = GetHeadBlocks();
92  if (old_heads.size() == 2) {
93  assert(old_heads[0] == hashBlock);
94  old_tip = old_heads[1];
95  }
96  }
97 
98  // In the first batch, mark the database as being in the middle of a
99  // transition from old_tip to hashBlock.
100  // A vector is used for future extensibility, as we may want to support
101  // interrupting after partial writes from multiple independent reorgs.
102  batch.Erase(DB_BEST_BLOCK);
103  batch.Write(DB_HEAD_BLOCKS, Vector(hashBlock, old_tip));
104 
105  for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
106  if (it->second.flags & CCoinsCacheEntry::DIRTY) {
107  CoinEntry entry(&it->first);
108  if (it->second.coin.IsSpent())
109  batch.Erase(entry);
110  else
111  batch.Write(entry, it->second.coin);
112  changed++;
113  }
114  count++;
115  CCoinsMap::iterator itOld = it++;
116  mapCoins.erase(itOld);
117  if (batch.SizeEstimate() > batch_size) {
118  LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
119  m_db->WriteBatch(batch);
120  batch.Clear();
121  if (crash_simulate) {
122  static FastRandomContext rng;
123  if (rng.randrange(crash_simulate) == 0) {
124  LogPrintf("Simulating a crash. Goodbye.\n");
125  _Exit(0);
126  }
127  }
128  }
129  }
130 
131  // In the last batch, mark the database as consistent with hashBlock again.
132  batch.Erase(DB_HEAD_BLOCKS);
133  batch.Write(DB_BEST_BLOCK, hashBlock);
134 
135  LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
136  bool ret = m_db->WriteBatch(batch);
137  LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
138  return ret;
139 }
140 
142 {
143  return m_db->EstimateSize(DB_COIN, (char)(DB_COIN+1));
144 }
145 
146 CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
147 }
148 
150  return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
151 }
152 
153 bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
154  if (fReindexing)
155  return Write(DB_REINDEX_FLAG, '1');
156  else
157  return Erase(DB_REINDEX_FLAG);
158 }
159 
160 void CBlockTreeDB::ReadReindexing(bool &fReindexing) {
161  fReindexing = Exists(DB_REINDEX_FLAG);
162 }
163 
165  return Read(DB_LAST_BLOCK, nFile);
166 }
167 
169 {
170  CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(*m_db).NewIterator(), GetBestBlock());
171  /* It seems that there are no "const iterators" for LevelDB. Since we
172  only need read operations on it, use a const-cast to get around
173  that restriction. */
174  i->pcursor->Seek(DB_COIN);
175  // Cache key of first record
176  if (i->pcursor->Valid()) {
177  CoinEntry entry(&i->keyTmp.second);
178  i->pcursor->GetKey(entry);
179  i->keyTmp.first = entry.key;
180  } else {
181  i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false
182  }
183  return i;
184 }
185 
187 {
188  // Return cached key
189  if (keyTmp.first == DB_COIN) {
190  key = keyTmp.second;
191  return true;
192  }
193  return false;
194 }
195 
197 {
198  return pcursor->GetValue(coin);
199 }
200 
202 {
203  return pcursor->GetValueSize();
204 }
205 
207 {
208  return keyTmp.first == DB_COIN;
209 }
210 
212 {
213  pcursor->Next();
214  CoinEntry entry(&keyTmp.second);
215  if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
216  keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false
217  } else {
218  keyTmp.first = entry.key;
219  }
220 }
221 
222 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
223  CDBBatch batch(*this);
224  for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
225  batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second);
226  }
227  batch.Write(DB_LAST_BLOCK, nLastFile);
228  for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
229  batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
230  }
231  return WriteBatch(batch, true);
232 }
233 
234 bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
235  return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
236 }
237 
238 bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
239  char ch;
240  if (!Read(std::make_pair(DB_FLAG, name), ch))
241  return false;
242  fValue = ch == '1';
243  return true;
244 }
245 
246 bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex)
247 {
248  std::unique_ptr<CDBIterator> pcursor(NewIterator());
249 
250  pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
251 
252  // Load m_block_index
253  while (pcursor->Valid()) {
254  if (ShutdownRequested()) return false;
255  std::pair<char, uint256> key;
256  if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
257  CDiskBlockIndex diskindex;
258  if (pcursor->GetValue(diskindex)) {
259  // Construct block index object
260  CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash());
261  pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
262  pindexNew->nHeight = diskindex.nHeight;
263  pindexNew->nFile = diskindex.nFile;
264  pindexNew->nDataPos = diskindex.nDataPos;
265  pindexNew->nUndoPos = diskindex.nUndoPos;
266  pindexNew->nVersion = diskindex.nVersion;
267  pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
268  pindexNew->nTime = diskindex.nTime;
269  pindexNew->nBits = diskindex.nBits;
270  pindexNew->nNonce = diskindex.nNonce;
271  pindexNew->nStatus = diskindex.nStatus;
272  pindexNew->nTx = diskindex.nTx;
273 
274  if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams))
275  return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
276 
277  pcursor->Next();
278  } else {
279  return error("%s: failed to read value", __func__);
280  }
281  } else {
282  break;
283  }
284  }
285 
286  return true;
287 }
288 
289 namespace {
290 
292 class CCoins
293 {
294 public:
296  bool fCoinBase;
297 
299  std::vector<CTxOut> vout;
300 
302  int nHeight;
303 
305  CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
306 
307  template<typename Stream>
308  void Unserialize(Stream &s) {
309  unsigned int nCode = 0;
310  // version
311  unsigned int nVersionDummy;
312  ::Unserialize(s, VARINT(nVersionDummy));
313  // header code
314  ::Unserialize(s, VARINT(nCode));
315  fCoinBase = nCode & 1;
316  std::vector<bool> vAvail(2, false);
317  vAvail[0] = (nCode & 2) != 0;
318  vAvail[1] = (nCode & 4) != 0;
319  unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
320  // spentness bitmask
321  while (nMaskCode > 0) {
322  unsigned char chAvail = 0;
323  ::Unserialize(s, chAvail);
324  for (unsigned int p = 0; p < 8; p++) {
325  bool f = (chAvail & (1 << p)) != 0;
326  vAvail.push_back(f);
327  }
328  if (chAvail != 0)
329  nMaskCode--;
330  }
331  // txouts themself
332  vout.assign(vAvail.size(), CTxOut());
333  for (unsigned int i = 0; i < vAvail.size(); i++) {
334  if (vAvail[i])
335  ::Unserialize(s, Using<TxOutCompression>(vout[i]));
336  }
337  // coinbase height
339  }
340 };
341 
342 }
343 
349  std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator());
350  pcursor->Seek(std::make_pair(DB_COINS, uint256()));
351  if (!pcursor->Valid()) {
352  return true;
353  }
354 
355  int64_t count = 0;
356  LogPrintf("Upgrading utxo-set database...\n");
357  LogPrintf("[0%%]..."); /* Continued */
358  uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true);
359  size_t batch_size = 1 << 24;
360  CDBBatch batch(*m_db);
361  int reportDone = 0;
362  std::pair<unsigned char, uint256> key;
363  std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
364  while (pcursor->Valid()) {
365  if (ShutdownRequested()) {
366  break;
367  }
368  if (pcursor->GetKey(key) && key.first == DB_COINS) {
369  if (count++ % 256 == 0) {
370  uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
371  int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
372  uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true);
373  if (reportDone < percentageDone/10) {
374  // report max. every 10% step
375  LogPrintf("[%d%%]...", percentageDone); /* Continued */
376  reportDone = percentageDone/10;
377  }
378  }
379  CCoins old_coins;
380  if (!pcursor->GetValue(old_coins)) {
381  return error("%s: cannot parse CCoins record", __func__);
382  }
383  COutPoint outpoint(key.second, 0);
384  for (size_t i = 0; i < old_coins.vout.size(); ++i) {
385  if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
386  Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
387  outpoint.n = i;
388  CoinEntry entry(&outpoint);
389  batch.Write(entry, newcoin);
390  }
391  }
392  batch.Erase(key);
393  if (batch.SizeEstimate() > batch_size) {
394  m_db->WriteBatch(batch);
395  batch.Clear();
396  m_db->CompactRange(prev_key, key);
397  prev_key = key;
398  }
399  pcursor->Next();
400  } else {
401  break;
402  }
403  }
404  m_db->WriteBatch(batch);
405  m_db->CompactRange({DB_COINS, uint256()}, key);
406  uiInterface.ShowProgress("", 100, false);
407  LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
408  return !ShutdownRequested();
409 }
bool Exists(const K &key) const
Definition: dbwrapper.h:264
bool GetValue(Coin &coin) const override
Definition: txdb.cpp:196
static const char DB_LAST_BLOCK
Definition: txdb.cpp:29
bool GetCoin(const COutPoint &outpoint, Coin &coin) const override
Retrieve the Coin (unspent transaction output) for a given outpoint.
Definition: txdb.cpp:57
#define VARINT(obj)
Definition: serialize.h:479
std::string ToString() const
Definition: chain.h:273
void Clear()
Definition: dbwrapper.h:65
bool ShutdownRequested()
Definition: shutdown.cpp:20
std::deque< CInv >::iterator it
bool Upgrade()
Attempt to update from an older database format. Returns whether an error occurred.
Definition: txdb.cpp:348
#define LogPrint(category,...)
Definition: logging.h:182
Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB.
Definition: txdb.h:74
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:144
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:46
uint32_t nStatus
Verification status of this block. See enum BlockStatus.
Definition: chain.h:174
A UTXO entry.
Definition: coins.h:30
static const char DB_BEST_BLOCK
Definition: txdb.cpp:25
bool ReadLastBlockFile(int &nFile)
Definition: txdb.cpp:164
CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe)
Definition: txdb.cpp:43
unsigned int nHeight
void Erase(const K &key)
Definition: dbwrapper.h:97
std::unique_ptr< CDBIterator > pcursor
Definition: txdb.h:89
void ReadReindexing(bool &fReindexing)
Definition: txdb.cpp:160
static const char DB_COIN
Definition: txdb.cpp:20
void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Dynamically alter the underlying leveldb cache size.
Definition: txdb.cpp:48
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:166
uint32_t nTime
Definition: chain.h:179
std::unique_ptr< T > MakeUnique(Args &&... args)
Substitute for C++14 std::make_unique.
Definition: memory.h:14
int nFile
Which # file this block is stored in (blk?????.dat)
Definition: chain.h:153
bool GetKey(COutPoint &key) const override
Definition: txdb.cpp:186
unsigned char * begin()
Definition: uint256.h:58
bool IsNull() const
Definition: uint256.h:31
bool WriteReindexing(bool fReindexing)
Definition: txdb.cpp:153
CDBIterator * NewIterator()
Definition: dbwrapper.h:295
DIRTY means the CCoinsCacheEntry is potentially different from the version in the parent cache...
Definition: coins.h:140
std::vector< typename std::common_type< Args... >::type > Vector(Args &&... args)
Construct a vector with the specified elements.
Definition: vector.h:20
uint256 GetBlockHash() const
Definition: chain.h:233
static const char DB_BLOCK_FILES
Definition: txdb.cpp:22
#define VARINT_MODE(obj, mode)
Definition: serialize.h:478
bool Erase(const K &key, bool fSync=false)
Definition: dbwrapper.h:283
uint32_t nNonce
Definition: chain.h:181
static const char DB_FLAG
Definition: txdb.cpp:27
unsigned int nDataPos
Byte offset within blk?????.dat where this block&#39;s data is stored.
Definition: chain.h:156
const char * name
Definition: rest.cpp:41
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:57
Fast randomness source.
Definition: random.h:119
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:706
uint32_t n
Definition: transaction.h:30
std::unordered_map< COutPoint, CCoinsCacheEntry, SaltedOutpointHasher > CCoinsMap
Definition: coins.h:157
uint256 hashMerkleRoot
Definition: chain.h:178
void Write(const K &key, const V &value)
Definition: dbwrapper.h:72
size_t SizeEstimate() const
Definition: dbwrapper.h:113
An output of a transaction.
Definition: transaction.h:128
Used to marshal pointers into hashes for db storage.
Definition: chain.h:320
static const int64_t nDefaultDbBatchSize
-dbbatchsize default (bytes)
Definition: txdb.h:26
size_t EstimateSize() const override
Estimate database size (0 if not implemented)
Definition: txdb.cpp:141
Parameters that influence chain consensus.
Definition: params.h:46
bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params &params)
Check whether a block hash satisfies the proof-of-work requirement specified by nBits.
Definition: pow.cpp:74
An outpoint - a combination of a transaction hash and an index n into its vout.
Definition: transaction.h:26
std::pair< char, COutPoint > keyTmp
Definition: txdb.h:90
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
bool m_is_memory
Definition: txdb.h:51
bool HaveCoin(const COutPoint &outpoint) const override
Just check whether a given outpoint is unspent.
Definition: txdb.cpp:61
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:238
CBlockTreeDB(size_t nCacheSize, bool fMemory=false, bool fWipe=false)
Definition: txdb.cpp:146
bool WriteBatchSync(const std::vector< std::pair< int, const CBlockFileInfo *> > &fileInfo, int nLastFile, const std::vector< const CBlockIndex *> &blockinfo)
Definition: txdb.cpp:222
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info)
Definition: txdb.cpp:149
unsigned int nUndoPos
Byte offset within rev?????.dat where this block&#39;s undo data is stored.
Definition: chain.h:159
uint256 GetBestBlock() const override
Retrieve the block hash whose state this CCoinsView currently represents.
Definition: txdb.cpp:65
int32_t nVersion
block header
Definition: chain.h:177
256-bit opaque blob.
Definition: uint256.h:124
uint256 hashPrev
Definition: chain.h:323
static const char DB_BLOCK_INDEX
Definition: txdb.cpp:23
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:256
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:137
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:466
ArgsManager gArgs
Definition: system.cpp:76
static int count
Definition: tests.c:35
void Unserialize(Stream &s, char &a)
Definition: serialize.h:245
bool LoadBlockIndexGuts(const Consensus::Params &consensusParams, std::function< CBlockIndex *(const uint256 &)> insertBlockIndex)
Definition: txdb.cpp:246
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:234
bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override
Do a bulk modification (multiple Coin changes + BestBlock change).
Definition: txdb.cpp:80
void Next() override
Definition: txdb.cpp:211
CCoinsViewCursor * Cursor() const override
Get a cursor to iterate over the whole state.
Definition: txdb.cpp:168
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
fs::path m_ldb_path
Definition: txdb.h:50
bool Valid() const override
Definition: txdb.cpp:206
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:150
uint256 GetBlockHash() const
Definition: chain.h:354
#define READWRITE(...)
Definition: serialize.h:175
static const char DB_REINDEX_FLAG
Definition: txdb.cpp:28
unsigned int GetValueSize() const override
Definition: txdb.cpp:201
static const char DB_COINS
Definition: txdb.cpp:21
static const char DB_HEAD_BLOCKS
Definition: txdb.cpp:26
SERIALIZE_METHODS(CService, obj)
Definition: netaddress.h:525
bool error(const char *fmt, const Args &... args)
Definition: system.h:52
std::vector< uint256 > GetHeadBlocks() const override
Retrieve the range of blocks that may have been only partially written.
Definition: txdb.cpp:72
uint32_t nBits
Definition: chain.h:180
uint64_t randrange(uint64_t range) noexcept
Generate a random integer in the range [0..range).
Definition: random.h:190
unsigned int nTx
Number of transactions in this block.
Definition: chain.h:166
std::unique_ptr< CDBWrapper > m_db
Definition: txdb.h:49
Cursor for iterating over CoinsView state.
Definition: coins.h:160