Bitcoin Core  0.20.99
P2P Digital Currency
txdb.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2020 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <txdb.h>
7 
8 #include <node/ui_interface.h>
9 #include <pow.h>
10 #include <random.h>
11 #include <shutdown.h>
12 #include <uint256.h>
13 #include <util/system.h>
14 #include <util/translation.h>
15 #include <util/vector.h>
16 
17 #include <stdint.h>
18 
19 static const char DB_COIN = 'C';
20 static const char DB_COINS = 'c';
21 static const char DB_BLOCK_FILES = 'f';
22 static const char DB_BLOCK_INDEX = 'b';
23 
24 static const char DB_BEST_BLOCK = 'B';
25 static const char DB_HEAD_BLOCKS = 'H';
26 static const char DB_FLAG = 'F';
27 static const char DB_REINDEX_FLAG = 'R';
28 static const char DB_LAST_BLOCK = 'l';
29 
30 namespace {
31 
32 struct CoinEntry {
33  COutPoint* outpoint;
34  char key;
35  explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {}
36 
37  SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); }
38 };
39 
40 }
41 
42 CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : db(ldb_path, nCacheSize, fMemory, fWipe, true)
43 {
44 }
45 
46 bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
47  return db.Read(CoinEntry(&outpoint), coin);
48 }
49 
50 bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
51  return db.Exists(CoinEntry(&outpoint));
52 }
53 
55  uint256 hashBestChain;
56  if (!db.Read(DB_BEST_BLOCK, hashBestChain))
57  return uint256();
58  return hashBestChain;
59 }
60 
61 std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
62  std::vector<uint256> vhashHeadBlocks;
63  if (!db.Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) {
64  return std::vector<uint256>();
65  }
66  return vhashHeadBlocks;
67 }
68 
69 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
70  CDBBatch batch(db);
71  size_t count = 0;
72  size_t changed = 0;
73  size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize);
74  int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
75  assert(!hashBlock.IsNull());
76 
77  uint256 old_tip = GetBestBlock();
78  if (old_tip.IsNull()) {
79  // We may be in the middle of replaying.
80  std::vector<uint256> old_heads = GetHeadBlocks();
81  if (old_heads.size() == 2) {
82  assert(old_heads[0] == hashBlock);
83  old_tip = old_heads[1];
84  }
85  }
86 
87  // In the first batch, mark the database as being in the middle of a
88  // transition from old_tip to hashBlock.
89  // A vector is used for future extensibility, as we may want to support
90  // interrupting after partial writes from multiple independent reorgs.
91  batch.Erase(DB_BEST_BLOCK);
92  batch.Write(DB_HEAD_BLOCKS, Vector(hashBlock, old_tip));
93 
94  for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
95  if (it->second.flags & CCoinsCacheEntry::DIRTY) {
96  CoinEntry entry(&it->first);
97  if (it->second.coin.IsSpent())
98  batch.Erase(entry);
99  else
100  batch.Write(entry, it->second.coin);
101  changed++;
102  }
103  count++;
104  CCoinsMap::iterator itOld = it++;
105  mapCoins.erase(itOld);
106  if (batch.SizeEstimate() > batch_size) {
107  LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
108  db.WriteBatch(batch);
109  batch.Clear();
110  if (crash_simulate) {
111  static FastRandomContext rng;
112  if (rng.randrange(crash_simulate) == 0) {
113  LogPrintf("Simulating a crash. Goodbye.\n");
114  _Exit(0);
115  }
116  }
117  }
118  }
119 
120  // In the last batch, mark the database as consistent with hashBlock again.
121  batch.Erase(DB_HEAD_BLOCKS);
122  batch.Write(DB_BEST_BLOCK, hashBlock);
123 
124  LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
125  bool ret = db.WriteBatch(batch);
126  LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
127  return ret;
128 }
129 
131 {
132  return db.EstimateSize(DB_COIN, (char)(DB_COIN+1));
133 }
134 
135 CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
136 }
137 
139  return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
140 }
141 
142 bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
143  if (fReindexing)
144  return Write(DB_REINDEX_FLAG, '1');
145  else
146  return Erase(DB_REINDEX_FLAG);
147 }
148 
149 void CBlockTreeDB::ReadReindexing(bool &fReindexing) {
150  fReindexing = Exists(DB_REINDEX_FLAG);
151 }
152 
154  return Read(DB_LAST_BLOCK, nFile);
155 }
156 
158 {
159  CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(db).NewIterator(), GetBestBlock());
160  /* It seems that there are no "const iterators" for LevelDB. Since we
161  only need read operations on it, use a const-cast to get around
162  that restriction. */
163  i->pcursor->Seek(DB_COIN);
164  // Cache key of first record
165  if (i->pcursor->Valid()) {
166  CoinEntry entry(&i->keyTmp.second);
167  i->pcursor->GetKey(entry);
168  i->keyTmp.first = entry.key;
169  } else {
170  i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false
171  }
172  return i;
173 }
174 
176 {
177  // Return cached key
178  if (keyTmp.first == DB_COIN) {
179  key = keyTmp.second;
180  return true;
181  }
182  return false;
183 }
184 
186 {
187  return pcursor->GetValue(coin);
188 }
189 
191 {
192  return pcursor->GetValueSize();
193 }
194 
196 {
197  return keyTmp.first == DB_COIN;
198 }
199 
201 {
202  pcursor->Next();
203  CoinEntry entry(&keyTmp.second);
204  if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
205  keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false
206  } else {
207  keyTmp.first = entry.key;
208  }
209 }
210 
211 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
212  CDBBatch batch(*this);
213  for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
214  batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second);
215  }
216  batch.Write(DB_LAST_BLOCK, nLastFile);
217  for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
218  batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
219  }
220  return WriteBatch(batch, true);
221 }
222 
223 bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
224  return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
225 }
226 
227 bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
228  char ch;
229  if (!Read(std::make_pair(DB_FLAG, name), ch))
230  return false;
231  fValue = ch == '1';
232  return true;
233 }
234 
235 bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex)
236 {
237  std::unique_ptr<CDBIterator> pcursor(NewIterator());
238 
239  pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
240 
241  // Load m_block_index
242  while (pcursor->Valid()) {
243  if (ShutdownRequested()) return false;
244  std::pair<char, uint256> key;
245  if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
246  CDiskBlockIndex diskindex;
247  if (pcursor->GetValue(diskindex)) {
248  // Construct block index object
249  CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash());
250  pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
251  pindexNew->nHeight = diskindex.nHeight;
252  pindexNew->nFile = diskindex.nFile;
253  pindexNew->nDataPos = diskindex.nDataPos;
254  pindexNew->nUndoPos = diskindex.nUndoPos;
255  pindexNew->nVersion = diskindex.nVersion;
256  pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
257  pindexNew->nTime = diskindex.nTime;
258  pindexNew->nBits = diskindex.nBits;
259  pindexNew->nNonce = diskindex.nNonce;
260  pindexNew->nStatus = diskindex.nStatus;
261  pindexNew->nTx = diskindex.nTx;
262 
263  if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams))
264  return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
265 
266  pcursor->Next();
267  } else {
268  return error("%s: failed to read value", __func__);
269  }
270  } else {
271  break;
272  }
273  }
274 
275  return true;
276 }
277 
278 namespace {
279 
281 class CCoins
282 {
283 public:
285  bool fCoinBase;
286 
288  std::vector<CTxOut> vout;
289 
291  int nHeight;
292 
294  CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
295 
296  template<typename Stream>
297  void Unserialize(Stream &s) {
298  unsigned int nCode = 0;
299  // version
300  unsigned int nVersionDummy;
301  ::Unserialize(s, VARINT(nVersionDummy));
302  // header code
303  ::Unserialize(s, VARINT(nCode));
304  fCoinBase = nCode & 1;
305  std::vector<bool> vAvail(2, false);
306  vAvail[0] = (nCode & 2) != 0;
307  vAvail[1] = (nCode & 4) != 0;
308  unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
309  // spentness bitmask
310  while (nMaskCode > 0) {
311  unsigned char chAvail = 0;
312  ::Unserialize(s, chAvail);
313  for (unsigned int p = 0; p < 8; p++) {
314  bool f = (chAvail & (1 << p)) != 0;
315  vAvail.push_back(f);
316  }
317  if (chAvail != 0)
318  nMaskCode--;
319  }
320  // txouts themself
321  vout.assign(vAvail.size(), CTxOut());
322  for (unsigned int i = 0; i < vAvail.size(); i++) {
323  if (vAvail[i])
324  ::Unserialize(s, Using<TxOutCompression>(vout[i]));
325  }
326  // coinbase height
328  }
329 };
330 
331 }
332 
338  std::unique_ptr<CDBIterator> pcursor(db.NewIterator());
339  pcursor->Seek(std::make_pair(DB_COINS, uint256()));
340  if (!pcursor->Valid()) {
341  return true;
342  }
343 
344  int64_t count = 0;
345  LogPrintf("Upgrading utxo-set database...\n");
346  LogPrintf("[0%%]..."); /* Continued */
347  uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true);
348  size_t batch_size = 1 << 24;
349  CDBBatch batch(db);
350  int reportDone = 0;
351  std::pair<unsigned char, uint256> key;
352  std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
353  while (pcursor->Valid()) {
354  if (ShutdownRequested()) {
355  break;
356  }
357  if (pcursor->GetKey(key) && key.first == DB_COINS) {
358  if (count++ % 256 == 0) {
359  uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
360  int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
361  uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true);
362  if (reportDone < percentageDone/10) {
363  // report max. every 10% step
364  LogPrintf("[%d%%]...", percentageDone); /* Continued */
365  reportDone = percentageDone/10;
366  }
367  }
368  CCoins old_coins;
369  if (!pcursor->GetValue(old_coins)) {
370  return error("%s: cannot parse CCoins record", __func__);
371  }
372  COutPoint outpoint(key.second, 0);
373  for (size_t i = 0; i < old_coins.vout.size(); ++i) {
374  if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
375  Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
376  outpoint.n = i;
377  CoinEntry entry(&outpoint);
378  batch.Write(entry, newcoin);
379  }
380  }
381  batch.Erase(key);
382  if (batch.SizeEstimate() > batch_size) {
383  db.WriteBatch(batch);
384  batch.Clear();
385  db.CompactRange(prev_key, key);
386  prev_key = key;
387  }
388  pcursor->Next();
389  } else {
390  break;
391  }
392  }
393  db.WriteBatch(batch);
394  db.CompactRange({DB_COINS, uint256()}, key);
395  uiInterface.ShowProgress("", 100, false);
396  LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
397  return !ShutdownRequested();
398 }
bool Exists(const K &key) const
Definition: dbwrapper.h:264
bool GetValue(Coin &coin) const override
Definition: txdb.cpp:185
static const char DB_LAST_BLOCK
Definition: txdb.cpp:28
bool GetCoin(const COutPoint &outpoint, Coin &coin) const override
Retrieve the Coin (unspent transaction output) for a given outpoint.
Definition: txdb.cpp:46
#define VARINT(obj)
Definition: serialize.h:468
std::string ToString() const
Definition: chain.h:273
void Clear()
Definition: dbwrapper.h:65
bool ShutdownRequested()
Definition: shutdown.cpp:20
bool Upgrade()
Attempt to update from an older database format. Returns whether an error occurred.
Definition: txdb.cpp:337
#define LogPrint(category,...)
Definition: logging.h:182
Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB.
Definition: txdb.h:66
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:144
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:46
uint32_t nStatus
Verification status of this block. See enum BlockStatus.
Definition: chain.h:174
A UTXO entry.
Definition: coins.h:30
static const char DB_BEST_BLOCK
Definition: txdb.cpp:24
bool ReadLastBlockFile(int &nFile)
Definition: txdb.cpp:153
CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe)
Definition: txdb.cpp:42
unsigned int nHeight
void Erase(const K &key)
Definition: dbwrapper.h:97
std::unique_ptr< CDBIterator > pcursor
Definition: txdb.h:81
void ReadReindexing(bool &fReindexing)
Definition: txdb.cpp:149
static const char DB_COIN
Definition: txdb.cpp:19
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:166
uint32_t nTime
Definition: chain.h:179
int nFile
Which # file this block is stored in (blk?????.dat)
Definition: chain.h:153
bool GetKey(COutPoint &key) const override
Definition: txdb.cpp:175
unsigned char * begin()
Definition: uint256.h:54
bool IsNull() const
Definition: uint256.h:30
bool WriteReindexing(bool fReindexing)
Definition: txdb.cpp:142
CDBIterator * NewIterator()
Definition: dbwrapper.h:295
DIRTY means the CCoinsCacheEntry is potentially different from the version in the parent cache...
Definition: coins.h:140
std::vector< typename std::common_type< Args... >::type > Vector(Args &&... args)
Construct a vector with the specified elements.
Definition: vector.h:20
uint256 GetBlockHash() const
Definition: chain.h:233
static const char DB_BLOCK_FILES
Definition: txdb.cpp:21
#define VARINT_MODE(obj, mode)
Definition: serialize.h:467
bool Erase(const K &key, bool fSync=false)
Definition: dbwrapper.h:283
uint32_t nNonce
Definition: chain.h:181
static const char DB_FLAG
Definition: txdb.cpp:26
unsigned int nDataPos
Byte offset within blk?????.dat where this block&#39;s data is stored.
Definition: chain.h:156
const char * name
Definition: rest.cpp:41
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:57
Fast randomness source.
Definition: random.h:119
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:620
uint32_t n
Definition: transaction.h:22
CDBWrapper db
Definition: txdb.h:46
std::unordered_map< COutPoint, CCoinsCacheEntry, SaltedOutpointHasher > CCoinsMap
Definition: coins.h:157
uint256 hashMerkleRoot
Definition: chain.h:178
void Write(const K &key, const V &value)
Definition: dbwrapper.h:72
size_t SizeEstimate() const
Definition: dbwrapper.h:113
An output of a transaction.
Definition: transaction.h:120
Used to marshal pointers into hashes for db storage.
Definition: chain.h:320
static const int64_t nDefaultDbBatchSize
-dbbatchsize default (bytes)
Definition: txdb.h:26
size_t EstimateSize() const override
Estimate database size (0 if not implemented)
Definition: txdb.cpp:130
Parameters that influence chain consensus.
Definition: params.h:45
bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params &params)
Check whether a block hash satisfies the proof-of-work requirement specified by nBits.
Definition: pow.cpp:74
An outpoint - a combination of a transaction hash and an index n into its vout.
Definition: transaction.h:18
std::pair< char, COutPoint > keyTmp
Definition: txdb.h:82
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
bool HaveCoin(const COutPoint &outpoint) const override
Just check whether a given outpoint is unspent.
Definition: txdb.cpp:50
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:227
CBlockTreeDB(size_t nCacheSize, bool fMemory=false, bool fWipe=false)
Definition: txdb.cpp:135
bool WriteBatchSync(const std::vector< std::pair< int, const CBlockFileInfo *> > &fileInfo, int nLastFile, const std::vector< const CBlockIndex *> &blockinfo)
Definition: txdb.cpp:211
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info)
Definition: txdb.cpp:138
#define SERIALIZE_METHODS(cls, obj)
Implement the Serialize and Unserialize methods by delegating to a single templated static method tha...
Definition: serialize.h:207
unsigned int nUndoPos
Byte offset within rev?????.dat where this block&#39;s undo data is stored.
Definition: chain.h:159
uint256 GetBestBlock() const override
Retrieve the block hash whose state this CCoinsView currently represents.
Definition: txdb.cpp:54
int32_t nVersion
block header
Definition: chain.h:177
256-bit opaque blob.
Definition: uint256.h:120
uint256 hashPrev
Definition: chain.h:323
static const char DB_BLOCK_INDEX
Definition: txdb.cpp:22
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:256
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:137
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:380
ArgsManager gArgs
Definition: system.cpp:77
static int count
Definition: tests.c:35
void Unserialize(Stream &s, char &a)
Definition: serialize.h:241
bool LoadBlockIndexGuts(const Consensus::Params &consensusParams, std::function< CBlockIndex *(const uint256 &)> insertBlockIndex)
Definition: txdb.cpp:235
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:223
bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override
Do a bulk modification (multiple Coin changes + BestBlock change).
Definition: txdb.cpp:69
void Next() override
Definition: txdb.cpp:200
CCoinsViewCursor * Cursor() const override
Get a cursor to iterate over the whole state.
Definition: txdb.cpp:157
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
bool Valid() const override
Definition: txdb.cpp:195
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:150
uint256 GetBlockHash() const
Definition: chain.h:354
#define READWRITE(...)
Definition: serialize.h:171
static const char DB_REINDEX_FLAG
Definition: txdb.cpp:27
unsigned int GetValueSize() const override
Definition: txdb.cpp:190
auto it
Definition: validation.cpp:384
static const char DB_COINS
Definition: txdb.cpp:20
size_t EstimateSize(const K &key_begin, const K &key_end) const
Definition: dbwrapper.h:306
static const char DB_HEAD_BLOCKS
Definition: txdb.cpp:25
bool error(const char *fmt, const Args &... args)
Definition: system.h:49
std::vector< uint256 > GetHeadBlocks() const override
Retrieve the range of blocks that may have been only partially written.
Definition: txdb.cpp:61
uint32_t nBits
Definition: chain.h:180
uint64_t randrange(uint64_t range) noexcept
Generate a random integer in the range [0..range).
Definition: random.h:190
unsigned int nTx
Number of transactions in this block.
Definition: chain.h:166
Cursor for iterating over CoinsView state.
Definition: coins.h:160