Bitcoin Core  0.20.99
P2P Digital Currency
txindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2017-2019 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <index/disktxpos.h>
6 #include <index/txindex.h>
7 #include <node/ui_interface.h>
8 #include <shutdown.h>
9 #include <util/system.h>
10 #include <util/translation.h>
11 #include <validation.h>
12 
13 constexpr char DB_BEST_BLOCK = 'B';
14 constexpr char DB_TXINDEX = 't';
15 constexpr char DB_TXINDEX_BLOCK = 'T';
16 
17 std::unique_ptr<TxIndex> g_txindex;
18 
19 
20 
22 class TxIndex::DB : public BaseIndex::DB
23 {
24 public:
25  explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
26 
29  bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
30 
32  bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
33 
36  bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
37 };
38 
39 TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
40  BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
41 {}
42 
43 bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
44 {
45  return Read(std::make_pair(DB_TXINDEX, txid), pos);
46 }
47 
48 bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
49 {
50  CDBBatch batch(*this);
51  for (const auto& tuple : v_pos) {
52  batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
53  }
54  return WriteBatch(batch);
55 }
56 
57 /*
58  * Safely persist a transfer of data from the old txindex database to the new one, and compact the
59  * range of keys updated. This is used internally by MigrateData.
60  */
62  CDBBatch& batch_newdb, CDBBatch& batch_olddb,
63  const std::pair<unsigned char, uint256>& begin_key,
64  const std::pair<unsigned char, uint256>& end_key)
65 {
66  // Sync new DB changes to disk before deleting from old DB.
67  newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
68  olddb.WriteBatch(batch_olddb);
69  olddb.CompactRange(begin_key, end_key);
70 
71  batch_newdb.Clear();
72  batch_olddb.Clear();
73 }
74 
75 bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
76 {
77  // The prior implementation of txindex was always in sync with block index
78  // and presence was indicated with a boolean DB flag. If the flag is set,
79  // this means the txindex from a previous version is valid and in sync with
80  // the chain tip. The first step of the migration is to unset the flag and
81  // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
82  // index entries are copied over in batches to the new database. Finally,
83  // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
84  // written to the new database.
85  //
86  // Unsetting the boolean flag ensures that if the node is downgraded to a
87  // previous version, it will not see a corrupted, partially migrated index
88  // -- it will see that the txindex is disabled. When the node is upgraded
89  // again, the migration will pick up where it left off and sync to the block
90  // with hash DB_TXINDEX_BLOCK.
91  bool f_legacy_flag = false;
92  block_tree_db.ReadFlag("txindex", f_legacy_flag);
93  if (f_legacy_flag) {
94  if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
95  return error("%s: cannot write block indicator", __func__);
96  }
97  if (!block_tree_db.WriteFlag("txindex", false)) {
98  return error("%s: cannot write block index db flag", __func__);
99  }
100  }
101 
102  CBlockLocator locator;
103  if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
104  return true;
105  }
106 
107  int64_t count = 0;
108  LogPrintf("Upgrading txindex database... [0%%]\n");
109  uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0, true);
110  int report_done = 0;
111  const size_t batch_size = 1 << 24; // 16 MiB
112 
113  CDBBatch batch_newdb(*this);
114  CDBBatch batch_olddb(block_tree_db);
115 
116  std::pair<unsigned char, uint256> key;
117  std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
118  std::pair<unsigned char, uint256> prev_key = begin_key;
119 
120  bool interrupted = false;
121  std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
122  for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
123  if (ShutdownRequested()) {
124  interrupted = true;
125  break;
126  }
127 
128  if (!cursor->GetKey(key)) {
129  return error("%s: cannot get key from valid cursor", __func__);
130  }
131  if (key.first != DB_TXINDEX) {
132  break;
133  }
134 
135  // Log progress every 10%.
136  if (++count % 256 == 0) {
137  // Since txids are uniformly random and traversed in increasing order, the high 16 bits
138  // of the hash can be used to estimate the current progress.
139  const uint256& txid = key.second;
140  uint32_t high_nibble =
141  (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
142  (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
143  int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
144 
145  uiInterface.ShowProgress(_("Upgrading txindex database").translated, percentage_done, true);
146  if (report_done < percentage_done/10) {
147  LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
148  report_done = percentage_done/10;
149  }
150  }
151 
152  CDiskTxPos value;
153  if (!cursor->GetValue(value)) {
154  return error("%s: cannot parse txindex record", __func__);
155  }
156  batch_newdb.Write(key, value);
157  batch_olddb.Erase(key);
158 
159  if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
160  // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
161  // because LevelDB iterators are guaranteed to provide a consistent view of the
162  // underlying data, like a lightweight snapshot.
163  WriteTxIndexMigrationBatches(*this, block_tree_db,
164  batch_newdb, batch_olddb,
165  prev_key, key);
166  prev_key = key;
167  }
168  }
169 
170  // If these final DB batches complete the migration, write the best block
171  // hash marker to the new database and delete from the old one. This signals
172  // that the former is fully caught up to that point in the blockchain and
173  // that all txindex entries have been removed from the latter.
174  if (!interrupted) {
175  batch_olddb.Erase(DB_TXINDEX_BLOCK);
176  batch_newdb.Write(DB_BEST_BLOCK, locator);
177  }
178 
179  WriteTxIndexMigrationBatches(*this, block_tree_db,
180  batch_newdb, batch_olddb,
181  begin_key, key);
182 
183  if (interrupted) {
184  LogPrintf("[CANCELLED].\n");
185  return false;
186  }
187 
188  uiInterface.ShowProgress("", 100, false);
189 
190  LogPrintf("[DONE].\n");
191  return true;
192 }
193 
194 TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
195  : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
196 {}
197 
199 
201 {
202  LOCK(cs_main);
203 
204  // Attempt to migrate txindex from the old database to the new one. Even if
205  // chain_tip is null, the node could be reindexing and we still want to
206  // delete txindex records in the old database.
207  if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) {
208  return false;
209  }
210 
211  return BaseIndex::Init();
212 }
213 
214 bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
215 {
216  // Exclude genesis block transaction because outputs are not spendable.
217  if (pindex->nHeight == 0) return true;
218 
219  CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
220  std::vector<std::pair<uint256, CDiskTxPos>> vPos;
221  vPos.reserve(block.vtx.size());
222  for (const auto& tx : block.vtx) {
223  vPos.emplace_back(tx->GetHash(), pos);
225  }
226  return m_db->WriteTxs(vPos);
227 }
228 
229 BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
230 
231 bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
232 {
233  CDiskTxPos postx;
234  if (!m_db->ReadTxPos(tx_hash, postx)) {
235  return false;
236  }
237 
238  CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
239  if (file.IsNull()) {
240  return error("%s: OpenBlockFile failed", __func__);
241  }
242  CBlockHeader header;
243  try {
244  file >> header;
245  if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
246  return error("%s: fseek(...) failed", __func__);
247  }
248  file >> tx;
249  } catch (const std::exception& e) {
250  return error("%s: Deserialize or I/O error - %s", __func__, e.what());
251  }
252  if (tx->GetHash() != tx_hash) {
253  return error("%s: txid mismatch", __func__);
254  }
255  block_hash = header.GetHash();
256  return true;
257 }
std::shared_ptr< const CTransaction > CTransactionRef
Definition: transaction.h:395
static void WriteTxIndexMigrationBatches(CDBWrapper &newdb, CDBWrapper &olddb, CDBBatch &batch_newdb, CDBBatch &batch_olddb, const std::pair< unsigned char, uint256 > &begin_key, const std::pair< unsigned char, uint256 > &end_key)
Definition: txindex.cpp:61
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
Definition: txindex.cpp:214
void Clear()
Definition: dbwrapper.h:65
bool ShutdownRequested()
Definition: shutdown.cpp:20
unsigned int nTxOffset
Definition: disktxpos.h:13
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:54
Describes a place in the block chain to another node such that if the other node doesn&#39;t have the sam...
Definition: block.h:114
constexpr char DB_TXINDEX_BLOCK
Definition: txindex.cpp:15
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:46
Definition: block.h:62
CChain & ChainActive()
Please prefer the identical ChainstateManager::ActiveChain.
Definition: validation.cpp:113
void Erase(const K &key)
Definition: dbwrapper.h:97
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 ...
Definition: serialize.h:276
constexpr char DB_TXINDEX
Definition: txindex.cpp:14
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:166
FlatFilePos GetBlockPos() const
Definition: chain.h:202
std::unique_ptr< T > MakeUnique(Args &&... args)
Substitute for C++14 std::make_unique.
Definition: memory.h:14
const std::unique_ptr< DB > m_db
Definition: txindex.h:20
unsigned char * begin()
Definition: uint256.h:58
bool Init() override
Override base class init to migrate from old database.
Definition: txindex.cpp:200
CDBIterator * NewIterator()
Definition: dbwrapper.h:295
bool IsNull() const
Return true if the wrapped FILE* is nullptr, false otherwise.
Definition: streams.h:647
Access to the txindex database (indexes/txindex/)
Definition: txindex.cpp:22
bool ReadTxPos(const uint256 &txid, CDiskTxPos &pos) const
Read the disk location of the transaction data with the given hash.
Definition: txindex.cpp:43
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1116
The database stores a block locator of the chain the database is synced to so that the index can effi...
Definition: base.h:37
Base class for indices of blockchain data.
Definition: base.h:27
bool FindTx(const uint256 &tx_hash, uint256 &block_hash, CTransactionRef &tx) const
Look up a transaction by hash.
Definition: txindex.cpp:231
Access to the block database (blocks/index/)
Definition: txdb.h:96
bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator)
Migrate txindex data from the block tree DB, where it may be for older nodes that have not been upgra...
Definition: txindex.cpp:75
#define LOCK(cs)
Definition: sync.h:230
std::unique_ptr< TxIndex > g_txindex
The global transaction index, used in GetTransaction. May be null.
Definition: txindex.cpp:17
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:57
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:706
RecursiveMutex cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate...
Definition: validation.cpp:129
void Write(const K &key, const V &value)
Definition: dbwrapper.h:72
BaseIndex::DB & GetDB() const override
Definition: txindex.cpp:229
size_t SizeEstimate() const
Definition: dbwrapper.h:113
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:238
TxIndex(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
Definition: txindex.cpp:194
uint256 GetHash() const
Definition: block.cpp:11
virtual ~TxIndex() override
Definition: txindex.cpp:198
256-bit opaque blob.
Definition: uint256.h:124
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:256
void CompactRange(const K &key_begin, const K &key_end) const
Compact a certain range of keys in the database.
Definition: dbwrapper.h:325
std::vector< CTransactionRef > vtx
Definition: block.h:66
FILE * Get() const
Get wrapped FILE* without transfer of ownership.
Definition: streams.h:643
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:137
bool WriteTxs(const std::vector< std::pair< uint256, CDiskTxPos >> &v_pos)
Write a batch of transaction positions to the DB.
Definition: txindex.cpp:48
FILE * OpenBlockFile(const FlatFilePos &pos, bool fReadOnly)
Open a block file (blk?????.dat)
DB(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Definition: txindex.cpp:39
constexpr char DB_BEST_BLOCK
Definition: txindex.cpp:13
std::unique_ptr< CBlockTreeDB > pblocktree
Global variable that points to the active block tree (protected by cs_main)
Definition: validation.cpp:199
static int count
Definition: tests.c:35
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:234
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:150
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
bool error(const char *fmt, const Args &... args)
Definition: system.h:52
TxIndex is used to look up transactions included in the blockchain by hash.
Definition: txindex.h:17
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:20
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:602