Bitcoin Core  0.20.99
P2P Digital Currency
txindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2017-2019 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <index/txindex.h>
6 #include <node/ui_interface.h>
7 #include <shutdown.h>
8 #include <util/system.h>
9 #include <util/translation.h>
10 #include <validation.h>
11 
12 constexpr char DB_BEST_BLOCK = 'B';
13 constexpr char DB_TXINDEX = 't';
14 constexpr char DB_TXINDEX_BLOCK = 'T';
15 
16 std::unique_ptr<TxIndex> g_txindex;
17 
18 struct CDiskTxPos : public FlatFilePos
19 {
20  unsigned int nTxOffset; // after header
21 
23  {
25  READWRITE(VARINT(obj.nTxOffset));
26  }
27 
28  CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
29  }
30 
32  SetNull();
33  }
34 
35  void SetNull() {
37  nTxOffset = 0;
38  }
39 };
40 
50 class TxIndex::DB : public BaseIndex::DB
51 {
52 public:
53  explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
54 
57  bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
58 
60  bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
61 
64  bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
65 };
66 
67 TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
68  BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
69 {}
70 
71 bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
72 {
73  return Read(std::make_pair(DB_TXINDEX, txid), pos);
74 }
75 
76 bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
77 {
78  CDBBatch batch(*this);
79  for (const auto& tuple : v_pos) {
80  batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
81  }
82  return WriteBatch(batch);
83 }
84 
85 /*
86  * Safely persist a transfer of data from the old txindex database to the new one, and compact the
87  * range of keys updated. This is used internally by MigrateData.
88  */
90  CDBBatch& batch_newdb, CDBBatch& batch_olddb,
91  const std::pair<unsigned char, uint256>& begin_key,
92  const std::pair<unsigned char, uint256>& end_key)
93 {
94  // Sync new DB changes to disk before deleting from old DB.
95  newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
96  olddb.WriteBatch(batch_olddb);
97  olddb.CompactRange(begin_key, end_key);
98 
99  batch_newdb.Clear();
100  batch_olddb.Clear();
101 }
102 
103 bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
104 {
105  // The prior implementation of txindex was always in sync with block index
106  // and presence was indicated with a boolean DB flag. If the flag is set,
107  // this means the txindex from a previous version is valid and in sync with
108  // the chain tip. The first step of the migration is to unset the flag and
109  // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
110  // index entries are copied over in batches to the new database. Finally,
111  // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
112  // written to the new database.
113  //
114  // Unsetting the boolean flag ensures that if the node is downgraded to a
115  // previous version, it will not see a corrupted, partially migrated index
116  // -- it will see that the txindex is disabled. When the node is upgraded
117  // again, the migration will pick up where it left off and sync to the block
118  // with hash DB_TXINDEX_BLOCK.
119  bool f_legacy_flag = false;
120  block_tree_db.ReadFlag("txindex", f_legacy_flag);
121  if (f_legacy_flag) {
122  if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
123  return error("%s: cannot write block indicator", __func__);
124  }
125  if (!block_tree_db.WriteFlag("txindex", false)) {
126  return error("%s: cannot write block index db flag", __func__);
127  }
128  }
129 
130  CBlockLocator locator;
131  if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
132  return true;
133  }
134 
135  int64_t count = 0;
136  LogPrintf("Upgrading txindex database... [0%%]\n");
137  uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0, true);
138  int report_done = 0;
139  const size_t batch_size = 1 << 24; // 16 MiB
140 
141  CDBBatch batch_newdb(*this);
142  CDBBatch batch_olddb(block_tree_db);
143 
144  std::pair<unsigned char, uint256> key;
145  std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
146  std::pair<unsigned char, uint256> prev_key = begin_key;
147 
148  bool interrupted = false;
149  std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
150  for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
151  if (ShutdownRequested()) {
152  interrupted = true;
153  break;
154  }
155 
156  if (!cursor->GetKey(key)) {
157  return error("%s: cannot get key from valid cursor", __func__);
158  }
159  if (key.first != DB_TXINDEX) {
160  break;
161  }
162 
163  // Log progress every 10%.
164  if (++count % 256 == 0) {
165  // Since txids are uniformly random and traversed in increasing order, the high 16 bits
166  // of the hash can be used to estimate the current progress.
167  const uint256& txid = key.second;
168  uint32_t high_nibble =
169  (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
170  (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
171  int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
172 
173  uiInterface.ShowProgress(_("Upgrading txindex database").translated, percentage_done, true);
174  if (report_done < percentage_done/10) {
175  LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
176  report_done = percentage_done/10;
177  }
178  }
179 
180  CDiskTxPos value;
181  if (!cursor->GetValue(value)) {
182  return error("%s: cannot parse txindex record", __func__);
183  }
184  batch_newdb.Write(key, value);
185  batch_olddb.Erase(key);
186 
187  if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
188  // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
189  // because LevelDB iterators are guaranteed to provide a consistent view of the
190  // underlying data, like a lightweight snapshot.
191  WriteTxIndexMigrationBatches(*this, block_tree_db,
192  batch_newdb, batch_olddb,
193  prev_key, key);
194  prev_key = key;
195  }
196  }
197 
198  // If these final DB batches complete the migration, write the best block
199  // hash marker to the new database and delete from the old one. This signals
200  // that the former is fully caught up to that point in the blockchain and
201  // that all txindex entries have been removed from the latter.
202  if (!interrupted) {
203  batch_olddb.Erase(DB_TXINDEX_BLOCK);
204  batch_newdb.Write(DB_BEST_BLOCK, locator);
205  }
206 
207  WriteTxIndexMigrationBatches(*this, block_tree_db,
208  batch_newdb, batch_olddb,
209  begin_key, key);
210 
211  if (interrupted) {
212  LogPrintf("[CANCELLED].\n");
213  return false;
214  }
215 
216  uiInterface.ShowProgress("", 100, false);
217 
218  LogPrintf("[DONE].\n");
219  return true;
220 }
221 
222 TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
223  : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
224 {}
225 
227 
229 {
230  LOCK(cs_main);
231 
232  // Attempt to migrate txindex from the old database to the new one. Even if
233  // chain_tip is null, the node could be reindexing and we still want to
234  // delete txindex records in the old database.
235  if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) {
236  return false;
237  }
238 
239  return BaseIndex::Init();
240 }
241 
242 bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
243 {
244  // Exclude genesis block transaction because outputs are not spendable.
245  if (pindex->nHeight == 0) return true;
246 
247  CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
248  std::vector<std::pair<uint256, CDiskTxPos>> vPos;
249  vPos.reserve(block.vtx.size());
250  for (const auto& tx : block.vtx) {
251  vPos.emplace_back(tx->GetHash(), pos);
253  }
254  return m_db->WriteTxs(vPos);
255 }
256 
257 BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
258 
259 bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
260 {
261  CDiskTxPos postx;
262  if (!m_db->ReadTxPos(tx_hash, postx)) {
263  return false;
264  }
265 
266  CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
267  if (file.IsNull()) {
268  return error("%s: OpenBlockFile failed", __func__);
269  }
270  CBlockHeader header;
271  try {
272  file >> header;
273  if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
274  return error("%s: fseek(...) failed", __func__);
275  }
276  file >> tx;
277  } catch (const std::exception& e) {
278  return error("%s: Deserialize or I/O error - %s", __func__, e.what());
279  }
280  if (tx->GetHash() != tx_hash) {
281  return error("%s: txid mismatch", __func__);
282  }
283  block_hash = header.GetHash();
284  return true;
285 }
std::shared_ptr< const CTransaction > CTransactionRef
Definition: transaction.h:387
#define VARINT(obj)
Definition: serialize.h:468
static void WriteTxIndexMigrationBatches(CDBWrapper &newdb, CDBWrapper &olddb, CDBBatch &batch_newdb, CDBBatch &batch_olddb, const std::pair< unsigned char, uint256 > &begin_key, const std::pair< unsigned char, uint256 > &end_key)
Definition: txindex.cpp:89
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
Definition: txindex.cpp:242
void Clear()
Definition: dbwrapper.h:65
bool ShutdownRequested()
Definition: shutdown.cpp:20
unsigned int nTxOffset
Definition: txindex.cpp:20
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:54
Describes a place in the block chain to another node such that if the other node doesn&#39;t have the sam...
Definition: block.h:114
constexpr char DB_TXINDEX_BLOCK
Definition: txindex.cpp:14
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:46
Definition: block.h:62
CChain & ChainActive()
Please prefer the identical ChainstateManager::ActiveChain.
Definition: validation.cpp:115
void Erase(const K &key)
Definition: dbwrapper.h:97
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 ...
Definition: serialize.h:272
constexpr char DB_TXINDEX
Definition: txindex.cpp:13
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:166
FlatFilePos GetBlockPos() const
Definition: chain.h:202
std::unique_ptr< T > MakeUnique(Args &&... args)
Substitute for C++14 std::make_unique.
Definition: memory.h:14
int nFile
Definition: flatfile.h:16
const std::unique_ptr< DB > m_db
Definition: txindex.h:20
#define READWRITEAS(type, obj)
Definition: serialize.h:172
unsigned char * begin()
Definition: uint256.h:54
bool Init() override
Override base class init to migrate from old database.
Definition: txindex.cpp:228
CDBIterator * NewIterator()
Definition: dbwrapper.h:307
bool IsNull() const
Return true if the wrapped FILE* is nullptr, false otherwise.
Definition: streams.h:646
Access to the txindex database (indexes/txindex/)
Definition: txindex.cpp:50
bool ReadTxPos(const uint256 &txid, CDiskTxPos &pos) const
Read the disk location of the transaction data with the given hash.
Definition: txindex.cpp:71
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1104
Base class for indices of blockchain data.
Definition: base.h:21
bool FindTx(const uint256 &tx_hash, uint256 &block_hash, CTransactionRef &tx) const
Look up a transaction by hash.
Definition: txindex.cpp:259
Access to the block database (blocks/index/)
Definition: txdb.h:88
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn)
Definition: txindex.cpp:28
bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator)
Migrate txindex data from the block tree DB, where it may be for older nodes that have not been upgra...
Definition: txindex.cpp:103
#define LOCK(cs)
Definition: sync.h:226
std::unique_ptr< TxIndex > g_txindex
The global transaction index, used in GetTransaction. May be null.
Definition: txindex.cpp:16
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:57
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:620
RecursiveMutex cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate...
Definition: validation.cpp:131
void SetNull()
Definition: flatfile.h:36
void Write(const K &key, const V &value)
Definition: dbwrapper.h:72
BaseIndex::DB & GetDB() const override
Definition: txindex.cpp:257
size_t SizeEstimate() const
Definition: dbwrapper.h:113
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:227
TxIndex(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
Definition: txindex.cpp:222
uint256 GetHash() const
Definition: block.cpp:11
virtual ~TxIndex() override
Definition: txindex.cpp:226
256-bit opaque blob.
Definition: uint256.h:120
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:256
void CompactRange(const K &key_begin, const K &key_end) const
Compact a certain range of keys in the database.
Definition: dbwrapper.h:337
std::vector< CTransactionRef > vtx
Definition: block.h:66
FILE * Get() const
Get wrapped FILE* without transfer of ownership.
Definition: streams.h:642
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:137
bool WriteTxs(const std::vector< std::pair< uint256, CDiskTxPos >> &v_pos)
Write a batch of transaction positions to the DB.
Definition: txindex.cpp:76
FILE * OpenBlockFile(const FlatFilePos &pos, bool fReadOnly)
Open a block file (blk?????.dat)
DB(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Definition: txindex.cpp:67
constexpr char DB_BEST_BLOCK
Definition: txindex.cpp:12
std::unique_ptr< CBlockTreeDB > pblocktree
Global variable that points to the active block tree (protected by cs_main)
Definition: validation.cpp:203
static int count
Definition: tests.c:35
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:223
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:150
void SetNull()
Definition: txindex.cpp:35
#define READWRITE(...)
Definition: serialize.h:171
CDiskTxPos()
Definition: txindex.cpp:31
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
unsigned int nPos
Definition: flatfile.h:17
bool error(const char *fmt, const Args &... args)
Definition: system.h:49
TxIndex is used to look up transactions included in the blockchain by hash.
Definition: txindex.h:17
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:20
SERIALIZE_METHODS(CDiskTxPos, obj)
Definition: txindex.cpp:22
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:601