Bitcoin Core 32.99.0
P2P Digital Currency
Loading...
Searching...
No Matches
txindex.cpp
Go to the documentation of this file.
1// Copyright (c) 2017-present The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5#include <index/txindex.h>
6
7#include <chain.h>
8#include <common/args.h>
9#include <crypto/siphash.h>
10#include <dbwrapper.h>
11#include <flatfile.h>
12#include <index/base.h>
13#include <index/disktxpos.h>
14#include <index/txindex_key.h>
15#include <interfaces/chain.h>
16#include <node/blockstorage.h>
17#include <primitives/block.h>
19#include <random.h>
20#include <serialize.h>
21#include <streams.h>
22#include <sync.h>
23#include <tinyformat.h>
24#include <uint256.h>
25#include <util/fs.h>
26#include <util/log.h>
27#include <util/string.h>
28#include <validation.h>
29
30#include <algorithm>
31#include <array>
32#include <cassert>
33#include <cstdint>
34#include <cstdio>
35#include <exception>
36#include <functional>
37#include <memory>
38#include <optional>
39#include <stdexcept>
40#include <string>
41#include <utility>
42#include <vector>
43
44std::unique_ptr<TxIndex> g_txindex;
45
46namespace {
48{
49 std::pair<uint64_t, uint64_t> salt;
52 salt = {rng.rand64(), rng.rand64()};
53 db.Write(txindex::DB_TXID_HASH_SALT, salt, /*fSync=*/true);
54 }
55 return SipHasher13UJ{salt.first, salt.second};
56}
57} // namespace
58
61{
62public:
63 explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
64
66 void WriteTxs(const interfaces::BlockInfo& block);
67
70
72 const bool m_has_legacy;
73
74 CBlockLocator ReadBestBlock() const override;
75 void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator) override;
76
77private:
78 DB(size_t n_cache_size, bool f_memory, bool f_wipe, bool has_legacy);
79};
80
81static fs::path TxIndexDBPath() { return gArgs.GetDataDirNet() / "indexes" / "txindex"; }
82
84 // Bloom filters are built for every key but only consulted by point reads,
85 // which iterators bypass: the per-tx hashed ('x') lookups seek with an
86 // iterator, and the 's'/'h' point reads are at most one per block against a
87 // tiny keyspace. Only the legacy entries' per-tx point lookups benefit, so
88 // enable the filters only for databases still containing them.
90 /*has_legacy=*/!f_memory && !f_wipe && CDBWrapper::HasKeyStartingWith(TxIndexDBPath(), txindex::DB_TXINDEX))
91{}
92
94 BaseIndex::DB(TxIndexDBPath(), n_cache_size, f_memory, f_wipe, /*f_obfuscate=*/false, /*f_bloom=*/has_legacy),
95 m_hasher{ReadOrCreateTxidHasher(*this)},
96 m_has_legacy{has_legacy}
97{}
98
100{
101 CBlockLocator locator;
102 if (Read(txindex::DB_BEST_BLOCK_V2, locator)) {
103 return locator;
104 }
105 // If we don't have a locator yet, start from the legacy best block.
107}
108
110{
111 batch.Write(txindex::DB_BEST_BLOCK_V2, locator);
112}
113
115{
116 // A block may be submitted again after it was already indexed, e.g. when it
117 // reconnects after a reorg or is re-processed after an unclean shutdown. It
118 // keeps its original sequence number, so skip it to avoid duplicate entries.
119 if (Exists(txindex::BlockHashKey{block.hash})) return;
120
121 uint32_t block_seq{0};
123
124 CDBBatch batch(*this);
125 batch.Write(txindex::BlockHashKey{block.hash}, block_seq);
126 batch.Write(txindex::BlockSeqKey{block_seq}, block.hash);
127 batch.Write(txindex::DB_NEXT_BLOCK_SEQ, block_seq + 1);
128 uint32_t tx_offset_in_block{txindex::BLOCK_HEADER_SIZE + GetSizeOfCompactSize(block.data->vtx.size())};
129 for (const auto& tx : block.data->vtx) {
130 const txindex::DBKey key{txindex::CreateKeyPrefix(m_hasher, tx->GetHash()),
131 txindex::BlockTxPosition{block_seq, tx_offset_in_block}};
132 batch.Write(key, txindex::EMPTY_VALUE);
133 tx_offset_in_block += tx->ComputeTotalSize();
134 }
135 WriteBatch(batch);
136}
137
138TxIndex::TxIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe)
139 : BaseIndex(std::move(chain), "txindex", "txidx"), m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
140{
141 if (m_db->m_has_legacy) {
142 LogInfo("txindex contains entries in the legacy format, which uses excessive disk space. "
143 "To reclaim disk space, stop the node, delete %s and restart to rebuild the index.",
145 }
146}
147
148TxIndex::~TxIndex() = default;
149
151{
152 // Exclude genesis block transaction because outputs are not spendable.
153 if (block.height == 0) return true;
154
155 assert(block.data);
156 m_db->WriteTxs(block);
157 return true;
158}
159
160BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
161
162template <typename... Args>
163[[noreturn]] static void WarnAndThrowErr(util::ConstevalFormatString<sizeof...(Args)> fmt, const Args&... args)
164{
165 const std::string err{tfm::format(fmt, args...)};
166 LogWarning("%s", err);
167 throw std::runtime_error{err};
168}
169
170std::optional<TxIndexResult> TxIndex::FindTx(const Txid& tx_hash) const
171{
172 struct Candidate {
174 uint256 block_hash;
175 uint32_t block_seq;
179 bool in_active_chain;
180 };
181 std::vector<Candidate> candidates;
182 {
183 std::unique_ptr<CDBIterator> it{m_db->NewIterator()};
185 txindex::DBKey key{prefix, {}};
186 for (it->Seek(key); it->Valid() && it->GetKey(key) && key.hash_prefix == prefix; it->Next()) {
188 if (!m_db->Read(txindex::BlockSeqKey{key.pos.block_seq}, candidate_block_hash)) {
189 // Read may throw dbwrapper_error.
190 // Read returns false for internal deserialize errors or a
191 // missing key. A missing key can only happen after corruption.
192 WarnAndThrowErr("%s: Corrupt index: block sequence %u not found for txid %s",
193 GetName(), key.pos.block_seq, tx_hash.ToString());
194 }
195 LOCK(cs_main);
197 // Stale entries can remain after an unclean shutdown. Unknown blocks
198 // cannot be in the active chain, so skip them.
199 if (!block_index) continue;
201 if (!(block_index->nStatus & BLOCK_HAVE_DATA)) {
202 // A stale entry may also refer to a known block without data.
203 // Skip it if it is not in the active chain.
204 if (!in_active_chain) continue;
205 WarnAndThrowErr("%s: Block data %s unexpectedly unavailable for txid %s. Caused by data corruption?",
206 GetName(), candidate_block_hash.ToString(), tx_hash.ToString());
207 }
208 const FlatFilePos tx_position{block_index->nFile, block_index->nDataPos + key.pos.tx_offset_in_block};
209 candidates.emplace_back(tx_position, candidate_block_hash, key.pos.block_seq, in_active_chain);
210 }
211 }
212
213 // Prefer active-chain matches, then later-connected blocks.
214 std::ranges::sort(candidates, std::greater{}, [](const Candidate& c) { return std::pair{c.in_active_chain, c.block_seq}; });
215
216 for (const auto& candidate : candidates) {
217 AutoFile file{m_chainstate->m_blockman.OpenBlockFile(candidate.tx_position, /*fReadOnly=*/true)};
218 if (file.IsNull()) {
219 WarnAndThrowErr("%s: Failed to open block file for txid %s at %s in block %s",
220 GetName(), tx_hash.ToString(), candidate.tx_position.ToString(), candidate.block_hash.ToString());
221 }
223 try {
224 file >> TX_WITH_WITNESS(tx);
225 } catch (const std::exception& e) {
226 WarnAndThrowErr("%s: Deserialize or I/O error (%s). txid %s at %s in block %s",
227 GetName(), e.what(), tx_hash.ToString(), candidate.tx_position.ToString(), candidate.block_hash.ToString());
228 }
229 if (tx->GetHash() == tx_hash) {
230 return TxIndexResult{candidate.block_hash, std::move(tx)};
231 }
232 }
233 // Fall back to legacy if no hashed entry matched. This makes misses pay an
234 // extra lookup, but keeps existing full-txid entries readable after upgrade.
235 return m_db->m_has_legacy ? FindLegacyTx(tx_hash) : std::nullopt;
236}
237
238std::optional<TxIndexResult> TxIndex::FindLegacyTx(const Txid& tx_hash) const
239{
241 const auto read{m_db->TryRead(txindex::LegacyTxKey(tx_hash), postx)};
242 if (!read) {
243 WarnAndThrowErr("%s: Failed to read legacy entry for txid %s: %s",
244 GetName(), tx_hash.ToString(), read.error().err_msg);
245 }
246 if (!read.value()) {
247 return std::nullopt;
248 }
249
250 AutoFile file{m_chainstate->m_blockman.OpenBlockFile(postx, /*fReadOnly=*/true)};
251 if (file.IsNull()) {
252 WarnAndThrowErr("%s: Failed to open block file for txid %s at %s",
253 GetName(), tx_hash.ToString(), postx.ToString());
254 }
255 CBlockHeader header;
257 try {
258 file >> header;
259 file.seek(postx.nTxOffset, SEEK_CUR);
260 file >> TX_WITH_WITNESS(tx);
261 } catch (const std::exception& e) {
262 WarnAndThrowErr("%s: Deserialize or I/O error (%s). txid %s at %s",
263 GetName(), e.what(), tx_hash.ToString(), postx.ToString());
264 }
265 if (tx->GetHash() != tx_hash) {
266 WarnAndThrowErr("%s: txid mismatch in legacy index. txid %s at %s",
267 GetName(), tx_hash.ToString(), postx.ToString());
268 }
269 return TxIndexResult{header.GetHash(), std::move(tx)};
270}
ArgsManager gArgs
Definition args.cpp:39
ArgsManager & args
Definition bitcoind.cpp:280
@ BLOCK_HAVE_DATA
full block available in blk*.dat
Definition chain.h:75
fs::path GetDataDirNet() const EXCLUSIVE_LOCKS_REQUIRED(!cs_args)
Get data directory path with appended network identifier.
Definition args.cpp:329
Non-refcounted RAII wrapper for FILE*.
Definition streams.h:395
The database stores a block locator of the chain the database is synced to so that the index can effi...
Definition base.h:65
virtual CBlockLocator ReadBestBlock() const
Read block locator of the chain that the index is in sync with.
Definition base.cpp:78
Base class for indices of blockchain data.
Definition base.h:55
const std::string & GetName() const LIFETIMEBOUND
Get the name of the index for display in logs.
Definition base.h:147
Chainstate * m_chainstate
Definition base.h:115
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition block.h:27
uint256 GetHash() const
Definition block.cpp:14
std::vector< CTransactionRef > vtx
Definition block.h:77
The block chain is a tree shaped structure starting with the genesis block at the root,...
Definition chain.h:94
bool Contains(const CBlockIndex &index) const
Efficiently check whether a block is present in this chain.
Definition chain.h:410
Batch of changes queued to be written to a CDBWrapper.
Definition dbwrapper.h:90
void Write(const K &key, const V &value)
Definition dbwrapper.h:114
bool Read(const K &key, V &value) const
Wrapper around TryRead() that preserves the original Read() semantics: returns true on success,...
Definition dbwrapper.h:288
void Write(const K &key, const V &value, bool fSync=false)
Definition dbwrapper.h:300
CChain m_chain
The current chain of blockheaders we consult and build on.
Definition validation.h:630
node::BlockManager & m_blockman
Reference to a BlockManager instance which itself is shared across all Chainstate instances.
Definition validation.h:583
Fast randomness source.
Definition random.h:386
uint64_t rand64() noexcept
Generate a random 64-bit integer.
Definition random.h:404
A custom weaker variant of SipHash-1-3 without padding, and supporting "jumbo" inputs.
Definition siphash.h:161
Access to the txindex database (indexes/txindex/)
Definition txindex.cpp:61
const bool m_has_legacy
Whether the database contains any legacy ('t' + txid) entries.
Definition txindex.cpp:72
DB(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Definition txindex.cpp:83
void WriteBestBlock(CDBBatch &batch, const CBlockLocator &locator) override
Write block locator of the chain that the index is in sync with.
Definition txindex.cpp:109
const SipHasher13UJ m_hasher
Used to hash the txid to compute the prefix.
Definition txindex.cpp:69
CBlockLocator ReadBestBlock() const override
Read block locator of the chain that the index is in sync with.
Definition txindex.cpp:99
void WriteTxs(const interfaces::BlockInfo &block)
Write a block of transaction positions to the DB.
Definition txindex.cpp:114
TxIndex is used to look up transactions included in the blockchain by hash.
Definition txindex.h:37
BaseIndex::DB & GetDB() const override
Definition txindex.cpp:160
std::optional< TxIndexResult > FindTx(const Txid &tx_hash) const
Look up a transaction by hash.
Definition txindex.cpp:170
bool CustomAppend(const interfaces::BlockInfo &block) override
Write update index entries for a newly connected block.
Definition txindex.cpp:150
TxIndex(std::unique_ptr< interfaces::Chain > chain, size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
Definition txindex.cpp:138
virtual ~TxIndex() override
const std::unique_ptr< DB > m_db
Definition txindex.h:43
std::optional< TxIndexResult > FindLegacyTx(const Txid &tx_hash) const
Look up a transaction among the legacy (full-txid) entries.
Definition txindex.cpp:238
CBlockIndex * LookupBlockIndex(const uint256 &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
bool IsBlockPruned(const CBlockIndex &block) const EXCLUSIVE_LOCKS_REQUIRED(void UpdatePruneLock(const std::string &name, const PruneLockInfo &lock_info) EXCLUSIVE_LOCKS_REQUIRED(bool DeletePruneLock(const std::string &name) EXCLUSIVE_LOCKS_REQUIRED(AutoFile OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) const
Check whether the block associated with this index entry is pruned or not.
256-bit opaque blob.
Definition uint256.h:196
RecursiveMutex cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate.
Definition cs_main.cpp:8
static std::string PathToString(const path &path)
Convert path object to a byte string.
Definition fs.h:160
#define LogWarning(...)
Definition log.h:137
#define LogInfo(...)
Definition log.h:136
void format(std::ostream &out, FormatStringCheck< sizeof...(Args)> fmt, const Args &... args)
Format list of arguments to the stream according to given format string.
const std::string DB_TXID_HASH_SALT
Definition txindex_key.h:39
constexpr std::array< std::byte, 0 > EMPTY_VALUE
Empty value of a hashed txindex row, whose position is encoded in its key.
Definition txindex_key.h:45
TxHashKeyPrefix CreateKeyPrefix(const SipHasher13UJ &hasher, const Txid &txid)
constexpr uint32_t BLOCK_HEADER_SIZE
Serialized size of a block header, the offset of the first byte after it.
Definition txindex_key.h:48
const std::string DB_NEXT_BLOCK_SEQ
Definition txindex_key.h:38
uint64_t TxHashKeyPrefix
std::pair< uint8_t, uint256 > LegacyTxKey(const Txid &txid)
Key of a legacy (pre-hashing) txindex row: the full txid under the 't' prefix.
const std::string DB_BEST_BLOCK_V2
Definition txindex_key.h:40
constexpr TransactionSerParams TX_WITH_WITNESS
std::shared_ptr< const CTransaction > CTransactionRef
const char * prefix
Definition rest.cpp:1201
constexpr unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition serialize.h:291
Describes a place in the block chain to another node such that if the other node doesn't have the sam...
Definition block.h:117
int32_t nFile
Definition flatfile.h:16
A found transaction and the hash of the block that contains it.
Definition txindex.h:26
uint256 block_hash
Definition txindex.h:27
Block data sent with blockConnected, blockDisconnected notifications.
Definition chain.h:20
const CBlock * data
Definition chain.h:26
const uint256 & hash
Definition chain.h:21
Key for looking up the sequence number assigned to the block with the given hash.
Definition txindex_key.h:87
Key for looking up the hash of the block with the given sequence number.
Definition txindex_key.h:74
The location of a transaction: the sequence number of the block that contains it and the transaction'...
Definition txindex_key.h:55
A wrapper for a compile-time partially validated format string.
Definition string.h:96
#define LOCK(cs)
Definition sync.h:268
CDBWrapper db
FastRandomContext rng
static fs::path TxIndexDBPath()
Definition txindex.cpp:81
static void WarnAndThrowErr(util::ConstevalFormatString< sizeof...(Args)> fmt, const Args &... args)
Definition txindex.cpp:163
std::unique_ptr< TxIndex > g_txindex
The global transaction index, used in GetTransaction. May be null.
Definition txindex.cpp:44
constexpr auto Ticks(Dur2 d)
Helper to count the seconds of a duration/time_point.
Definition time.h:82
assert(!tx.IsCoinBase())