Bitcoin Core  0.20.99
P2P Digital Currency
blockfilterindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <map>
6 
7 #include <dbwrapper.h>
9 #include <util/system.h>
10 #include <validation.h>
11 
12 /* The index database stores three items for each block: the disk location of the encoded filter,
13  * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by
14  * height, and those belonging to blocks that have been reorganized out of the active chain are
15  * indexed by block hash. This ensures that filter data for any block that becomes part of the
16  * active chain can always be retrieved, alleviating timing concerns.
17  *
18  * The filters themselves are stored in flat files and referenced by the LevelDB entries. This
19  * minimizes the amount of data written to LevelDB and keeps the database values constant size. The
20  * disk location of the next block filter to be written (represented as a FlatFilePos) is stored
21  * under the DB_FILTER_POS key.
22  *
23  * Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented
24  * as big-endian so that sequential reads of filters by height are fast.
25  * Keys for the hash index have the type [DB_BLOCK_HASH, uint256].
26  */
27 constexpr char DB_BLOCK_HASH = 's';
28 constexpr char DB_BLOCK_HEIGHT = 't';
29 constexpr char DB_FILTER_POS = 'P';
30 
31 constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
33 constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB
39 constexpr size_t CF_HEADERS_CACHE_MAX_SZ{2000};
40 
41 namespace {
42 
43 struct DBVal {
44  uint256 hash;
45  uint256 header;
46  FlatFilePos pos;
47 
48  SERIALIZE_METHODS(DBVal, obj) { READWRITE(obj.hash, obj.header, obj.pos); }
49 };
50 
51 struct DBHeightKey {
52  int height;
53 
54  DBHeightKey() : height(0) {}
55  explicit DBHeightKey(int height_in) : height(height_in) {}
56 
57  template<typename Stream>
58  void Serialize(Stream& s) const
59  {
61  ser_writedata32be(s, height);
62  }
63 
64  template<typename Stream>
65  void Unserialize(Stream& s)
66  {
67  char prefix = ser_readdata8(s);
68  if (prefix != DB_BLOCK_HEIGHT) {
69  throw std::ios_base::failure("Invalid format for block filter index DB height key");
70  }
71  height = ser_readdata32be(s);
72  }
73 };
74 
75 struct DBHashKey {
76  uint256 hash;
77 
78  explicit DBHashKey(const uint256& hash_in) : hash(hash_in) {}
79 
80  SERIALIZE_METHODS(DBHashKey, obj) {
81  char prefix = DB_BLOCK_HASH;
82  READWRITE(prefix);
83  if (prefix != DB_BLOCK_HASH) {
84  throw std::ios_base::failure("Invalid format for block filter index DB hash key");
85  }
86 
87  READWRITE(obj.hash);
88  }
89 };
90 
91 }; // namespace
92 
93 static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
94 
96  size_t n_cache_size, bool f_memory, bool f_wipe)
97  : m_filter_type(filter_type)
98 {
99  const std::string& filter_name = BlockFilterTypeName(filter_type);
100  if (filter_name.empty()) throw std::invalid_argument("unknown filter_type");
101 
102  fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name;
103  fs::create_directories(path);
104 
105  m_name = filter_name + " block filter index";
106  m_db = MakeUnique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
107  m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE);
108 }
109 
111 {
112  if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) {
113  // Check that the cause of the read failure is that the key does not exist. Any other errors
114  // indicate database corruption or a disk failure, and starting the index would cause
115  // further corruption.
116  if (m_db->Exists(DB_FILTER_POS)) {
117  return error("%s: Cannot read current %s state; index may be corrupted",
118  __func__, GetName());
119  }
120 
121  // If the DB_FILTER_POS is not set, then initialize to the first location.
124  }
125  return BaseIndex::Init();
126 }
127 
129 {
130  const FlatFilePos& pos = m_next_filter_pos;
131 
132  // Flush current filter file to disk.
134  if (file.IsNull()) {
135  return error("%s: Failed to open filter file %d", __func__, pos.nFile);
136  }
137  if (!FileCommit(file.Get())) {
138  return error("%s: Failed to commit filter file %d", __func__, pos.nFile);
139  }
140 
141  batch.Write(DB_FILTER_POS, pos);
142  return BaseIndex::CommitInternal(batch);
143 }
144 
146 {
147  CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION);
148  if (filein.IsNull()) {
149  return false;
150  }
151 
152  uint256 block_hash;
153  std::vector<unsigned char> encoded_filter;
154  try {
155  filein >> block_hash >> encoded_filter;
156  filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
157  }
158  catch (const std::exception& e) {
159  return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what());
160  }
161 
162  return true;
163 }
164 
166 {
167  assert(filter.GetFilterType() == GetFilterType());
168 
169  size_t data_size =
172 
173  // If writing the filter would overflow the file, flush and move to the next one.
174  if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
175  CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
176  if (last_file.IsNull()) {
177  LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
178  return 0;
179  }
180  if (!TruncateFile(last_file.Get(), pos.nPos)) {
181  LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile);
182  return 0;
183  }
184  if (!FileCommit(last_file.Get())) {
185  LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile);
186  return 0;
187  }
188 
189  pos.nFile++;
190  pos.nPos = 0;
191  }
192 
193  // Pre-allocate sufficient space for filter data.
194  bool out_of_space;
195  m_filter_fileseq->Allocate(pos, data_size, out_of_space);
196  if (out_of_space) {
197  LogPrintf("%s: out of disk space\n", __func__);
198  return 0;
199  }
200 
201  CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
202  if (fileout.IsNull()) {
203  LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
204  return 0;
205  }
206 
207  fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
208  return data_size;
209 }
210 
211 bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
212 {
213  CBlockUndo block_undo;
214  uint256 prev_header;
215 
216  if (pindex->nHeight > 0) {
217  if (!UndoReadFromDisk(block_undo, pindex)) {
218  return false;
219  }
220 
221  std::pair<uint256, DBVal> read_out;
222  if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
223  return false;
224  }
225 
226  uint256 expected_block_hash = pindex->pprev->GetBlockHash();
227  if (read_out.first != expected_block_hash) {
228  return error("%s: previous block header belongs to unexpected block %s; expected %s",
229  __func__, read_out.first.ToString(), expected_block_hash.ToString());
230  }
231 
232  prev_header = read_out.second.header;
233  }
234 
235  BlockFilter filter(m_filter_type, block, block_undo);
236 
237  size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
238  if (bytes_written == 0) return false;
239 
240  std::pair<uint256, DBVal> value;
241  value.first = pindex->GetBlockHash();
242  value.second.hash = filter.GetHash();
243  value.second.header = filter.ComputeHeader(prev_header);
244  value.second.pos = m_next_filter_pos;
245 
246  if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) {
247  return false;
248  }
249 
250  m_next_filter_pos.nPos += bytes_written;
251  return true;
252 }
253 
255  const std::string& index_name,
256  int start_height, int stop_height)
257 {
258  DBHeightKey key(start_height);
259  db_it.Seek(key);
260 
261  for (int height = start_height; height <= stop_height; ++height) {
262  if (!db_it.GetKey(key) || key.height != height) {
263  return error("%s: unexpected key in %s: expected (%c, %d)",
264  __func__, index_name, DB_BLOCK_HEIGHT, height);
265  }
266 
267  std::pair<uint256, DBVal> value;
268  if (!db_it.GetValue(value)) {
269  return error("%s: unable to read value in %s at key (%c, %d)",
270  __func__, index_name, DB_BLOCK_HEIGHT, height);
271  }
272 
273  batch.Write(DBHashKey(value.first), std::move(value.second));
274 
275  db_it.Next();
276  }
277  return true;
278 }
279 
280 bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
281 {
282  assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
283 
284  CDBBatch batch(*m_db);
285  std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
286 
287  // During a reorg, we need to copy all filters for blocks that are getting disconnected from the
288  // height index to the hash index so we can still find them when the height index entries are
289  // overwritten.
290  if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) {
291  return false;
292  }
293 
294  // The latest filter position gets written in Commit by the call to the BaseIndex::Rewind.
295  // But since this creates new references to the filter, the position should get updated here
296  // atomically as well in case Commit fails.
297  batch.Write(DB_FILTER_POS, m_next_filter_pos);
298  if (!m_db->WriteBatch(batch)) return false;
299 
300  return BaseIndex::Rewind(current_tip, new_tip);
301 }
302 
303 static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result)
304 {
305  // First check if the result is stored under the height index and the value there matches the
306  // block hash. This should be the case if the block is on the active chain.
307  std::pair<uint256, DBVal> read_out;
308  if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) {
309  return false;
310  }
311  if (read_out.first == block_index->GetBlockHash()) {
312  result = std::move(read_out.second);
313  return true;
314  }
315 
316  // If value at the height index corresponds to an different block, the result will be stored in
317  // the hash index.
318  return db.Read(DBHashKey(block_index->GetBlockHash()), result);
319 }
320 
321 static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start_height,
322  const CBlockIndex* stop_index, std::vector<DBVal>& results)
323 {
324  if (start_height < 0) {
325  return error("%s: start height (%d) is negative", __func__, start_height);
326  }
327  if (start_height > stop_index->nHeight) {
328  return error("%s: start height (%d) is greater than stop height (%d)",
329  __func__, start_height, stop_index->nHeight);
330  }
331 
332  size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1);
333  std::vector<std::pair<uint256, DBVal>> values(results_size);
334 
335  DBHeightKey key(start_height);
336  std::unique_ptr<CDBIterator> db_it(db.NewIterator());
337  db_it->Seek(DBHeightKey(start_height));
338  for (int height = start_height; height <= stop_index->nHeight; ++height) {
339  if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) {
340  return false;
341  }
342 
343  size_t i = static_cast<size_t>(height - start_height);
344  if (!db_it->GetValue(values[i])) {
345  return error("%s: unable to read value in %s at key (%c, %d)",
346  __func__, index_name, DB_BLOCK_HEIGHT, height);
347  }
348 
349  db_it->Next();
350  }
351 
352  results.resize(results_size);
353 
354  // Iterate backwards through block indexes collecting results in order to access the block hash
355  // of each entry in case we need to look it up in the hash index.
356  for (const CBlockIndex* block_index = stop_index;
357  block_index && block_index->nHeight >= start_height;
358  block_index = block_index->pprev) {
359  uint256 block_hash = block_index->GetBlockHash();
360 
361  size_t i = static_cast<size_t>(block_index->nHeight - start_height);
362  if (block_hash == values[i].first) {
363  results[i] = std::move(values[i].second);
364  continue;
365  }
366 
367  if (!db.Read(DBHashKey(block_hash), results[i])) {
368  return error("%s: unable to read value in %s at key (%c, %s)",
369  __func__, index_name, DB_BLOCK_HASH, block_hash.ToString());
370  }
371  }
372 
373  return true;
374 }
375 
376 bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const
377 {
378  DBVal entry;
379  if (!LookupOne(*m_db, block_index, entry)) {
380  return false;
381  }
382 
383  return ReadFilterFromDisk(entry.pos, filter_out);
384 }
385 
386 bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out)
387 {
389 
390  bool is_checkpoint{block_index->nHeight % CFCHECKPT_INTERVAL == 0};
391 
392  if (is_checkpoint) {
393  // Try to find the block in the headers cache if this is a checkpoint height.
394  auto header = m_headers_cache.find(block_index->GetBlockHash());
395  if (header != m_headers_cache.end()) {
396  header_out = header->second;
397  return true;
398  }
399  }
400 
401  DBVal entry;
402  if (!LookupOne(*m_db, block_index, entry)) {
403  return false;
404  }
405 
406  if (is_checkpoint &&
407  m_headers_cache.size() < CF_HEADERS_CACHE_MAX_SZ) {
408  // Add to the headers cache if this is a checkpoint height.
409  m_headers_cache.emplace(block_index->GetBlockHash(), entry.header);
410  }
411 
412  header_out = entry.header;
413  return true;
414 }
415 
416 bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index,
417  std::vector<BlockFilter>& filters_out) const
418 {
419  std::vector<DBVal> entries;
420  if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
421  return false;
422  }
423 
424  filters_out.resize(entries.size());
425  auto filter_pos_it = filters_out.begin();
426  for (const auto& entry : entries) {
427  if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
428  return false;
429  }
430  ++filter_pos_it;
431  }
432 
433  return true;
434 }
435 
436 bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
437  std::vector<uint256>& hashes_out) const
438 
439 {
440  std::vector<DBVal> entries;
441  if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
442  return false;
443  }
444 
445  hashes_out.clear();
446  hashes_out.reserve(entries.size());
447  for (const auto& entry : entries) {
448  hashes_out.push_back(entry.hash);
449  }
450  return true;
451 }
452 
454 {
455  auto it = g_filter_indexes.find(filter_type);
456  return it != g_filter_indexes.end() ? &it->second : nullptr;
457 }
458 
459 void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn)
460 {
461  for (auto& entry : g_filter_indexes) fn(entry.second);
462 }
463 
465  size_t n_cache_size, bool f_memory, bool f_wipe)
466 {
467  auto result = g_filter_indexes.emplace(std::piecewise_construct,
468  std::forward_as_tuple(filter_type),
469  std::forward_as_tuple(filter_type,
470  n_cache_size, f_memory, f_wipe));
471  return result.second;
472 }
473 
475 {
476  return g_filter_indexes.erase(filter_type);
477 }
478 
480 {
481  g_filter_indexes.clear();
482 }
bool GetKey(K &key)
Definition: dbwrapper.h:146
bool LookupFilter(const CBlockIndex *block_index, BlockFilter &filter_out) const
Get a single filter by block.
BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of bloc...
constexpr char DB_BLOCK_HEIGHT
bool FileCommit(FILE *file)
Definition: system.cpp:929
constexpr char DB_BLOCK_HASH
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:54
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:85
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:144
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:46
Definition: block.h:62
static bool LookupRange(CDBWrapper &db, const std::string &index_name, int start_height, const CBlockIndex *stop_index, std::vector< DBVal > &results)
BlockFilterIndex * GetBlockFilterIndex(BlockFilterType filter_type)
Get a block filter index by type.
const char * prefix
Definition: rest.cpp:648
void ForEachBlockFilterIndex(std::function< void(BlockFilterIndex &)> fn)
Iterate over all running block filter indexes, invoking fn on each.
constexpr size_t CF_HEADERS_CACHE_MAX_SZ
Maximum size of the cfheaders cache We have a limit to prevent a bug in filling this cache potentiall...
constexpr unsigned int MAX_FLTR_FILE_SIZE
bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) override
Rewind index to an earlier chain tip during a chain reorg.
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:166
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
bool InitBlockFilterIndex(BlockFilterType filter_type, size_t n_cache_size, bool f_memory, bool f_wipe)
Initialize a block filter index for the given type if one does not already exist. ...
virtual bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip)
Rewind index to an earlier chain tip during a chain reorg.
Definition: base.cpp:174
int nFile
Definition: flatfile.h:16
std::unique_ptr< BaseIndex::DB > m_db
CDBIterator * NewIterator()
Definition: dbwrapper.h:295
uint256 GetHash() const
Compute the filter hash.
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1104
void Serialize(Stream &s, char a)
Definition: serialize.h:223
constexpr char DB_FILTER_POS
bool GetValue(V &value)
Definition: dbwrapper.h:157
uint256 GetBlockHash() const
Definition: chain.h:233
BlockFilterType
Definition: blockfilter.h:88
static bool CopyHeightIndexToHashIndex(CDBIterator &db_it, CDBBatch &batch, const std::string &index_name, int start_height, int stop_height)
const std::vector< unsigned char > & GetEncodedFilter() const
Definition: blockfilter.h:134
bool ReadFilterFromDisk(const FlatFilePos &pos, BlockFilter &filter) const
size_t WriteFilterToDisk(FlatFilePos &pos, const BlockFilter &filter)
#define LOCK(cs)
Definition: sync.h:226
Complete block filter struct as defined in BIP 157.
Definition: blockfilter.h:110
bool UndoReadFromDisk(CBlockUndo &blockundo, const CBlockIndex *pindex)
std::string m_name
BlockFilterIndex(BlockFilterType filter_type, size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
void DestroyAllBlockFilterIndexes()
Destroy all open block filter indexes.
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:620
constexpr unsigned int FLTR_FILE_CHUNK_SIZE
The pre-allocation chunk size for fltr?????.dat files.
void Write(const K &key, const V &value)
Definition: dbwrapper.h:72
const uint256 & GetBlockHash() const
Definition: blockfilter.h:131
bool LookupFilterHashRange(int start_height, const CBlockIndex *stop_index, std::vector< uint256 > &hashes_out) const
Get a range of filter hashes between two heights on a chain.
virtual bool CommitInternal(CDBBatch &batch)
Virtual method called internally by Commit that can be overridden to atomically commit more index sta...
Definition: base.cpp:167
std::string ToString() const
Definition: uint256.cpp:60
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:75
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
#define SERIALIZE_METHODS(cls, obj)
Implement the Serialize and Unserialize methods by delegating to a single templated static method tha...
Definition: serialize.h:207
BlockFilterType m_filter_type
static constexpr int CFCHECKPT_INTERVAL
Interval between compact filter checkpoints.
uint256 ComputeHeader(const uint256 &prev_header) const
Compute the filter header given the previous one.
void Next()
Definition: dbwrapper.cpp:239
256-bit opaque blob.
Definition: uint256.h:120
bool TruncateFile(FILE *file, unsigned int length)
Definition: system.cpp:962
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:137
bool DestroyBlockFilterIndex(BlockFilterType filter_type)
Destroy the block filter index with the given type.
Undo information for a CBlock.
Definition: undo.h:63
FlatFilePos m_next_filter_pos
BlockFilterType GetFilterType() const
bool LookupFilterHeader(const CBlockIndex *block_index, uint256 &header_out)
Get a single filter header by block.
static std::map< BlockFilterType, BlockFilterIndex > g_filter_indexes
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:109
void Seek(const K &key)
Definition: dbwrapper.h:136
void Unserialize(Stream &s, char &a)
Definition: serialize.h:241
bool CommitInternal(CDBBatch &batch) override
Virtual method called internally by Commit that can be overridden to atomically commit more index sta...
std::unique_ptr< FlatFileSeq > m_filter_fileseq
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:150
CBlockIndex * GetAncestor(int height)
Efficiently find an ancestor of this block.
Definition: chain.cpp:111
#define READWRITE(...)
Definition: serialize.h:171
bool Init() override
Initialize internal state from the database and block index.
BlockFilterType GetFilterType() const
Definition: blockfilter.h:130
auto it
Definition: validation.cpp:384
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
unsigned int nPos
Definition: flatfile.h:17
bool LookupFilterRange(int start_height, const CBlockIndex *stop_index, std::vector< BlockFilter > &filters_out) const
Get a range of filters between two heights on a chain.
bool error(const char *fmt, const Args &... args)
Definition: system.h:49
static bool LookupOne(const CDBWrapper &db, const CBlockIndex *block_index, DBVal &result)
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:601
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:56
const char * GetName() const override
Get the name of the index for display in logs.
const std::string & BlockFilterTypeName(BlockFilterType filter_type)
Get the human-readable name for a filter type.