Bitcoin Core  0.20.99
P2P Digital Currency
bdb.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2020 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <wallet/bdb.h>
7 #include <wallet/db.h>
8 
9 #include <util/strencodings.h>
10 #include <util/translation.h>
11 
12 #include <stdint.h>
13 
14 #ifndef WIN32
15 #include <sys/stat.h>
16 #endif
17 
18 namespace {
19 
29 void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
30 {
31  if (env.IsMock()) return;
32 
33  int ret = db.get_mpf()->get_fileid(fileid.value);
34  if (ret != 0) {
35  throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (get_fileid failed with %d)", filename, ret));
36  }
37 
38  for (const auto& item : env.m_fileids) {
39  if (fileid == item.second && &fileid != &item.second) {
40  throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (duplicates fileid %s from %s)", filename,
41  HexStr(item.second.value), item.first));
42  }
43  }
44 }
45 
46 RecursiveMutex cs_db;
47 std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db);
48 } // namespace
49 
51 {
52  return memcmp(value, &rhs.value, sizeof(value)) == 0;
53 }
54 
62 std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename)
63 {
64  fs::path env_directory;
65  SplitWalletPath(wallet_path, env_directory, database_filename);
66  LOCK(cs_db);
67  auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>());
68  if (inserted.second) {
69  auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string());
70  inserted.first->second = env;
71  return env;
72  }
73  return inserted.first->second.lock();
74 }
75 
76 //
77 // BerkeleyBatch
78 //
79 
81 {
82  if (!fDbEnvInit)
83  return;
84 
85  fDbEnvInit = false;
86 
87  for (auto& db : m_databases) {
88  BerkeleyDatabase& database = db.second.get();
89  assert(database.m_refcount <= 0);
90  if (database.m_db) {
91  database.m_db->close(0);
92  database.m_db.reset();
93  }
94  }
95 
96  FILE* error_file = nullptr;
97  dbenv->get_errfile(&error_file);
98 
99  int ret = dbenv->close(0);
100  if (ret != 0)
101  LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret));
102  if (!fMockDb)
103  DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
104 
105  if (error_file) fclose(error_file);
106 
107  UnlockDirectory(strPath, ".walletlock");
108 }
109 
111 {
112  dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
113  fDbEnvInit = false;
114  fMockDb = false;
115 }
116 
117 BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string())
118 {
119  Reset();
120 }
121 
123 {
124  LOCK(cs_db);
125  g_dbenvs.erase(strPath);
126  Close();
127 }
128 
130 {
131  if (fDbEnvInit) {
132  return true;
133  }
134 
135  fs::path pathIn = strPath;
136  TryCreateDirectories(pathIn);
137  if (!LockDirectory(pathIn, ".walletlock")) {
138  LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath);
139  err = strprintf(_("Error initializing wallet database environment %s!"), Directory());
140  return false;
141  }
142 
143  fs::path pathLogDir = pathIn / "database";
144  TryCreateDirectories(pathLogDir);
145  fs::path pathErrorFile = pathIn / "db.log";
146  LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
147 
148  unsigned int nEnvFlags = 0;
149  if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
150  nEnvFlags |= DB_PRIVATE;
151 
152  dbenv->set_lg_dir(pathLogDir.string().c_str());
153  dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
154  dbenv->set_lg_bsize(0x10000);
155  dbenv->set_lg_max(1048576);
156  dbenv->set_lk_max_locks(40000);
157  dbenv->set_lk_max_objects(40000);
158  dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a"));
159  dbenv->set_flags(DB_AUTO_COMMIT, 1);
160  dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
161  dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
162  int ret = dbenv->open(strPath.c_str(),
163  DB_CREATE |
164  DB_INIT_LOCK |
165  DB_INIT_LOG |
166  DB_INIT_MPOOL |
167  DB_INIT_TXN |
168  DB_THREAD |
169  DB_RECOVER |
170  nEnvFlags,
171  S_IRUSR | S_IWUSR);
172  if (ret != 0) {
173  LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
174  int ret2 = dbenv->close(0);
175  if (ret2 != 0) {
176  LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2));
177  }
178  Reset();
179  err = strprintf(_("Error initializing wallet database environment %s!"), Directory());
180  if (ret == DB_RUNRECOVERY) {
181  err += Untranslated(" ") + _("This error could occur if this wallet was not shutdown cleanly and was last loaded using a build with a newer version of Berkeley DB. If so, please use the software that last loaded this wallet");
182  }
183  return false;
184  }
185 
186  fDbEnvInit = true;
187  fMockDb = false;
188  return true;
189 }
190 
193 {
194  Reset();
195 
196  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n");
197 
198  dbenv->set_cachesize(1, 0, 1);
199  dbenv->set_lg_bsize(10485760 * 4);
200  dbenv->set_lg_max(10485760);
201  dbenv->set_lk_max_locks(10000);
202  dbenv->set_lk_max_objects(10000);
203  dbenv->set_flags(DB_AUTO_COMMIT, 1);
204  dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
205  int ret = dbenv->open(nullptr,
206  DB_CREATE |
207  DB_INIT_LOCK |
208  DB_INIT_LOG |
209  DB_INIT_MPOOL |
210  DB_INIT_TXN |
211  DB_THREAD |
212  DB_PRIVATE,
213  S_IRUSR | S_IWUSR);
214  if (ret > 0) {
215  throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret));
216  }
217 
218  fDbEnvInit = true;
219  fMockDb = true;
220 }
221 
223 {
224  m_dbt.set_flags(DB_DBT_MALLOC);
225 }
226 
227 BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size)
228  : m_dbt(data, size)
229 {
230 }
231 
233 {
234  if (m_dbt.get_data() != nullptr) {
235  // Clear memory, e.g. in case it was a private key
236  memory_cleanse(m_dbt.get_data(), m_dbt.get_size());
237  // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be
238  // freed by the caller.
239  // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html
240  if (m_dbt.get_flags() & DB_DBT_MALLOC) {
241  free(m_dbt.get_data());
242  }
243  }
244 }
245 
247 {
248  return m_dbt.get_data();
249 }
250 
252 {
253  return m_dbt.get_size();
254 }
255 
256 BerkeleyBatch::SafeDbt::operator Dbt*()
257 {
258  return &m_dbt;
259 }
260 
262 {
263  fs::path walletDir = env->Directory();
264  fs::path file_path = walletDir / strFile;
265 
266  LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion());
267  LogPrintf("Using wallet %s\n", file_path.string());
268 
269  if (!env->Open(errorStr)) {
270  return false;
271  }
272 
273  if (fs::exists(file_path))
274  {
275  assert(m_refcount == 0);
276 
277  Db db(env->dbenv.get(), 0);
278  int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
279  if (result != 0) {
280  errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), file_path);
281  return false;
282  }
283  }
284  // also return true if files does not exists
285  return true;
286 }
287 
289 {
290  dbenv->txn_checkpoint(0, 0, 0);
291  if (fMockDb)
292  return;
293  dbenv->lsn_reset(strFile.c_str(), 0);
294 }
295 
297 {
298  if (env) {
299  LOCK(cs_db);
300  env->CloseDb(strFile);
301  assert(!m_db);
302  size_t erased = env->m_databases.erase(strFile);
303  assert(erased == 1);
304  env->m_fileids.erase(strFile);
305  }
306 }
307 
308 BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr), m_cursor(nullptr), m_database(database)
309 {
310  database.AddRef();
311  database.Open();
312  fReadOnly = read_only;
313  fFlushOnClose = fFlushOnCloseIn;
314  env = database.env.get();
315  pdb = database.m_db.get();
316  strFile = database.strFile;
317  if (!Exists(std::string("version"))) {
318  bool fTmp = fReadOnly;
319  fReadOnly = false;
320  Write(std::string("version"), CLIENT_VERSION);
321  fReadOnly = fTmp;
322  }
323 }
324 
326 {
327  unsigned int nFlags = DB_THREAD | DB_CREATE;
328 
329  {
330  LOCK(cs_db);
331  bilingual_str open_err;
332  if (!env->Open(open_err))
333  throw std::runtime_error("BerkeleyDatabase: Failed to open database environment.");
334 
335  if (m_db == nullptr) {
336  int ret;
337  std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0);
338 
339  bool fMockDb = env->IsMock();
340  if (fMockDb) {
341  DbMpoolFile* mpf = pdb_temp->get_mpf();
342  ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
343  if (ret != 0) {
344  throw std::runtime_error(strprintf("BerkeleyDatabase: Failed to configure for no temp file backing for database %s", strFile));
345  }
346  }
347 
348  ret = pdb_temp->open(nullptr, // Txn pointer
349  fMockDb ? nullptr : strFile.c_str(), // Filename
350  fMockDb ? strFile.c_str() : "main", // Logical db name
351  DB_BTREE, // Database type
352  nFlags, // Flags
353  0);
354 
355  if (ret != 0) {
356  throw std::runtime_error(strprintf("BerkeleyDatabase: Error %d, can't open database %s", ret, strFile));
357  }
358 
359  // Call CheckUniqueFileid on the containing BDB environment to
360  // avoid BDB data consistency bugs that happen when different data
361  // files in the same environment have the same fileid.
362  CheckUniqueFileid(*env, strFile, *pdb_temp, this->env->m_fileids[strFile]);
363 
364  m_db.reset(pdb_temp.release());
365 
366  }
367  }
368 }
369 
371 {
372  if (activeTxn)
373  return;
374 
375  // Flush database activity from memory pool to disk log
376  unsigned int nMinutes = 0;
377  if (fReadOnly)
378  nMinutes = 1;
379 
380  if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault
381  env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
382  }
383 }
384 
386 {
387  ++nUpdateCounter;
388 }
389 
391 {
392  Close();
394 }
395 
397 {
398  if (!pdb)
399  return;
400  if (activeTxn)
401  activeTxn->abort();
402  activeTxn = nullptr;
403  pdb = nullptr;
404  CloseCursor();
405 
406  if (fFlushOnClose)
407  Flush();
408 }
409 
410 void BerkeleyEnvironment::CloseDb(const std::string& strFile)
411 {
412  {
413  LOCK(cs_db);
414  auto it = m_databases.find(strFile);
415  assert(it != m_databases.end());
416  BerkeleyDatabase& database = it->second.get();
417  if (database.m_db) {
418  // Close the database handle
419  database.m_db->close(0);
420  database.m_db.reset();
421  }
422  }
423 }
424 
426 {
427  // Make sure that no Db's are in use
428  AssertLockNotHeld(cs_db);
429  std::unique_lock<RecursiveMutex> lock(cs_db);
430  m_db_in_use.wait(lock, [this](){
431  for (auto& db : m_databases) {
432  if (db.second.get().m_refcount > 0) return false;
433  }
434  return true;
435  });
436 
437  std::vector<std::string> filenames;
438  for (auto it : m_databases) {
439  filenames.push_back(it.first);
440  }
441  // Close the individual Db's
442  for (const std::string& filename : filenames) {
443  CloseDb(filename);
444  }
445  // Reset the environment
446  Flush(true); // This will flush and close the environment
447  Reset();
448  bilingual_str open_err;
449  Open(open_err);
450 }
451 
452 bool BerkeleyDatabase::Rewrite(const char* pszSkip)
453 {
454  while (true) {
455  {
456  LOCK(cs_db);
457  if (m_refcount <= 0) {
458  // Flush log data to the dat file
459  env->CloseDb(strFile);
460  env->CheckpointLSN(strFile);
461  m_refcount = -1;
462 
463  bool fSuccess = true;
464  LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
465  std::string strFileRes = strFile + ".rewrite";
466  { // surround usage of db with extra {}
467  BerkeleyBatch db(*this, true);
468  std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0);
469 
470  int ret = pdbCopy->open(nullptr, // Txn pointer
471  strFileRes.c_str(), // Filename
472  "main", // Logical db name
473  DB_BTREE, // Database type
474  DB_CREATE, // Flags
475  0);
476  if (ret > 0) {
477  LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes);
478  fSuccess = false;
479  }
480 
481  if (db.StartCursor()) {
482  while (fSuccess) {
485  bool complete;
486  bool ret1 = db.ReadAtCursor(ssKey, ssValue, complete);
487  if (complete) {
488  break;
489  } else if (!ret1) {
490  fSuccess = false;
491  break;
492  }
493  if (pszSkip &&
494  strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
495  continue;
496  if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
497  // Update version:
498  ssValue.clear();
499  ssValue << CLIENT_VERSION;
500  }
501  Dbt datKey(ssKey.data(), ssKey.size());
502  Dbt datValue(ssValue.data(), ssValue.size());
503  int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
504  if (ret2 > 0)
505  fSuccess = false;
506  }
507  db.CloseCursor();
508  }
509  if (fSuccess) {
510  db.Close();
511  env->CloseDb(strFile);
512  if (pdbCopy->close(0))
513  fSuccess = false;
514  } else {
515  pdbCopy->close(0);
516  }
517  }
518  if (fSuccess) {
519  Db dbA(env->dbenv.get(), 0);
520  if (dbA.remove(strFile.c_str(), nullptr, 0))
521  fSuccess = false;
522  Db dbB(env->dbenv.get(), 0);
523  if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
524  fSuccess = false;
525  }
526  if (!fSuccess)
527  LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes);
528  return fSuccess;
529  }
530  }
531  UninterruptibleSleep(std::chrono::milliseconds{100});
532  }
533 }
534 
535 
536 void BerkeleyEnvironment::Flush(bool fShutdown)
537 {
538  int64_t nStart = GetTimeMillis();
539  // Flush log data to the actual data file on all files that are not in use
540  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
541  if (!fDbEnvInit)
542  return;
543  {
544  LOCK(cs_db);
545  bool no_dbs_accessed = true;
546  for (auto& db_it : m_databases) {
547  std::string strFile = db_it.first;
548  int nRefCount = db_it.second.get().m_refcount;
549  if (nRefCount < 0) continue;
550  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
551  if (nRefCount == 0) {
552  // Move log data to the dat file
553  CloseDb(strFile);
554  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile);
555  dbenv->txn_checkpoint(0, 0, 0);
556  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile);
557  if (!fMockDb)
558  dbenv->lsn_reset(strFile.c_str(), 0);
559  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile);
560  nRefCount = -1;
561  } else {
562  no_dbs_accessed = false;
563  }
564  }
565  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
566  if (fShutdown) {
567  char** listp;
568  if (no_dbs_accessed) {
569  dbenv->log_archive(&listp, DB_ARCH_REMOVE);
570  Close();
571  if (!fMockDb) {
572  fs::remove_all(fs::path(strPath) / "database");
573  }
574  }
575  }
576  }
577 }
578 
580 {
581  // Don't flush if we can't acquire the lock.
582  TRY_LOCK(cs_db, lockDb);
583  if (!lockDb) return false;
584 
585  // Don't flush if any databases are in use
586  for (auto& it : env->m_databases) {
587  if (it.second.get().m_refcount > 0) return false;
588  }
589 
590  // Don't flush if there haven't been any batch writes for this database.
591  if (m_refcount < 0) return false;
592 
593  LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile);
594  int64_t nStart = GetTimeMillis();
595 
596  // Flush wallet file so it's self contained
597  env->CloseDb(strFile);
598  env->CheckpointLSN(strFile);
599  m_refcount = -1;
600 
601  LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
602 
603  return true;
604 }
605 
606 bool BerkeleyDatabase::Backup(const std::string& strDest) const
607 {
608  while (true)
609  {
610  {
611  LOCK(cs_db);
612  if (m_refcount <= 0)
613  {
614  // Flush log data to the dat file
615  env->CloseDb(strFile);
616  env->CheckpointLSN(strFile);
617 
618  // Copy wallet file
619  fs::path pathSrc = env->Directory() / strFile;
620  fs::path pathDest(strDest);
621  if (fs::is_directory(pathDest))
622  pathDest /= strFile;
623 
624  try {
625  if (fs::equivalent(pathSrc, pathDest)) {
626  LogPrintf("cannot backup to wallet source file %s\n", pathDest.string());
627  return false;
628  }
629 
630  fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
631  LogPrintf("copied %s to %s\n", strFile, pathDest.string());
632  return true;
633  } catch (const fs::filesystem_error& e) {
634  LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e));
635  return false;
636  }
637  }
638  }
639  UninterruptibleSleep(std::chrono::milliseconds{100});
640  }
641 }
642 
644 {
645  env->Flush(false);
646 }
647 
649 {
650  env->Flush(true);
651 }
652 
654 {
655  env->ReloadDbEnv();
656 }
657 
659 {
660  assert(!m_cursor);
661  if (!pdb)
662  return false;
663  int ret = pdb->cursor(nullptr, &m_cursor, 0);
664  return ret == 0;
665 }
666 
667 bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete)
668 {
669  complete = false;
670  if (m_cursor == nullptr) return false;
671  // Read at cursor
672  SafeDbt datKey;
673  SafeDbt datValue;
674  int ret = m_cursor->get(datKey, datValue, DB_NEXT);
675  if (ret == DB_NOTFOUND) {
676  complete = true;
677  }
678  if (ret != 0)
679  return false;
680  else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr)
681  return false;
682 
683  // Convert to streams
684  ssKey.SetType(SER_DISK);
685  ssKey.clear();
686  ssKey.write((char*)datKey.get_data(), datKey.get_size());
687  ssValue.SetType(SER_DISK);
688  ssValue.clear();
689  ssValue.write((char*)datValue.get_data(), datValue.get_size());
690  return true;
691 }
692 
694 {
695  if (!m_cursor) return;
696  m_cursor->close();
697  m_cursor = nullptr;
698 }
699 
701 {
702  if (!pdb || activeTxn)
703  return false;
704  DbTxn* ptxn = env->TxnBegin();
705  if (!ptxn)
706  return false;
707  activeTxn = ptxn;
708  return true;
709 }
710 
712 {
713  if (!pdb || !activeTxn)
714  return false;
715  int ret = activeTxn->commit(0);
716  activeTxn = nullptr;
717  return (ret == 0);
718 }
719 
721 {
722  if (!pdb || !activeTxn)
723  return false;
724  int ret = activeTxn->abort();
725  activeTxn = nullptr;
726  return (ret == 0);
727 }
728 
730 {
731  return DbEnv::version(nullptr, nullptr, nullptr);
732 }
733 
735 {
736  if (!pdb)
737  return false;
738 
739  SafeDbt datKey(key.data(), key.size());
740 
741  SafeDbt datValue;
742  int ret = pdb->get(activeTxn, datKey, datValue, 0);
743  if (ret == 0 && datValue.get_data() != nullptr) {
744  value.write((char*)datValue.get_data(), datValue.get_size());
745  return true;
746  }
747  return false;
748 }
749 
750 bool BerkeleyBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite)
751 {
752  if (!pdb)
753  return false;
754  if (fReadOnly)
755  assert(!"Write called on database in read-only mode");
756 
757  SafeDbt datKey(key.data(), key.size());
758 
759  SafeDbt datValue(value.data(), value.size());
760 
761  int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE));
762  return (ret == 0);
763 }
764 
766 {
767  if (!pdb)
768  return false;
769  if (fReadOnly)
770  assert(!"Erase called on database in read-only mode");
771 
772  SafeDbt datKey(key.data(), key.size());
773 
774  int ret = pdb->del(activeTxn, datKey, 0);
775  return (ret == 0 || ret == DB_NOTFOUND);
776 }
777 
779 {
780  if (!pdb)
781  return false;
782 
783  SafeDbt datKey(key.data(), key.size());
784 
785  int ret = pdb->exists(activeTxn, datKey, 0);
786  return ret == 0;
787 }
788 
790 {
791  LOCK(cs_db);
792  if (m_refcount < 0) {
793  m_refcount = 1;
794  } else {
795  m_refcount++;
796  }
797 }
798 
800 {
801  LOCK(cs_db);
802  m_refcount--;
803  if (env) env->m_db_in_use.notify_all();
804 }
805 
806 std::unique_ptr<DatabaseBatch> BerkeleyDatabase::MakeBatch(bool flush_on_close)
807 {
808  return MakeUnique<BerkeleyBatch>(*this, false, flush_on_close);
809 }
810 
811 bool ExistsBerkeleyDatabase(const fs::path& path)
812 {
813  fs::path env_directory;
814  std::string data_filename;
815  SplitWalletPath(path, env_directory, data_filename);
816  return IsBDBFile(env_directory / data_filename);
817 }
818 
819 std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error)
820 {
821  std::unique_ptr<BerkeleyDatabase> db;
822  {
823  LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor
824  std::string data_filename;
825  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(path, data_filename);
826  if (env->m_databases.count(data_filename)) {
827  error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", (env->Directory() / data_filename).string()));
829  return nullptr;
830  }
831  db = MakeUnique<BerkeleyDatabase>(std::move(env), std::move(data_filename));
832  }
833 
834  if (options.verify && !db->Verify(error)) {
836  return nullptr;
837  }
838 
839  status = DatabaseStatus::SUCCESS;
840  return db;
841 }
842 
843 bool IsBDBFile(const fs::path& path)
844 {
845  if (!fs::exists(path)) return false;
846 
847  // A Berkeley DB Btree file has at least 4K.
848  // This check also prevents opening lock files.
849  boost::system::error_code ec;
850  auto size = fs::file_size(path, ec);
851  if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string());
852  if (size < 4096) return false;
853 
854  fsbridge::ifstream file(path, std::ios::binary);
855  if (!file.is_open()) return false;
856 
857  file.seekg(12, std::ios::beg); // Magic bytes start at offset 12
858  uint32_t data = 0;
859  file.read((char*) &data, sizeof(data)); // Read 4 bytes of file to compare against magic
860 
861  // Berkeley DB Btree magic bytes, from:
862  // https://github.com/file/file/blob/5824af38469ec1ca9ac3ffd251e7afe9dc11e227/magic/Magdir/database#L74-L75
863  // - big endian systems - 00 05 31 62
864  // - little endian systems - 62 31 05 00
865  return data == 0x00053162 || data == 0x62310500;
866 }
bool fReadOnly
Definition: bdb.h:204
std::unique_ptr< DatabaseBatch > MakeBatch(bool flush_on_close=true) override
Make a BerkeleyBatch connected to this database.
Definition: bdb.cpp:806
void ReloadDbEnv() override
Definition: bdb.cpp:653
std::deque< CInv >::iterator it
void Open() override
Open the database if it is not already opened.
Definition: bdb.cpp:325
BerkeleyEnvironment()
Construct an in-memory mock Berkeley environment for testing.
Definition: bdb.cpp:192
bool StartCursor() override
Definition: bdb.cpp:658
bool fDbEnvInit
Definition: bdb.h:47
#define LogPrint(category,...)
Definition: logging.h:182
FILE * fopen(const fs::path &p, const char *mode)
Definition: fs.cpp:24
std::shared_ptr< BerkeleyEnvironment > GetWalletEnv(const fs::path &wallet_path, std::string &database_filename)
Get BerkeleyEnvironment and database filename given a wallet path.
Definition: bdb.cpp:62
void Close()
Definition: bdb.cpp:80
#define TRY_LOCK(cs, name)
Definition: sync.h:234
std::unordered_map< std::string, WalletDatabaseFileId > m_fileids
Definition: bdb.h:56
Bilingual messages:
Definition: translation.h:16
int64_t GetTimeMillis()
Returns the system time (not mockable)
Definition: time.cpp:57
#define strprintf
Format arguments and return the string or write to given std::ostream (see tinyformat::format doc for...
Definition: tinyformat.h:1164
bool fFlushOnClose
Definition: bdb.h:205
bilingual_str Untranslated(std::string original)
Mark a bilingual_str as untranslated.
Definition: translation.h:40
BerkeleyDatabase & m_database
Definition: bdb.h:207
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by Boost&#39;s create_directories if the requested directory exists...
Definition: system.cpp:1009
BerkeleyBatch(BerkeleyDatabase &database, const bool fReadOnly, bool fFlushOnCloseIn=true)
Definition: bdb.cpp:308
~BerkeleyBatch() override
Definition: bdb.cpp:390
fs::ifstream ifstream
Definition: fs.h:90
value_type * data()
Definition: streams.h:302
An instance of this class represents one database.
Definition: bdb.h:97
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:166
void Flush() override
Make sure all changes are flushed to database file.
Definition: bdb.cpp:643
static const unsigned int DEFAULT_WALLET_DBLOGSIZE
Definition: bdb.h:34
Double ended buffer combining vector and stream-like interfaces.
Definition: streams.h:202
void write(const char *pch, size_t nSize)
Definition: streams.h:435
bool GetBoolArg(const std::string &strArg, bool fDefault) const
Return boolean argument or default value.
Definition: system.cpp:478
static const bool DEFAULT_WALLET_PRIVDB
Definition: bdb.h:35
static const char * filenames[]
Definition: unitester.cpp:80
fs::path Directory() const
Definition: bdb.h:66
u_int32_t get_size() const
Definition: bdb.cpp:251
DbTxn * TxnBegin(int flags=DB_TXN_WRITE_NOSYNC)
Definition: bdb.h:76
std::shared_ptr< BerkeleyEnvironment > env
Pointer to shared database environment.
Definition: bdb.h:159
bool Rewrite(const char *pszSkip=nullptr) override
Rewrite the entire database on disk, with the exception of key pszSkip if non-zero.
Definition: bdb.cpp:452
bool Write(const K &key, const T &value, bool fOverwrite=true)
Definition: db.h:61
std::map< std::string, std::reference_wrapper< BerkeleyDatabase > > m_databases
Definition: bdb.h:55
std::unique_ptr< DbEnv > dbenv
Definition: bdb.h:54
bool Backup(const std::string &strDest) const override
Back up the entire database to a file.
Definition: bdb.cpp:606
void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename)
Definition: db.cpp:11
void CheckpointLSN(const std::string &strFile)
Definition: bdb.cpp:288
bool LockDirectory(const fs::path &directory, const std::string lockfile_name, bool probe_only)
Definition: system.cpp:87
bool verify
Definition: db.h:210
bool TxnAbort() override
Definition: bdb.cpp:720
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
Definition: cleanse.cpp:14
bool IsBDBFile(const fs::path &path)
Check format of database file.
Definition: bdb.cpp:843
bool WriteKey(CDataStream &&key, CDataStream &&value, bool overwrite=true) override
Definition: bdb.cpp:750
size_type size() const
Definition: streams.h:293
void Flush() override
Definition: bdb.cpp:370
void Close() override
Definition: bdb.cpp:396
std::string strFile
Definition: bdb.h:201
void RemoveRef() override
Indicate that database user has stopped using the database and that it could be flushed or closed...
Definition: bdb.cpp:799
#define LOCK(cs)
Definition: sync.h:230
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:57
bool ExistsBerkeleyDatabase(const fs::path &path)
Check if Berkeley database exists at specified path.
Definition: bdb.cpp:811
bool HasKey(CDataStream &&key) override
Definition: bdb.cpp:778
std::string BerkeleyDatabaseVersion()
Definition: bdb.cpp:729
DbTxn * activeTxn
Definition: bdb.h:202
RAII class that provides access to a Berkeley database.
Definition: bdb.h:171
std::string strFile
Definition: bdb.h:164
bool PeriodicFlush() override
Definition: bdb.cpp:579
bool operator==(const WalletDatabaseFileId &rhs) const
Definition: bdb.cpp:50
std::string HexStr(const Span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.
~BerkeleyDatabase() override
Definition: bdb.cpp:296
bool EraseKey(CDataStream &&key) override
Definition: bdb.cpp:765
bool ReadAtCursor(CDataStream &ssKey, CDataStream &ssValue, bool &complete) override
Definition: bdb.cpp:667
std::unique_ptr< BerkeleyDatabase > MakeBerkeleyDatabase(const fs::path &path, const DatabaseOptions &options, DatabaseStatus &status, bilingual_str &error)
Return object giving access to Berkeley database at specified path.
Definition: bdb.cpp:819
void ReloadDbEnv()
Definition: bdb.cpp:425
std::string get_filesystem_error_message(const fs::filesystem_error &e)
Definition: fs.cpp:131
void AddRef() override
Indicate the a new database user has began using the database.
Definition: bdb.cpp:789
Dbc * m_cursor
Definition: bdb.h:203
void UninterruptibleSleep(const std::chrono::microseconds &n)
Definition: time.cpp:19
std::string strPath
Definition: bdb.h:51
void IncrementUpdateCounter() override
Definition: bdb.cpp:385
void UnlockDirectory(const fs::path &directory, const std::string &lockfile_name)
Definition: system.cpp:111
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:466
void Close() override
Flush to the database file and close the database.
Definition: bdb.cpp:648
u_int8_t value[DB_FILE_ID_LEN]
Definition: bdb.h:38
ArgsManager gArgs
Definition: system.cpp:76
void Flush(bool fShutdown)
Definition: bdb.cpp:536
Db * pdb
Definition: bdb.h:200
#define AssertLockNotHeld(cs)
Definition: sync.h:80
bool Open(bilingual_str &error)
Definition: bdb.cpp:129
void CloseCursor() override
Definition: bdb.cpp:693
#define GUARDED_BY(x)
Definition: threadsafety.h:38
void clear()
Definition: streams.h:299
void CloseDb(const std::string &strFile)
Definition: bdb.cpp:410
bool Exists(const K &key)
Definition: db.h:85
bool IsMock() const
Definition: bdb.h:64
DatabaseStatus
Definition: db.h:213
bool fMockDb
Definition: bdb.h:48
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
std::atomic< int > m_refcount
Counts the number of active database users to be sure that the database is not closed while someone i...
Definition: db.h:115
std::unique_ptr< Db > m_db
Database pointer.
Definition: bdb.h:162
bool error(const char *fmt, const Args &... args)
Definition: system.h:52
bool Verify(bilingual_str &error)
Verifies the environment and database file.
Definition: bdb.cpp:261
bool ReadKey(CDataStream &&key, CDataStream &value) override
Definition: bdb.cpp:734
const void * get_data() const
Definition: bdb.cpp:246
bool TxnBegin() override
Definition: bdb.cpp:700
bool TxnCommit() override
Definition: bdb.cpp:711
RAII class that automatically cleanses its data on destruction.
Definition: bdb.h:174
void SetType(int n)
Definition: streams.h:393
std::condition_variable_any m_db_in_use
Definition: bdb.h:57