Bitcoin Core  0.19.99
P2P Digital Currency
db.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2020 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <wallet/db.h>
7 
8 #include <util/strencodings.h>
9 #include <util/translation.h>
10 
11 #include <stdint.h>
12 
13 #ifndef WIN32
14 #include <sys/stat.h>
15 #endif
16 
17 #include <boost/thread.hpp>
18 
19 namespace {
20 
30 void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
31 {
32  if (env.IsMock()) return;
33 
34  int ret = db.get_mpf()->get_fileid(fileid.value);
35  if (ret != 0) {
36  throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret));
37  }
38 
39  for (const auto& item : env.m_fileids) {
40  if (fileid == item.second && &fileid != &item.second) {
41  throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (duplicates fileid %s from %s)", filename,
42  HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first));
43  }
44  }
45 }
46 
47 RecursiveMutex cs_db;
48 std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db);
49 } // namespace
50 
52 {
53  return memcmp(value, &rhs.value, sizeof(value)) == 0;
54 }
55 
56 static void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename)
57 {
58  if (fs::is_regular_file(wallet_path)) {
59  // Special case for backwards compatibility: if wallet path points to an
60  // existing file, treat it as the path to a BDB data file in a parent
61  // directory that also contains BDB log files.
62  env_directory = wallet_path.parent_path();
63  database_filename = wallet_path.filename().string();
64  } else {
65  // Normal case: Interpret wallet path as a directory path containing
66  // data and log files.
67  env_directory = wallet_path;
68  database_filename = "wallet.dat";
69  }
70 }
71 
72 bool IsWalletLoaded(const fs::path& wallet_path)
73 {
74  fs::path env_directory;
75  std::string database_filename;
76  SplitWalletPath(wallet_path, env_directory, database_filename);
77  LOCK(cs_db);
78  auto env = g_dbenvs.find(env_directory.string());
79  if (env == g_dbenvs.end()) return false;
80  auto database = env->second.lock();
81  return database && database->IsDatabaseLoaded(database_filename);
82 }
83 
84 fs::path WalletDataFilePath(const fs::path& wallet_path)
85 {
86  fs::path env_directory;
87  std::string database_filename;
88  SplitWalletPath(wallet_path, env_directory, database_filename);
89  return env_directory / database_filename;
90 }
91 
99 std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename)
100 {
101  fs::path env_directory;
102  SplitWalletPath(wallet_path, env_directory, database_filename);
103  LOCK(cs_db);
104  auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>());
105  if (inserted.second) {
106  auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string());
107  inserted.first->second = env;
108  return env;
109  }
110  return inserted.first->second.lock();
111 }
112 
113 //
114 // BerkeleyBatch
115 //
116 
118 {
119  if (!fDbEnvInit)
120  return;
121 
122  fDbEnvInit = false;
123 
124  for (auto& db : m_databases) {
125  auto count = mapFileUseCount.find(db.first);
126  assert(count == mapFileUseCount.end() || count->second == 0);
127  BerkeleyDatabase& database = db.second.get();
128  if (database.m_db) {
129  database.m_db->close(0);
130  database.m_db.reset();
131  }
132  }
133 
134  FILE* error_file = nullptr;
135  dbenv->get_errfile(&error_file);
136 
137  int ret = dbenv->close(0);
138  if (ret != 0)
139  LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret));
140  if (!fMockDb)
141  DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
142 
143  if (error_file) fclose(error_file);
144 
145  UnlockDirectory(strPath, ".walletlock");
146 }
147 
149 {
150  dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
151  fDbEnvInit = false;
152  fMockDb = false;
153 }
154 
155 BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string())
156 {
157  Reset();
158 }
159 
161 {
162  LOCK(cs_db);
163  g_dbenvs.erase(strPath);
164  Close();
165 }
166 
168 {
169  if (fDbEnvInit) {
170  return true;
171  }
172 
173  fs::path pathIn = strPath;
174  TryCreateDirectories(pathIn);
175  if (!LockDirectory(pathIn, ".walletlock")) {
176  LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath);
177  return false;
178  }
179 
180  fs::path pathLogDir = pathIn / "database";
181  TryCreateDirectories(pathLogDir);
182  fs::path pathErrorFile = pathIn / "db.log";
183  LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
184 
185  unsigned int nEnvFlags = 0;
186  if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
187  nEnvFlags |= DB_PRIVATE;
188 
189  dbenv->set_lg_dir(pathLogDir.string().c_str());
190  dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
191  dbenv->set_lg_bsize(0x10000);
192  dbenv->set_lg_max(1048576);
193  dbenv->set_lk_max_locks(40000);
194  dbenv->set_lk_max_objects(40000);
195  dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a"));
196  dbenv->set_flags(DB_AUTO_COMMIT, 1);
197  dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
198  dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
199  int ret = dbenv->open(strPath.c_str(),
200  DB_CREATE |
201  DB_INIT_LOCK |
202  DB_INIT_LOG |
203  DB_INIT_MPOOL |
204  DB_INIT_TXN |
205  DB_THREAD |
206  DB_RECOVER |
207  nEnvFlags,
208  S_IRUSR | S_IWUSR);
209  if (ret != 0) {
210  LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
211  int ret2 = dbenv->close(0);
212  if (ret2 != 0) {
213  LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2));
214  }
215  Reset();
216  if (retry) {
217  // try moving the database env out of the way
218  fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime());
219  try {
220  fs::rename(pathLogDir, pathDatabaseBak);
221  LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string());
222  } catch (const fs::filesystem_error&) {
223  // failure is ok (well, not really, but it's not worse than what we started with)
224  }
225  // try opening it again one more time
226  if (!Open(false /* retry */)) {
227  // if it still fails, it probably means we can't even create the database env
228  return false;
229  }
230  } else {
231  return false;
232  }
233  }
234 
235  fDbEnvInit = true;
236  fMockDb = false;
237  return true;
238 }
239 
242 {
243  Reset();
244 
245  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n");
246 
247  dbenv->set_cachesize(1, 0, 1);
248  dbenv->set_lg_bsize(10485760 * 4);
249  dbenv->set_lg_max(10485760);
250  dbenv->set_lk_max_locks(10000);
251  dbenv->set_lk_max_objects(10000);
252  dbenv->set_flags(DB_AUTO_COMMIT, 1);
253  dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
254  int ret = dbenv->open(nullptr,
255  DB_CREATE |
256  DB_INIT_LOCK |
257  DB_INIT_LOG |
258  DB_INIT_MPOOL |
259  DB_INIT_TXN |
260  DB_THREAD |
261  DB_PRIVATE,
262  S_IRUSR | S_IWUSR);
263  if (ret > 0) {
264  throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret));
265  }
266 
267  fDbEnvInit = true;
268  fMockDb = true;
269 }
270 
271 BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename)
272 {
273  LOCK(cs_db);
274  assert(mapFileUseCount.count(strFile) == 0);
275 
276  Db db(dbenv.get(), 0);
277  int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
278  if (result == 0)
280  else if (recoverFunc == nullptr)
282 
283  // Try to recover:
284  bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename);
286 }
287 
289 {
290  m_dbt.set_flags(DB_DBT_MALLOC);
291 }
292 
293 BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size)
294  : m_dbt(data, size)
295 {
296 }
297 
299 {
300  if (m_dbt.get_data() != nullptr) {
301  // Clear memory, e.g. in case it was a private key
302  memory_cleanse(m_dbt.get_data(), m_dbt.get_size());
303  // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be
304  // freed by the caller.
305  // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html
306  if (m_dbt.get_flags() & DB_DBT_MALLOC) {
307  free(m_dbt.get_data());
308  }
309  }
310 }
311 
313 {
314  return m_dbt.get_data();
315 }
316 
318 {
319  return m_dbt.get_size();
320 }
321 
322 BerkeleyBatch::SafeDbt::operator Dbt*()
323 {
324  return &m_dbt;
325 }
326 
327 bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename)
328 {
329  std::string filename;
330  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename);
331 
332  // Recovery procedure:
333  // move wallet file to walletfilename.timestamp.bak
334  // Call Salvage with fAggressive=true to
335  // get as much data as possible.
336  // Rewrite salvaged data to fresh wallet file
337  // Set -rescan so any missing transactions will be
338  // found.
339  int64_t now = GetTime();
340  newFilename = strprintf("%s.%d.bak", filename, now);
341 
342  int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr,
343  newFilename.c_str(), DB_AUTO_COMMIT);
344  if (result == 0)
345  LogPrintf("Renamed %s to %s\n", filename, newFilename);
346  else
347  {
348  LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
349  return false;
350  }
351 
352  std::vector<BerkeleyEnvironment::KeyValPair> salvagedData;
353  bool fSuccess = env->Salvage(newFilename, true, salvagedData);
354  if (salvagedData.empty())
355  {
356  LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
357  return false;
358  }
359  LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
360 
361  std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0);
362  int ret = pdbCopy->open(nullptr, // Txn pointer
363  filename.c_str(), // Filename
364  "main", // Logical db name
365  DB_BTREE, // Database type
366  DB_CREATE, // Flags
367  0);
368  if (ret > 0) {
369  LogPrintf("Cannot create database file %s\n", filename);
370  pdbCopy->close(0);
371  return false;
372  }
373 
374  DbTxn* ptxn = env->TxnBegin();
375  for (BerkeleyEnvironment::KeyValPair& row : salvagedData)
376  {
377  if (recoverKVcallback)
378  {
379  CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
380  CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
381  if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
382  continue;
383  }
384  Dbt datKey(&row.first[0], row.first.size());
385  Dbt datValue(&row.second[0], row.second.size());
386  int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
387  if (ret2 > 0)
388  fSuccess = false;
389  }
390  ptxn->commit(0);
391  pdbCopy->close(0);
392 
393  return fSuccess;
394 }
395 
396 bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& errorStr)
397 {
398  std::string walletFile;
399  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile);
400  fs::path walletDir = env->Directory();
401 
402  LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr));
403  LogPrintf("Using wallet %s\n", file_path.string());
404 
405  if (!env->Open(true /* retry */)) {
406  errorStr = strprintf(_("Error initializing wallet database environment %s!").translated, walletDir);
407  return false;
408  }
409 
410  return true;
411 }
412 
413 bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<std::string>& warnings, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc)
414 {
415  std::string walletFile;
416  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile);
417  fs::path walletDir = env->Directory();
418 
419  if (fs::exists(walletDir / walletFile))
420  {
421  std::string backup_filename;
422  BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename);
424  {
425  warnings.push_back(strprintf(_("Warning: Wallet file corrupt, data salvaged!"
426  " Original %s saved as %s in %s; if"
427  " your balance or transactions are incorrect you should"
428  " restore from a backup.").translated,
429  walletFile, backup_filename, walletDir));
430  }
432  {
433  errorStr = strprintf(_("%s corrupt, salvage failed").translated, walletFile);
434  return false;
435  }
436  }
437  // also return true if files does not exists
438  return true;
439 }
440 
441 /* End of headers, beginning of key/value data */
442 static const char *HEADER_END = "HEADER=END";
443 /* End of key/value data */
444 static const char *DATA_END = "DATA=END";
445 
446 bool BerkeleyEnvironment::Salvage(const std::string& strFile, bool fAggressive, std::vector<BerkeleyEnvironment::KeyValPair>& vResult)
447 {
448  LOCK(cs_db);
449  assert(mapFileUseCount.count(strFile) == 0);
450 
451  u_int32_t flags = DB_SALVAGE;
452  if (fAggressive)
453  flags |= DB_AGGRESSIVE;
454 
455  std::stringstream strDump;
456 
457  Db db(dbenv.get(), 0);
458  int result = db.verify(strFile.c_str(), nullptr, &strDump, flags);
459  if (result == DB_VERIFY_BAD) {
460  LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found errors, all data may not be recoverable.\n");
461  if (!fAggressive) {
462  LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
463  return false;
464  }
465  }
466  if (result != 0 && result != DB_VERIFY_BAD) {
467  LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with result %d.\n", result);
468  return false;
469  }
470 
471  // Format of bdb dump is ascii lines:
472  // header lines...
473  // HEADER=END
474  // hexadecimal key
475  // hexadecimal value
476  // ... repeated
477  // DATA=END
478 
479  std::string strLine;
480  while (!strDump.eof() && strLine != HEADER_END)
481  getline(strDump, strLine); // Skip past header
482 
483  std::string keyHex, valueHex;
484  while (!strDump.eof() && keyHex != DATA_END) {
485  getline(strDump, keyHex);
486  if (keyHex != DATA_END) {
487  if (strDump.eof())
488  break;
489  getline(strDump, valueHex);
490  if (valueHex == DATA_END) {
491  LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of keys in data does not match number of values.\n");
492  break;
493  }
494  vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
495  }
496  }
497 
498  if (keyHex != DATA_END) {
499  LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
500  return false;
501  }
502 
503  return (result == 0);
504 }
505 
506 
508 {
509  dbenv->txn_checkpoint(0, 0, 0);
510  if (fMockDb)
511  return;
512  dbenv->lsn_reset(strFile.c_str(), 0);
513 }
514 
515 
516 BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr)
517 {
518  fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
519  fFlushOnClose = fFlushOnCloseIn;
520  env = database.env.get();
521  if (database.IsDummy()) {
522  return;
523  }
524  const std::string &strFilename = database.strFile;
525 
526  bool fCreate = strchr(pszMode, 'c') != nullptr;
527  unsigned int nFlags = DB_THREAD;
528  if (fCreate)
529  nFlags |= DB_CREATE;
530 
531  {
532  LOCK(cs_db);
533  if (!env->Open(false /* retry */))
534  throw std::runtime_error("BerkeleyBatch: Failed to open database environment.");
535 
536  pdb = database.m_db.get();
537  if (pdb == nullptr) {
538  int ret;
539  std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0);
540 
541  bool fMockDb = env->IsMock();
542  if (fMockDb) {
543  DbMpoolFile* mpf = pdb_temp->get_mpf();
544  ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
545  if (ret != 0) {
546  throw std::runtime_error(strprintf("BerkeleyBatch: Failed to configure for no temp file backing for database %s", strFilename));
547  }
548  }
549 
550  ret = pdb_temp->open(nullptr, // Txn pointer
551  fMockDb ? nullptr : strFilename.c_str(), // Filename
552  fMockDb ? strFilename.c_str() : "main", // Logical db name
553  DB_BTREE, // Database type
554  nFlags, // Flags
555  0);
556 
557  if (ret != 0) {
558  throw std::runtime_error(strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename));
559  }
560 
561  // Call CheckUniqueFileid on the containing BDB environment to
562  // avoid BDB data consistency bugs that happen when different data
563  // files in the same environment have the same fileid.
564  //
565  // Also call CheckUniqueFileid on all the other g_dbenvs to prevent
566  // bitcoin from opening the same data file through another
567  // environment when the file is referenced through equivalent but
568  // not obviously identical symlinked or hard linked or bind mounted
569  // paths. In the future a more relaxed check for equal inode and
570  // device ids could be done instead, which would allow opening
571  // different backup copies of a wallet at the same time. Maybe even
572  // more ideally, an exclusive lock for accessing the database could
573  // be implemented, so no equality checks are needed at all. (Newer
574  // versions of BDB have an set_lk_exclusive method for this
575  // purpose, but the older version we use does not.)
576  for (const auto& env : g_dbenvs) {
577  CheckUniqueFileid(*env.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]);
578  }
579 
580  pdb = pdb_temp.release();
581  database.m_db.reset(pdb);
582 
583  if (fCreate && !Exists(std::string("version"))) {
584  bool fTmp = fReadOnly;
585  fReadOnly = false;
586  Write(std::string("version"), CLIENT_VERSION);
587  fReadOnly = fTmp;
588  }
589  }
590  ++env->mapFileUseCount[strFilename];
591  strFile = strFilename;
592  }
593 }
594 
596 {
597  if (activeTxn)
598  return;
599 
600  // Flush database activity from memory pool to disk log
601  unsigned int nMinutes = 0;
602  if (fReadOnly)
603  nMinutes = 1;
604 
605  if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault
606  env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
607  }
608 }
609 
611 {
612  ++nUpdateCounter;
613 }
614 
616 {
617  if (!pdb)
618  return;
619  if (activeTxn)
620  activeTxn->abort();
621  activeTxn = nullptr;
622  pdb = nullptr;
623 
624  if (fFlushOnClose)
625  Flush();
626 
627  {
628  LOCK(cs_db);
629  --env->mapFileUseCount[strFile];
630  }
631  env->m_db_in_use.notify_all();
632 }
633 
634 void BerkeleyEnvironment::CloseDb(const std::string& strFile)
635 {
636  {
637  LOCK(cs_db);
638  auto it = m_databases.find(strFile);
639  assert(it != m_databases.end());
640  BerkeleyDatabase& database = it->second.get();
641  if (database.m_db) {
642  // Close the database handle
643  database.m_db->close(0);
644  database.m_db.reset();
645  }
646  }
647 }
648 
650 {
651  // Make sure that no Db's are in use
652  AssertLockNotHeld(cs_db);
653  std::unique_lock<RecursiveMutex> lock(cs_db);
654  m_db_in_use.wait(lock, [this](){
655  for (auto& count : mapFileUseCount) {
656  if (count.second > 0) return false;
657  }
658  return true;
659  });
660 
661  std::vector<std::string> filenames;
662  for (auto it : m_databases) {
663  filenames.push_back(it.first);
664  }
665  // Close the individual Db's
666  for (const std::string& filename : filenames) {
667  CloseDb(filename);
668  }
669  // Reset the environment
670  Flush(true); // This will flush and close the environment
671  Reset();
672  Open(true);
673 }
674 
675 bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip)
676 {
677  if (database.IsDummy()) {
678  return true;
679  }
680  BerkeleyEnvironment *env = database.env.get();
681  const std::string& strFile = database.strFile;
682  while (true) {
683  {
684  LOCK(cs_db);
685  if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
686  // Flush log data to the dat file
687  env->CloseDb(strFile);
688  env->CheckpointLSN(strFile);
689  env->mapFileUseCount.erase(strFile);
690 
691  bool fSuccess = true;
692  LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
693  std::string strFileRes = strFile + ".rewrite";
694  { // surround usage of db with extra {}
695  BerkeleyBatch db(database, "r");
696  std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0);
697 
698  int ret = pdbCopy->open(nullptr, // Txn pointer
699  strFileRes.c_str(), // Filename
700  "main", // Logical db name
701  DB_BTREE, // Database type
702  DB_CREATE, // Flags
703  0);
704  if (ret > 0) {
705  LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes);
706  fSuccess = false;
707  }
708 
709  Dbc* pcursor = db.GetCursor();
710  if (pcursor)
711  while (fSuccess) {
714  int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
715  if (ret1 == DB_NOTFOUND) {
716  pcursor->close();
717  break;
718  } else if (ret1 != 0) {
719  pcursor->close();
720  fSuccess = false;
721  break;
722  }
723  if (pszSkip &&
724  strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
725  continue;
726  if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
727  // Update version:
728  ssValue.clear();
729  ssValue << CLIENT_VERSION;
730  }
731  Dbt datKey(ssKey.data(), ssKey.size());
732  Dbt datValue(ssValue.data(), ssValue.size());
733  int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
734  if (ret2 > 0)
735  fSuccess = false;
736  }
737  if (fSuccess) {
738  db.Close();
739  env->CloseDb(strFile);
740  if (pdbCopy->close(0))
741  fSuccess = false;
742  } else {
743  pdbCopy->close(0);
744  }
745  }
746  if (fSuccess) {
747  Db dbA(env->dbenv.get(), 0);
748  if (dbA.remove(strFile.c_str(), nullptr, 0))
749  fSuccess = false;
750  Db dbB(env->dbenv.get(), 0);
751  if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
752  fSuccess = false;
753  }
754  if (!fSuccess)
755  LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes);
756  return fSuccess;
757  }
758  }
759  MilliSleep(100);
760  }
761 }
762 
763 
764 void BerkeleyEnvironment::Flush(bool fShutdown)
765 {
766  int64_t nStart = GetTimeMillis();
767  // Flush log data to the actual data file on all files that are not in use
768  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
769  if (!fDbEnvInit)
770  return;
771  {
772  LOCK(cs_db);
773  std::map<std::string, int>::iterator mi = mapFileUseCount.begin();
774  while (mi != mapFileUseCount.end()) {
775  std::string strFile = (*mi).first;
776  int nRefCount = (*mi).second;
777  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
778  if (nRefCount == 0) {
779  // Move log data to the dat file
780  CloseDb(strFile);
781  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile);
782  dbenv->txn_checkpoint(0, 0, 0);
783  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile);
784  if (!fMockDb)
785  dbenv->lsn_reset(strFile.c_str(), 0);
786  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile);
787  mapFileUseCount.erase(mi++);
788  } else
789  mi++;
790  }
791  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
792  if (fShutdown) {
793  char** listp;
794  if (mapFileUseCount.empty()) {
795  dbenv->log_archive(&listp, DB_ARCH_REMOVE);
796  Close();
797  if (!fMockDb) {
798  fs::remove_all(fs::path(strPath) / "database");
799  }
800  }
801  }
802  }
803 }
804 
806 {
807  if (database.IsDummy()) {
808  return true;
809  }
810  bool ret = false;
811  BerkeleyEnvironment *env = database.env.get();
812  const std::string& strFile = database.strFile;
813  TRY_LOCK(cs_db, lockDb);
814  if (lockDb)
815  {
816  // Don't do this if any databases are in use
817  int nRefCount = 0;
818  std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin();
819  while (mit != env->mapFileUseCount.end())
820  {
821  nRefCount += (*mit).second;
822  mit++;
823  }
824 
825  if (nRefCount == 0)
826  {
827  boost::this_thread::interruption_point();
828  std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile);
829  if (mi != env->mapFileUseCount.end())
830  {
831  LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile);
832  int64_t nStart = GetTimeMillis();
833 
834  // Flush wallet file so it's self contained
835  env->CloseDb(strFile);
836  env->CheckpointLSN(strFile);
837 
838  env->mapFileUseCount.erase(mi++);
839  LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
840  ret = true;
841  }
842  }
843  }
844 
845  return ret;
846 }
847 
848 bool BerkeleyDatabase::Rewrite(const char* pszSkip)
849 {
850  return BerkeleyBatch::Rewrite(*this, pszSkip);
851 }
852 
853 bool BerkeleyDatabase::Backup(const std::string& strDest)
854 {
855  if (IsDummy()) {
856  return false;
857  }
858  while (true)
859  {
860  {
861  LOCK(cs_db);
862  if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
863  {
864  // Flush log data to the dat file
865  env->CloseDb(strFile);
866  env->CheckpointLSN(strFile);
867  env->mapFileUseCount.erase(strFile);
868 
869  // Copy wallet file
870  fs::path pathSrc = env->Directory() / strFile;
871  fs::path pathDest(strDest);
872  if (fs::is_directory(pathDest))
873  pathDest /= strFile;
874 
875  try {
876  if (fs::equivalent(pathSrc, pathDest)) {
877  LogPrintf("cannot backup to wallet source file %s\n", pathDest.string());
878  return false;
879  }
880 
881  fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
882  LogPrintf("copied %s to %s\n", strFile, pathDest.string());
883  return true;
884  } catch (const fs::filesystem_error& e) {
885  LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e));
886  return false;
887  }
888  }
889  }
890  MilliSleep(100);
891  }
892 }
893 
894 void BerkeleyDatabase::Flush(bool shutdown)
895 {
896  if (!IsDummy()) {
897  env->Flush(shutdown);
898  if (shutdown) {
899  LOCK(cs_db);
900  g_dbenvs.erase(env->Directory().string());
901  env = nullptr;
902  } else {
903  // TODO: To avoid g_dbenvs.erase erasing the environment prematurely after the
904  // first database shutdown when multiple databases are open in the same
905  // environment, should replace raw database `env` pointers with shared or weak
906  // pointers, or else separate the database and environment shutdowns so
907  // environments can be shut down after databases.
908  env->m_fileids.erase(strFile);
909  }
910  }
911 }
912 
914 {
915  if (!IsDummy()) {
916  env->ReloadDbEnv();
917  }
918 }
void Flush(bool shutdown)
Make sure all changes are flushed to disk.
Definition: db.cpp:894
bool fReadOnly
Definition: db.h:226
BerkeleyEnvironment()
Construct an in-memory mock Berkeley environment for testing.
Definition: db.cpp:241
bool fDbEnvInit
Definition: db.h:37
#define LogPrint(category,...)
Definition: logging.h:179
FILE * fopen(const fs::path &p, const char *mode)
Definition: fs.cpp:19
void Close()
Definition: db.cpp:117
#define TRY_LOCK(cs, name)
Definition: sync.h:183
VerifyResult
Verify that database file strFile is OK.
Definition: db.h:66
std::unordered_map< std::string, WalletDatabaseFileId > m_fileids
Definition: db.h:47
int64_t GetTimeMillis()
Returns the system time (not mockable)
Definition: time.cpp:54
void Close()
Definition: db.cpp:615
#define strprintf
Format arguments and return the string or write to given std::ostream (see tinyformat::format doc for...
Definition: tinyformat.h:1164
bool fFlushOnClose
Definition: db.h:227
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by Boost&#39;s create_directories if the requested directory exists...
Definition: system.cpp:907
std::vector< unsigned char > ParseHex(const char *psz)
bool Exists(const K &key)
Definition: db.h:323
bool Write(const K &key, const T &value, bool fOverwrite=true)
Definition: db.h:279
void MilliSleep(int64_t n)
Definition: time.cpp:75
value_type * data()
Definition: streams.h:301
An instance of this class represents one database.
Definition: db.h:111
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:163
bool IsDatabaseLoaded(const std::string &db_filename) const
Definition: db.h:57
Dbc * GetCursor()
Definition: db.h:339
Double ended buffer combining vector and stream-like interfaces.
Definition: streams.h:201
void ReloadDbEnv()
Definition: db.cpp:913
bool Backup(const std::string &strDest)
Back up the entire database to a file.
Definition: db.cpp:853
bool GetBoolArg(const std::string &strArg, bool fDefault) const
Return boolean argument or default value.
Definition: system.cpp:384
static const char * filenames[]
Definition: unitester.cpp:80
fs::path Directory() const
Definition: db.h:58
bool Open(bool retry)
Definition: db.cpp:167
u_int32_t get_size() const
Definition: db.cpp:317
bool IsWalletLoaded(const fs::path &wallet_path)
Return whether a wallet database is currently loaded.
Definition: db.cpp:72
static bool PeriodicFlush(BerkeleyDatabase &database)
Definition: db.cpp:805
int ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, CDataStream &ssValue)
Definition: db.h:350
std::shared_ptr< BerkeleyEnvironment > env
Pointer to shared database environment.
Definition: db.h:184
static bool VerifyDatabaseFile(const fs::path &file_path, std::vector< std::string > &warnings, std::string &errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc)
Definition: db.cpp:413
std::map< std::string, int > mapFileUseCount
Definition: db.h:45
std::unique_ptr< DbEnv > dbenv
Definition: db.h:44
void CheckpointLSN(const std::string &strFile)
Definition: db.cpp:507
static const char * DATA_END
Definition: db.cpp:444
std::pair< std::vector< unsigned char >, std::vector< unsigned char > > KeyValPair
Salvage data from a file that Verify says is bad.
Definition: db.h:78
bool LockDirectory(const fs::path &directory, const std::string lockfile_name, bool probe_only)
Definition: system.cpp:87
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
Definition: cleanse.cpp:14
size_type size() const
Definition: streams.h:292
std::string strFile
Definition: db.h:224
static void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename)
Definition: db.cpp:56
bool Rewrite(const char *pszSkip=nullptr)
Rewrite the entire database on disk, with the exception of key pszSkip if non-zero.
Definition: db.cpp:848
#define LOCK(cs)
Definition: sync.h:179
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:36
static const char * HEADER_END
Definition: db.cpp:442
fs::path WalletDataFilePath(const fs::path &wallet_path)
Given a wallet directory path or legacy file path, return path to main data file in the wallet databa...
Definition: db.cpp:84
DbTxn * activeTxn
Definition: db.h:225
RAII class that provides access to a Berkeley database.
Definition: db.h:200
void Flush()
Definition: db.cpp:595
std::string strFile
Definition: db.h:190
static const unsigned int DEFAULT_WALLET_DBLOGSIZE
Definition: db.h:24
bool operator==(const WalletDatabaseFileId &rhs) const
Definition: db.cpp:51
static bool VerifyEnvironment(const fs::path &file_path, std::string &errorStr)
Definition: db.cpp:396
static bool Recover(const fs::path &file_path, void *callbackDataIn, bool(*recoverKVcallback)(void *callbackData, CDataStream ssKey, CDataStream ssValue), std::string &out_backup_filename)
Definition: db.cpp:327
bool Salvage(const std::string &strFile, bool fAggressive, std::vector< KeyValPair > &vResult)
Definition: db.cpp:446
int flags
Definition: bitcoin-tx.cpp:508
void ReloadDbEnv()
Definition: db.cpp:649
static bool Rewrite(BerkeleyDatabase &database, const char *pszSkip=nullptr)
Definition: db.cpp:675
std::string get_filesystem_error_message(const fs::filesystem_error &e)
Definition: fs.cpp:107
void IncrementUpdateCounter()
Definition: db.cpp:610
std::shared_ptr< BerkeleyEnvironment > GetWalletEnv(const fs::path &wallet_path, std::string &database_filename)
Get BerkeleyEnvironment and database filename given a wallet path.
Definition: db.cpp:99
std::string strPath
Definition: db.h:41
BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode="r+", bool fFlushOnCloseIn=true)
Definition: db.cpp:516
void UnlockDirectory(const fs::path &directory, const std::string &lockfile_name)
Definition: system.cpp:111
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:372
u_int8_t value[DB_FILE_ID_LEN]
Definition: db.h:28
std::string HexStr(const T itbegin, const T itend)
Definition: strencodings.h:125
ArgsManager gArgs
Definition: system.cpp:76
void Flush(bool fShutdown)
Definition: db.cpp:764
Db * pdb
Definition: db.h:223
#define AssertLockNotHeld(cs)
Definition: sync.h:72
static int count
Definition: tests.c:45
#define GUARDED_BY(x)
Definition: threadsafety.h:38
bool(* recoverFunc_type)(const fs::path &file_path, std::string &out_backup_filename)
Definition: db.h:69
VerifyResult Verify(const std::string &strFile, recoverFunc_type recoverFunc, std::string &out_backup_filename)
Definition: db.cpp:271
static const bool DEFAULT_WALLET_PRIVDB
Definition: db.h:25
void clear()
Definition: streams.h:298
void Reset()
Definition: db.cpp:148
void CloseDb(const std::string &strFile)
Definition: db.cpp:634
bool IsMock() const
Definition: db.h:55
~BerkeleyEnvironment()
Definition: db.cpp:160
int64_t GetTime()
Return system time (or mocked time, if set)
Definition: time.cpp:20
auto it
Definition: validation.cpp:362
bool fMockDb
Definition: db.h:38
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
std::unique_ptr< Db > m_db
Database pointer.
Definition: db.h:187
const void * get_data() const
Definition: db.cpp:312
std::condition_variable_any m_db_in_use
Definition: db.h:48
bool IsDummy()
Return whether this database handle is a dummy for testing.
Definition: db.h:196