Bitcoin Core 29.99.0
P2P Digital Currency
dbwrapper.cpp
Go to the documentation of this file.
1// Copyright (c) 2012-present The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5#include <dbwrapper.h>
6
7#include <logging.h>
8#include <random.h>
9#include <serialize.h>
10#include <span.h>
11#include <streams.h>
12#include <util/fs.h>
13#include <util/fs_helpers.h>
14#include <util/obfuscation.h>
15#include <util/strencodings.h>
16
17#include <algorithm>
18#include <cassert>
19#include <cstdarg>
20#include <cstdint>
21#include <cstdio>
22#include <leveldb/cache.h>
23#include <leveldb/db.h>
24#include <leveldb/env.h>
25#include <leveldb/filter_policy.h>
26#include <leveldb/helpers/memenv/memenv.h>
27#include <leveldb/iterator.h>
28#include <leveldb/options.h>
29#include <leveldb/slice.h>
30#include <leveldb/status.h>
31#include <leveldb/write_batch.h>
32#include <memory>
33#include <optional>
34#include <utility>
35
36static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
37
38bool DestroyDB(const std::string& path_str)
39{
40 return leveldb::DestroyDB(path_str, {}).ok();
41}
42
45static void HandleError(const leveldb::Status& status)
46{
47 if (status.ok())
48 return;
49 const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
50 LogPrintf("%s\n", errmsg);
51 LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
52 throw dbwrapper_error(errmsg);
53}
54
55class CBitcoinLevelDBLogger : public leveldb::Logger {
56public:
57 // This code is adapted from posix_logger.h, which is why it is using vsprintf.
58 // Please do not do this in normal code
59 void Logv(const char * format, va_list ap) override {
61 return;
62 }
63 char buffer[500];
64 for (int iter = 0; iter < 2; iter++) {
65 char* base;
66 int bufsize;
67 if (iter == 0) {
68 bufsize = sizeof(buffer);
69 base = buffer;
70 }
71 else {
72 bufsize = 30000;
73 base = new char[bufsize];
74 }
75 char* p = base;
76 char* limit = base + bufsize;
77
78 // Print the message
79 if (p < limit) {
80 va_list backup_ap;
81 va_copy(backup_ap, ap);
82 // Do not use vsnprintf elsewhere in bitcoin source code, see above.
83 p += vsnprintf(p, limit - p, format, backup_ap);
84 va_end(backup_ap);
85 }
86
87 // Truncate to available space if necessary
88 if (p >= limit) {
89 if (iter == 0) {
90 continue; // Try again with larger buffer
91 }
92 else {
93 p = limit - 1;
94 }
95 }
96
97 // Add newline if necessary
98 if (p == base || p[-1] != '\n') {
99 *p++ = '\n';
100 }
101
102 assert(p <= limit);
103 base[std::min(bufsize - 1, (int)(p - base))] = '\0';
104 LogDebug(BCLog::LEVELDB, "%s\n", util::RemoveSuffixView(base, "\n"));
105 if (base != buffer) {
106 delete[] base;
107 }
108 break;
109 }
110 }
111};
112
113static void SetMaxOpenFiles(leveldb::Options *options) {
114 // On most platforms the default setting of max_open_files (which is 1000)
115 // is optimal. On Windows using a large file count is OK because the handles
116 // do not interfere with select() loops. On 64-bit Unix hosts this value is
117 // also OK, because up to that amount LevelDB will use an mmap
118 // implementation that does not use extra file descriptors (the fds are
119 // closed after being mmap'ed).
120 //
121 // Increasing the value beyond the default is dangerous because LevelDB will
122 // fall back to a non-mmap implementation when the file count is too large.
123 // On 32-bit Unix host we should decrease the value because the handles use
124 // up real fds, and we want to avoid fd exhaustion issues.
125 //
126 // See PR #12495 for further discussion.
127
128 int default_open_files = options->max_open_files;
129#ifndef WIN32
130 if (sizeof(void*) < 8) {
131 options->max_open_files = 64;
132 }
133#endif
134 LogDebug(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
135 options->max_open_files, default_open_files);
136}
137
138static leveldb::Options GetOptions(size_t nCacheSize)
139{
140 leveldb::Options options;
141 options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
142 options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
143 options.filter_policy = leveldb::NewBloomFilterPolicy(10);
144 options.compression = leveldb::kNoCompression;
145 options.info_log = new CBitcoinLevelDBLogger();
146 if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
147 // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
148 // on corruption in later versions.
149 options.paranoid_checks = true;
150 }
151 options.max_file_size = std::max(options.max_file_size, DBWRAPPER_MAX_FILE_SIZE);
152 SetMaxOpenFiles(&options);
153 return options;
154}
155
157 leveldb::WriteBatch batch;
158};
159
161 : parent{_parent},
162 m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()}
163{
164 Clear();
165};
166
167CDBBatch::~CDBBatch() = default;
168
170{
171 m_impl_batch->batch.Clear();
172}
173
174void CDBBatch::WriteImpl(std::span<const std::byte> key, DataStream& ssValue)
175{
176 leveldb::Slice slKey(CharCast(key.data()), key.size());
178 leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
179 m_impl_batch->batch.Put(slKey, slValue);
180}
181
182void CDBBatch::EraseImpl(std::span<const std::byte> key)
183{
184 leveldb::Slice slKey(CharCast(key.data()), key.size());
185 m_impl_batch->batch.Delete(slKey);
186}
187
189{
190 return m_impl_batch->batch.ApproximateSize();
191}
192
195 leveldb::Env* penv;
196
198 leveldb::Options options;
199
201 leveldb::ReadOptions readoptions;
202
204 leveldb::ReadOptions iteroptions;
205
207 leveldb::WriteOptions writeoptions;
208
210 leveldb::WriteOptions syncoptions;
211
213 leveldb::DB* pdb;
214};
215
217 : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
218{
219 DBContext().penv = nullptr;
220 DBContext().readoptions.verify_checksums = true;
221 DBContext().iteroptions.verify_checksums = true;
222 DBContext().iteroptions.fill_cache = false;
223 DBContext().syncoptions.sync = true;
224 DBContext().options = GetOptions(params.cache_bytes);
225 DBContext().options.create_if_missing = true;
226 if (params.memory_only) {
227 DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
228 DBContext().options.env = DBContext().penv;
229 } else {
230 if (params.wipe_data) {
231 LogInfo("Wiping LevelDB in %s", fs::PathToString(params.path));
232 leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
233 HandleError(result);
234 }
236 LogInfo("Opening LevelDB in %s", fs::PathToString(params.path));
237 }
238 // PathToString() return value is safe to pass to leveldb open function,
239 // because on POSIX leveldb passes the byte string directly to ::open(), and
240 // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
241 // (see env_posix.cc and env_windows.cc).
242 leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
243 HandleError(status);
244 LogInfo("Opened LevelDB successfully");
245
246 if (params.options.force_compact) {
247 LogInfo("Starting database compaction of %s", fs::PathToString(params.path));
248 DBContext().pdb->CompactRange(nullptr, nullptr);
249 LogInfo("Finished database compaction of %s", fs::PathToString(params.path));
250 }
251
252 if (!Read(OBFUSCATION_KEY, m_obfuscation) && params.obfuscate && IsEmpty()) {
253 // Generate and write the new obfuscation key.
255 assert(!m_obfuscation); // Make sure the key is written without obfuscation.
256 Write(OBFUSCATION_KEY, obfuscation);
257 m_obfuscation = obfuscation;
258 LogInfo("Wrote new obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
259 }
260 LogInfo("Using obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
261}
262
264{
265 delete DBContext().pdb;
266 DBContext().pdb = nullptr;
267 delete DBContext().options.filter_policy;
268 DBContext().options.filter_policy = nullptr;
269 delete DBContext().options.info_log;
270 DBContext().options.info_log = nullptr;
271 delete DBContext().options.block_cache;
272 DBContext().options.block_cache = nullptr;
273 delete DBContext().penv;
274 DBContext().options.env = nullptr;
275}
276
277bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
278{
279 const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug);
280 double mem_before = 0;
281 if (log_memory) {
282 mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
283 }
284 leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
285 HandleError(status);
286 if (log_memory) {
287 double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
288 LogDebug(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
289 m_name, mem_before, mem_after);
290 }
291 return true;
292}
293
295{
296 std::string memory;
297 std::optional<size_t> parsed;
298 if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
299 LogDebug(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
300 return 0;
301 }
302 return parsed.value();
303}
304
305std::optional<std::string> CDBWrapper::ReadImpl(std::span<const std::byte> key) const
306{
307 leveldb::Slice slKey(CharCast(key.data()), key.size());
308 std::string strValue;
309 leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
310 if (!status.ok()) {
311 if (status.IsNotFound())
312 return std::nullopt;
313 LogPrintf("LevelDB read failure: %s\n", status.ToString());
314 HandleError(status);
315 }
316 return strValue;
317}
318
319bool CDBWrapper::ExistsImpl(std::span<const std::byte> key) const
320{
321 leveldb::Slice slKey(CharCast(key.data()), key.size());
322
323 std::string strValue;
324 leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
325 if (!status.ok()) {
326 if (status.IsNotFound())
327 return false;
328 LogPrintf("LevelDB read failure: %s\n", status.ToString());
329 HandleError(status);
330 }
331 return true;
332}
333
334size_t CDBWrapper::EstimateSizeImpl(std::span<const std::byte> key1, std::span<const std::byte> key2) const
335{
336 leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
337 leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
338 uint64_t size = 0;
339 leveldb::Range range(slKey1, slKey2);
340 DBContext().pdb->GetApproximateSizes(&range, 1, &size);
341 return size;
342}
343
345{
346 std::unique_ptr<CDBIterator> it(NewIterator());
347 it->SeekToFirst();
348 return !(it->Valid());
349}
350
352 const std::unique_ptr<leveldb::Iterator> iter;
353
354 explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
355};
356
357CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
358 m_impl_iter(std::move(_piter)) {}
359
361{
362 return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
363}
364
365void CDBIterator::SeekImpl(std::span<const std::byte> key)
366{
367 leveldb::Slice slKey(CharCast(key.data()), key.size());
368 m_impl_iter->iter->Seek(slKey);
369}
370
371std::span<const std::byte> CDBIterator::GetKeyImpl() const
372{
373 return MakeByteSpan(m_impl_iter->iter->key());
374}
375
376std::span<const std::byte> CDBIterator::GetValueImpl() const
377{
378 return MakeByteSpan(m_impl_iter->iter->value());
379}
380
381CDBIterator::~CDBIterator() = default;
382bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
383void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
384void CDBIterator::Next() { m_impl_iter->iter->Next(); }
385
387
389{
390 return w.m_obfuscation;
391}
392
393} // namespace dbwrapper_private
void Logv(const char *format, va_list ap) override
Definition: dbwrapper.cpp:59
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:72
void WriteImpl(std::span< const std::byte > key, DataStream &ssValue)
Definition: dbwrapper.cpp:174
const std::unique_ptr< WriteBatchImpl > m_impl_batch
Definition: dbwrapper.h:79
void EraseImpl(std::span< const std::byte > key)
Definition: dbwrapper.cpp:182
DataStream ssValue
Definition: dbwrapper.h:82
void Clear()
Definition: dbwrapper.cpp:169
CDBBatch(const CDBWrapper &_parent)
Definition: dbwrapper.cpp:160
size_t ApproximateSize() const
Definition: dbwrapper.cpp:188
const CDBWrapper & parent
Definition: dbwrapper.h:76
CDBIterator(const CDBWrapper &_parent, std::unique_ptr< IteratorImpl > _piter)
Definition: dbwrapper.cpp:357
const std::unique_ptr< IteratorImpl > m_impl_iter
Definition: dbwrapper.h:126
void SeekImpl(std::span< const std::byte > key)
Definition: dbwrapper.cpp:365
std::span< const std::byte > GetKeyImpl() const
Definition: dbwrapper.cpp:371
bool Valid() const
Definition: dbwrapper.cpp:382
void SeekToFirst()
Definition: dbwrapper.cpp:383
void Next()
Definition: dbwrapper.cpp:384
std::span< const std::byte > GetValueImpl() const
Definition: dbwrapper.cpp:376
std::optional< std::string > ReadImpl(std::span< const std::byte > key) const
Definition: dbwrapper.cpp:305
size_t EstimateSizeImpl(std::span< const std::byte > key1, std::span< const std::byte > key2) const
Definition: dbwrapper.cpp:334
size_t DynamicMemoryUsage() const
Definition: dbwrapper.cpp:294
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:277
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:213
CDBIterator * NewIterator()
Definition: dbwrapper.cpp:360
std::string m_name
the name of this database
Definition: dbwrapper.h:186
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:233
CDBWrapper(const DBParams &params)
Definition: dbwrapper.cpp:216
bool ExistsImpl(std::span< const std::byte > key) const
Definition: dbwrapper.cpp:319
Obfuscation m_obfuscation
optional XOR-obfuscation of the database
Definition: dbwrapper.h:189
static const std::string OBFUSCATION_KEY
obfuscation key storage key, null-prefixed to avoid collisions
Definition: dbwrapper.h:192
auto & DBContext() const LIFETIMEBOUND
Definition: dbwrapper.h:203
bool IsEmpty()
Return true if the database managed by this class contains no entries.
Definition: dbwrapper.cpp:344
Double ended buffer combining vector and stream-like interfaces.
Definition: streams.h:130
size_type size() const
Definition: streams.h:164
value_type * data()
Definition: streams.h:171
Fast randomness source.
Definition: random.h:386
std::string HexKey() const
Definition: obfuscation.h:79
static constexpr size_t KEY_SIZE
Definition: obfuscation.h:23
std::vector< B > randbytes(size_t len) noexcept
Generate random bytes.
Definition: random.h:297
static leveldb::Options GetOptions(size_t nCacheSize)
Definition: dbwrapper.cpp:138
static auto CharCast(const std::byte *data)
Definition: dbwrapper.cpp:36
bool DestroyDB(const std::string &path_str)
Definition: dbwrapper.cpp:38
static void SetMaxOpenFiles(leveldb::Options *options)
Definition: dbwrapper.cpp:113
static void HandleError(const leveldb::Status &status)
Handle database error by throwing dbwrapper_error exception.
Definition: dbwrapper.cpp:45
static const size_t DBWRAPPER_MAX_FILE_SIZE
Definition: dbwrapper.h:24
static std::string PathToString(const path &path)
Convert path object to a byte string.
Definition: fs.h:151
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by create_directories if the requested directory exists.
Definition: fs_helpers.cpp:255
#define LogInfo(...)
Definition: logging.h:356
static bool LogAcceptCategory(BCLog::LogFlags category, BCLog::Level level)
Return true if log accepts specified category, at the specified level.
Definition: logging.h:328
#define LogDebug(category,...)
Definition: logging.h:381
#define LogPrintf(...)
Definition: logging.h:361
@ LEVELDB
Definition: logging.h:86
These should be considered an implementation detail of the specific database.
Definition: dbwrapper.cpp:386
const Obfuscation & GetObfuscation(const CDBWrapper &w)
Work around circular dependency, as well as for testing in dbwrapper_tests.
Definition: dbwrapper.cpp:388
void format(std::ostream &out, FormatStringCheck< sizeof...(Args)> fmt, const Args &... args)
Format list of arguments to the stream according to given format string.
Definition: tinyformat.h:1079
std::string_view RemoveSuffixView(std::string_view str, std::string_view suffix)
Definition: string.h:173
auto MakeByteSpan(const V &v) noexcept
Definition: span.h:84
leveldb::WriteBatch batch
Definition: dbwrapper.cpp:157
IteratorImpl(leveldb::Iterator *_iter)
Definition: dbwrapper.cpp:354
const std::unique_ptr< leveldb::Iterator > iter
Definition: dbwrapper.cpp:352
bool force_compact
Compact database on startup.
Definition: dbwrapper.h:29
Application-specific storage settings.
Definition: dbwrapper.h:33
DBOptions options
Passed-through options.
Definition: dbwrapper.h:46
bool obfuscate
If true, store data obfuscated via simple XOR.
Definition: dbwrapper.h:44
bool wipe_data
If true, remove all existing data.
Definition: dbwrapper.h:41
size_t cache_bytes
Configures various leveldb cache settings.
Definition: dbwrapper.h:37
fs::path path
Location in the filesystem where leveldb data will be stored.
Definition: dbwrapper.h:35
bool memory_only
If true, use leveldb's memory environment.
Definition: dbwrapper.h:39
leveldb::Env * penv
custom environment this database is using (may be nullptr in case of default environment)
Definition: dbwrapper.cpp:195
leveldb::ReadOptions iteroptions
options used when iterating over values of the database
Definition: dbwrapper.cpp:204
leveldb::ReadOptions readoptions
options used when reading from the database
Definition: dbwrapper.cpp:201
leveldb::Options options
database options used
Definition: dbwrapper.cpp:198
leveldb::DB * pdb
the database itself
Definition: dbwrapper.cpp:213
leveldb::WriteOptions syncoptions
options used when sync writing to the database
Definition: dbwrapper.cpp:210
leveldb::WriteOptions writeoptions
options used when writing to the database
Definition: dbwrapper.cpp:207
assert(!tx.IsCoinBase())