Bitcoin Core 29.99.0
P2P Digital Currency
lockedpool.cpp
Go to the documentation of this file.
1// Copyright (c) 2016-present The Bitcoin Core developers
2// Distributed under the MIT software license, see the accompanying
3// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
6#include <support/cleanse.h>
7
8#ifdef WIN32
9#include <windows.h>
10#else
11#include <sys/mman.h>
12#include <sys/resource.h>
13#include <unistd.h>
14#endif
15
16#include <algorithm>
17#include <limits>
18#include <stdexcept>
19#include <utility>
20#ifdef ARENA_DEBUG
21#include <iomanip>
22#include <iostream>
23#endif
24
26
27/*******************************************************************************/
28// Utilities
29//
31static inline size_t align_up(size_t x, size_t align)
32{
33 return (x + align - 1) & ~(align - 1);
34}
35
36/*******************************************************************************/
37// Implementation: Arena
38
39Arena::Arena(void *base_in, size_t size_in, size_t alignment_in):
40 base(base_in), end(static_cast<char*>(base_in) + size_in), alignment(alignment_in)
41{
42 // Start with one free chunk that covers the entire arena
43 auto it = size_to_free_chunk.emplace(size_in, base);
44 chunks_free.emplace(base, it);
45 chunks_free_end.emplace(static_cast<char*>(base) + size_in, it);
46}
47
48Arena::~Arena() = default;
49
50void* Arena::alloc(size_t size)
51{
52 // Round to next multiple of alignment
53 size = align_up(size, alignment);
54
55 // Don't handle zero-sized chunks
56 if (size == 0)
57 return nullptr;
58
59 // Pick a large enough free-chunk. Returns an iterator pointing to the first element that is not less than key.
60 // This allocation strategy is best-fit. According to "Dynamic Storage Allocation: A Survey and Critical Review",
61 // Wilson et. al. 1995, https://www.scs.stanford.edu/14wi-cs140/sched/readings/wilson.pdf, best-fit and first-fit
62 // policies seem to work well in practice.
63 auto size_ptr_it = size_to_free_chunk.lower_bound(size);
64 if (size_ptr_it == size_to_free_chunk.end())
65 return nullptr;
66
67 // Create the used-chunk, taking its space from the end of the free-chunk
68 const size_t size_remaining = size_ptr_it->first - size;
69 char* const free_chunk = static_cast<char*>(size_ptr_it->second);
70 auto allocated = chunks_used.emplace(free_chunk + size_remaining, size).first;
71 chunks_free_end.erase(free_chunk + size_ptr_it->first);
72 if (size_ptr_it->first == size) {
73 // whole chunk is used up
74 chunks_free.erase(size_ptr_it->second);
75 } else {
76 // still some memory left in the chunk
77 auto it_remaining = size_to_free_chunk.emplace(size_remaining, size_ptr_it->second);
78 chunks_free[size_ptr_it->second] = it_remaining;
79 chunks_free_end.emplace(free_chunk + size_remaining, it_remaining);
80 }
81 size_to_free_chunk.erase(size_ptr_it);
82
83 return allocated->first;
84}
85
86void Arena::free(void *ptr)
87{
88 // Freeing the nullptr pointer is OK.
89 if (ptr == nullptr) {
90 return;
91 }
92
93 // Remove chunk from used map
94 auto i = chunks_used.find(ptr);
95 if (i == chunks_used.end()) {
96 throw std::runtime_error("Arena: invalid or double free");
97 }
98 auto freed = std::make_pair(static_cast<char*>(i->first), i->second);
99 chunks_used.erase(i);
100
101 // coalesce freed with previous chunk
102 auto prev = chunks_free_end.find(freed.first);
103 if (prev != chunks_free_end.end()) {
104 freed.first -= prev->second->first;
105 freed.second += prev->second->first;
106 size_to_free_chunk.erase(prev->second);
107 chunks_free_end.erase(prev);
108 }
109
110 // coalesce freed with chunk after freed
111 auto next = chunks_free.find(freed.first + freed.second);
112 if (next != chunks_free.end()) {
113 freed.second += next->second->first;
114 size_to_free_chunk.erase(next->second);
115 chunks_free.erase(next);
116 }
117
118 // Add/set space with coalesced free chunk
119 auto it = size_to_free_chunk.emplace(freed.second, freed.first);
120 chunks_free[freed.first] = it;
121 chunks_free_end[freed.first + freed.second] = it;
122}
123
125{
126 Arena::Stats r{ 0, 0, 0, chunks_used.size(), chunks_free.size() };
127 for (const auto& chunk: chunks_used)
128 r.used += chunk.second;
129 for (const auto& chunk: chunks_free)
130 r.free += chunk.second->first;
131 r.total = r.used + r.free;
132 return r;
133}
134
135#ifdef ARENA_DEBUG
136static void printchunk(void* base, size_t sz, bool used) {
137 std::cout <<
138 "0x" << std::hex << std::setw(16) << std::setfill('0') << base <<
139 " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz <<
140 " 0x" << used << std::endl;
141}
142void Arena::walk() const
143{
144 for (const auto& chunk: chunks_used)
145 printchunk(chunk.first, chunk.second, true);
146 std::cout << std::endl;
147 for (const auto& chunk: chunks_free)
148 printchunk(chunk.first, chunk.second->first, false);
149 std::cout << std::endl;
150}
151#endif
152
153/*******************************************************************************/
154// Implementation: Win32LockedPageAllocator
155
156#ifdef WIN32
159class Win32LockedPageAllocator: public LockedPageAllocator
160{
161public:
162 Win32LockedPageAllocator();
163 void* AllocateLocked(size_t len, bool *lockingSuccess) override;
164 void FreeLocked(void* addr, size_t len) override;
165 size_t GetLimit() override;
166private:
167 size_t page_size;
168};
169
170Win32LockedPageAllocator::Win32LockedPageAllocator()
171{
172 // Determine system page size in bytes
173 SYSTEM_INFO sSysInfo;
174 GetSystemInfo(&sSysInfo);
175 page_size = sSysInfo.dwPageSize;
176}
177void *Win32LockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess)
178{
179 len = align_up(len, page_size);
180 void *addr = VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
181 if (addr) {
182 // VirtualLock is used to attempt to keep keying material out of swap. Note
183 // that it does not provide this as a guarantee, but, in practice, memory
184 // that has been VirtualLock'd almost never gets written to the pagefile
185 // except in rare circumstances where memory is extremely low.
186 *lockingSuccess = VirtualLock(const_cast<void*>(addr), len) != 0;
187 }
188 return addr;
189}
190void Win32LockedPageAllocator::FreeLocked(void* addr, size_t len)
191{
192 len = align_up(len, page_size);
193 memory_cleanse(addr, len);
194 VirtualUnlock(const_cast<void*>(addr), len);
195}
196
197size_t Win32LockedPageAllocator::GetLimit()
198{
199 size_t min, max;
200 if(GetProcessWorkingSetSize(GetCurrentProcess(), &min, &max) != 0) {
201 return min;
202 }
203 return std::numeric_limits<size_t>::max();
204}
205#endif
206
207/*******************************************************************************/
208// Implementation: PosixLockedPageAllocator
209
210#ifndef WIN32
215{
216public:
218 void* AllocateLocked(size_t len, bool *lockingSuccess) override;
219 void FreeLocked(void* addr, size_t len) override;
220 size_t GetLimit() override;
221private:
222 size_t page_size;
223};
224
226{
227 // Determine system page size in bytes
228#if defined(PAGESIZE) // defined in limits.h
229 page_size = PAGESIZE;
230#else // assume some POSIX OS
231 page_size = sysconf(_SC_PAGESIZE);
232#endif
233}
234
235void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess)
236{
237 void *addr;
238 len = align_up(len, page_size);
239 addr = mmap(nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
240 if (addr == MAP_FAILED) {
241 return nullptr;
242 }
243 if (addr) {
244 *lockingSuccess = mlock(addr, len) == 0;
245#if defined(MADV_DONTDUMP) // Linux
246 madvise(addr, len, MADV_DONTDUMP);
247#elif defined(MADV_NOCORE) // FreeBSD
248 madvise(addr, len, MADV_NOCORE);
249#endif
250 }
251 return addr;
252}
253void PosixLockedPageAllocator::FreeLocked(void* addr, size_t len)
254{
255 len = align_up(len, page_size);
256 memory_cleanse(addr, len);
257 munlock(addr, len);
258 munmap(addr, len);
259}
261{
262#ifdef RLIMIT_MEMLOCK
263 struct rlimit rlim;
264 if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) {
265 if (rlim.rlim_cur != RLIM_INFINITY) {
266 return rlim.rlim_cur;
267 }
268 }
269#endif
270 return std::numeric_limits<size_t>::max();
271}
272#endif
273
274/*******************************************************************************/
275// Implementation: LockedPool
276
277LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in)
278 : allocator(std::move(allocator_in)), lf_cb(lf_cb_in)
279{
280}
281
282LockedPool::~LockedPool() = default;
283
284void* LockedPool::alloc(size_t size)
285{
286 std::lock_guard<std::mutex> lock(mutex);
287
288 // Don't handle impossible sizes
289 if (size == 0 || size > ARENA_SIZE)
290 return nullptr;
291
292 // Try allocating from each current arena
293 for (auto &arena: arenas) {
294 void *addr = arena.alloc(size);
295 if (addr) {
296 return addr;
297 }
298 }
299 // If that fails, create a new one
301 return arenas.back().alloc(size);
302 }
303 return nullptr;
304}
305
306void LockedPool::free(void *ptr)
307{
308 std::lock_guard<std::mutex> lock(mutex);
309 // TODO we can do better than this linear search by keeping a map of arena
310 // extents to arena, and looking up the address.
311 for (auto &arena: arenas) {
312 if (arena.addressInArena(ptr)) {
313 arena.free(ptr);
314 return;
315 }
316 }
317 throw std::runtime_error("LockedPool: invalid address not pointing to any arena");
318}
319
321{
322 std::lock_guard<std::mutex> lock(mutex);
324 for (const auto &arena: arenas) {
325 Arena::Stats i = arena.stats();
326 r.used += i.used;
327 r.free += i.free;
328 r.total += i.total;
329 r.chunks_used += i.chunks_used;
330 r.chunks_free += i.chunks_free;
331 }
332 return r;
333}
334
335bool LockedPool::new_arena(size_t size, size_t align)
336{
337 bool locked;
338 // If this is the first arena, handle this specially: Cap the upper size
339 // by the process limit. This makes sure that the first arena will at least
340 // be locked. An exception to this is if the process limit is 0:
341 // in this case no memory can be locked at all so we'll skip past this logic.
342 if (arenas.empty()) {
343 size_t limit = allocator->GetLimit();
344 if (limit > 0) {
345 size = std::min(size, limit);
346 }
347 }
348 void *addr = allocator->AllocateLocked(size, &locked);
349 if (!addr) {
350 return false;
351 }
352 if (locked) {
354 } else if (lf_cb) { // Call the locking-failed callback if locking failed
355 if (!lf_cb()) { // If the callback returns false, free the memory and fail, otherwise consider the user warned and proceed.
356 allocator->FreeLocked(addr, size);
357 return false;
358 }
359 }
360 arenas.emplace_back(allocator.get(), addr, size, align);
361 return true;
362}
363
364LockedPool::LockedPageArena::LockedPageArena(LockedPageAllocator *allocator_in, void *base_in, size_t size_in, size_t align_in):
365 Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in)
366{
367}
369{
370 allocator->FreeLocked(base, size);
371}
372
373/*******************************************************************************/
374// Implementation: LockedPoolManager
375//
376LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator_in):
377 LockedPool(std::move(allocator_in), &LockedPoolManager::LockingFailed)
378{
379}
380
382{
383 // TODO: log something but how? without including util.h
384 return true;
385}
386
388{
389 // Using a local static instance guarantees that the object is initialized
390 // when it's first needed and also deinitialized after all objects that use
391 // it are done with it. I can think of one unlikely scenario where we may
392 // have a static deinitialization order/problem, but the check in
393 // LockedPoolManagerBase's destructor helps us detect if that ever happens.
394#ifdef WIN32
395 std::unique_ptr<LockedPageAllocator> allocator(new Win32LockedPageAllocator());
396#else
397 std::unique_ptr<LockedPageAllocator> allocator(new PosixLockedPageAllocator());
398#endif
399 static LockedPoolManager instance(std::move(allocator));
401}
void * base
Base address of arena.
Definition: lockedpool.h:106
size_t alignment
Minimum chunk alignment.
Definition: lockedpool.h:110
ChunkToSizeMap chunks_free_end
Map from end of free chunk to its node in size_to_free_chunk.
Definition: lockedpool.h:100
std::unordered_map< void *, size_t > chunks_used
Map from begin of used chunk to its size.
Definition: lockedpool.h:103
void * alloc(size_t size)
Allocate size bytes from this arena.
Definition: lockedpool.cpp:50
SizeToChunkSortedMap size_to_free_chunk
Map to enable O(log(n)) best-fit allocation, as it's sorted by size.
Definition: lockedpool.h:94
Arena(void *base, size_t size, size_t alignment)
Definition: lockedpool.cpp:39
Stats stats() const
Get arena usage statistics.
Definition: lockedpool.cpp:124
ChunkToSizeMap chunks_free
Map from begin of free chunk to its node in size_to_free_chunk.
Definition: lockedpool.h:98
virtual ~Arena()
void free(void *ptr)
Free a previously allocated chunk of memory.
Definition: lockedpool.cpp:86
OS-dependent allocation and deallocation of locked/pinned memory pages.
Definition: lockedpool.h:20
virtual void * AllocateLocked(size_t len, bool *lockingSuccess)=0
Allocate and lock memory pages.
virtual void FreeLocked(void *addr, size_t len)=0
Unlock and free memory pages.
virtual size_t GetLimit()=0
Get the total limit on the amount of memory that may be locked by this process, in bytes.
LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align)
Definition: lockedpool.cpp:364
Pool for locked memory chunks.
Definition: lockedpool.h:127
void free(void *ptr)
Free a previously allocated chunk of memory.
Definition: lockedpool.cpp:306
Stats stats() const
Get pool usage statistics.
Definition: lockedpool.cpp:320
std::unique_ptr< LockedPageAllocator > allocator
Definition: lockedpool.h:183
void * alloc(size_t size)
Allocate size bytes from this arena.
Definition: lockedpool.cpp:284
size_t cumulative_bytes_locked
Definition: lockedpool.h:201
LockedPool(std::unique_ptr< LockedPageAllocator > allocator, LockingFailed_Callback lf_cb_in=nullptr)
Create a new LockedPool.
Definition: lockedpool.cpp:277
LockingFailed_Callback lf_cb
Definition: lockedpool.h:200
std::list< LockedPageArena > arenas
Definition: lockedpool.h:199
bool new_arena(size_t size, size_t align)
Definition: lockedpool.cpp:335
static const size_t ARENA_ALIGN
Chunk alignment.
Definition: lockedpool.h:138
static const size_t ARENA_SIZE
Size of one arena of locked memory.
Definition: lockedpool.h:134
std::mutex mutex
Mutex protects access to this pool's data structures, including arenas.
Definition: lockedpool.h:204
Singleton class to keep track of locked (ie, non-swappable) memory, for use in std::allocator templat...
Definition: lockedpool.h:219
LockedPoolManager(std::unique_ptr< LockedPageAllocator > allocator)
Definition: lockedpool.cpp:376
static bool LockingFailed()
Called when locking fails, warn the user here.
Definition: lockedpool.cpp:381
static LockedPoolManager * _instance
Definition: lockedpool.h:237
static void CreateInstance()
Create a new LockedPoolManager specialized to the OS.
Definition: lockedpool.cpp:387
LockedPageAllocator specialized for OSes that don't try to be special snowflakes.
Definition: lockedpool.cpp:215
void * AllocateLocked(size_t len, bool *lockingSuccess) override
Allocate and lock memory pages.
Definition: lockedpool.cpp:235
void FreeLocked(void *addr, size_t len) override
Unlock and free memory pages.
Definition: lockedpool.cpp:253
size_t GetLimit() override
Get the total limit on the amount of memory that may be locked by this process, in bytes.
Definition: lockedpool.cpp:260
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
Definition: cleanse.cpp:14
Memory statistics.
Definition: lockedpool.h:59
size_t used
Definition: lockedpool.h:60
size_t chunks_used
Definition: lockedpool.h:63
size_t total
Definition: lockedpool.h:62
size_t free
Definition: lockedpool.h:61
size_t chunks_free
Definition: lockedpool.h:64
Memory statistics.
Definition: lockedpool.h:146
static size_t align_up(size_t x, size_t align)
Align up to power of 2.
Definition: lockedpool.cpp:31