12#include <sys/resource.h>
31static inline size_t align_up(
size_t x,
size_t align)
33 return (x + align - 1) & ~(align - 1);
40 base(base_in), end(static_cast<char*>(base_in) + size_in), alignment(alignment_in)
68 const size_t size_remaining = size_ptr_it->first - size;
69 char*
const free_chunk =
static_cast<char*
>(size_ptr_it->second);
70 auto allocated =
chunks_used.emplace(free_chunk + size_remaining, size).first;
72 if (size_ptr_it->first == size) {
83 return allocated->first;
96 throw std::runtime_error(
"Arena: invalid or double free");
98 auto freed = std::make_pair(
static_cast<char*
>(i->first), i->second);
104 freed.first -= prev->second->first;
105 freed.second += prev->second->first;
111 auto next =
chunks_free.find(freed.first + freed.second);
113 freed.second += next->second->first;
128 r.used += chunk.second;
130 r.free += chunk.second->first;
131 r.total = r.used + r.free;
136static void printchunk(
void* base,
size_t sz,
bool used) {
138 "0x" << std::hex << std::setw(16) << std::setfill(
'0') << base <<
139 " 0x" << std::hex << std::setw(16) << std::setfill(
'0') << sz <<
140 " 0x" << used << std::endl;
142void Arena::walk()
const
145 printchunk(chunk.first, chunk.second,
true);
146 std::cout << std::endl;
148 printchunk(chunk.first, chunk.second->first,
false);
149 std::cout << std::endl;
162 Win32LockedPageAllocator();
164 void FreeLocked(
void* addr,
size_t len)
override;
170Win32LockedPageAllocator::Win32LockedPageAllocator()
173 SYSTEM_INFO sSysInfo;
174 GetSystemInfo(&sSysInfo);
175 page_size = sSysInfo.dwPageSize;
177void *Win32LockedPageAllocator::AllocateLocked(
size_t len,
bool *lockingSuccess)
180 void *addr = VirtualAlloc(
nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
186 *lockingSuccess = VirtualLock(
const_cast<void*
>(addr), len) != 0;
190void Win32LockedPageAllocator::FreeLocked(
void* addr,
size_t len)
194 VirtualUnlock(
const_cast<void*
>(addr), len);
197size_t Win32LockedPageAllocator::GetLimit()
200 if(GetProcessWorkingSetSize(GetCurrentProcess(), &min, &max) != 0) {
203 return std::numeric_limits<size_t>::max();
219 void FreeLocked(
void* addr,
size_t len)
override;
239 addr = mmap(
nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
240 if (addr == MAP_FAILED) {
244 *lockingSuccess = mlock(addr, len) == 0;
245#if defined(MADV_DONTDUMP)
246 madvise(addr, len, MADV_DONTDUMP);
247#elif defined(MADV_NOCORE)
248 madvise(addr, len, MADV_NOCORE);
264 if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) {
265 if (rlim.rlim_cur != RLIM_INFINITY) {
266 return rlim.rlim_cur;
270 return std::numeric_limits<size_t>::max();
278 : allocator(
std::move(allocator_in)), lf_cb(lf_cb_in)
286 std::lock_guard<std::mutex> lock(
mutex);
293 for (
auto &arena:
arenas) {
294 void *addr = arena.alloc(size);
301 return arenas.back().alloc(size);
308 std::lock_guard<std::mutex> lock(
mutex);
311 for (
auto &arena:
arenas) {
312 if (arena.addressInArena(ptr)) {
317 throw std::runtime_error(
"LockedPool: invalid address not pointing to any arena");
322 std::lock_guard<std::mutex> lock(
mutex);
324 for (
const auto &arena:
arenas) {
345 size = std::min(size, limit);
348 void *addr =
allocator->AllocateLocked(size, &locked);
365 Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in)
395 std::unique_ptr<LockedPageAllocator>
allocator(
new Win32LockedPageAllocator());
void * base
Base address of arena.
size_t alignment
Minimum chunk alignment.
ChunkToSizeMap chunks_free_end
Map from end of free chunk to its node in size_to_free_chunk.
std::unordered_map< void *, size_t > chunks_used
Map from begin of used chunk to its size.
void * alloc(size_t size)
Allocate size bytes from this arena.
SizeToChunkSortedMap size_to_free_chunk
Map to enable O(log(n)) best-fit allocation, as it's sorted by size.
Arena(void *base, size_t size, size_t alignment)
Stats stats() const
Get arena usage statistics.
ChunkToSizeMap chunks_free
Map from begin of free chunk to its node in size_to_free_chunk.
void free(void *ptr)
Free a previously allocated chunk of memory.
OS-dependent allocation and deallocation of locked/pinned memory pages.
virtual void * AllocateLocked(size_t len, bool *lockingSuccess)=0
Allocate and lock memory pages.
virtual void FreeLocked(void *addr, size_t len)=0
Unlock and free memory pages.
virtual size_t GetLimit()=0
Get the total limit on the amount of memory that may be locked by this process, in bytes.
LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align)
Pool for locked memory chunks.
void free(void *ptr)
Free a previously allocated chunk of memory.
Stats stats() const
Get pool usage statistics.
std::unique_ptr< LockedPageAllocator > allocator
void * alloc(size_t size)
Allocate size bytes from this arena.
size_t cumulative_bytes_locked
LockedPool(std::unique_ptr< LockedPageAllocator > allocator, LockingFailed_Callback lf_cb_in=nullptr)
Create a new LockedPool.
LockingFailed_Callback lf_cb
std::list< LockedPageArena > arenas
bool new_arena(size_t size, size_t align)
static const size_t ARENA_ALIGN
Chunk alignment.
static const size_t ARENA_SIZE
Size of one arena of locked memory.
std::mutex mutex
Mutex protects access to this pool's data structures, including arenas.
Singleton class to keep track of locked (ie, non-swappable) memory, for use in std::allocator templat...
LockedPoolManager(std::unique_ptr< LockedPageAllocator > allocator)
static bool LockingFailed()
Called when locking fails, warn the user here.
static LockedPoolManager * _instance
static void CreateInstance()
Create a new LockedPoolManager specialized to the OS.
LockedPageAllocator specialized for OSes that don't try to be special snowflakes.
void * AllocateLocked(size_t len, bool *lockingSuccess) override
Allocate and lock memory pages.
void FreeLocked(void *addr, size_t len) override
Unlock and free memory pages.
size_t GetLimit() override
Get the total limit on the amount of memory that may be locked by this process, in bytes.
PosixLockedPageAllocator()
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
static size_t align_up(size_t x, size_t align)
Align up to power of 2.