facil.io
API Reference

#./fio-stl/010 mem.h

50 public symbols.

#Macros

#FIO_MEMORY_ALIGN_LOG

c
#define FIO_MEMORY_ALIGN_LOG 6

Allocation alignment, MUST be >= 3 and <= 10

Symbol type: macro

#FIO_MEMORY_ALIGN_SIZE

c
#define FIO_MEMORY_ALIGN_SIZE (1UL << (FIO_MEMORY_ALIGN_LOG))

The minimal allocation size & alignment.

Symbol type: macro

#FIO_MEMORY_SYS_ALLOCATION_SIZE_LOG

c
#define FIO_MEMORY_SYS_ALLOCATION_SIZE_LOG 21

The logarithmic size of a single allocation "chunk" (16 blocks).

Limited to >=17 and <=24.

By default 21, which is a ~2Mb allocation per system call, resulting in a maximum allocation size of 64Kb.

Symbol type: macro

#FIO_MEMORY_CACHE_SLOTS

c
#define FIO_MEMORY_CACHE_SLOTS 4

The number of system allocation "chunks" to cache even if they are not in use.

Symbol type: macro

#FIO_MEMORY_INITIALIZE_ALLOCATIONS

c
#define FIO_MEMORY_INITIALIZE_ALLOCATIONS   \
  FIO_MEMORY_INITIALIZE_ALLOCATIONS_DEFAULT

Forces the allocator to zero out memory early and often, so allocations return initialized memory (bytes are all zeros).

This will make the realloc2 safe for use (all data not copied is zero).

Symbol type: macro

#FIO_MEMORY_BLOCKS_PER_ALLOCATION_LOG

c
#define FIO_MEMORY_BLOCKS_PER_ALLOCATION_LOG 2

The number of blocks per system allocation.

More blocks protect against fragmentation, but lower the maximum number that can be allocated without reverting to mmap.

Range: 0-4 Recommended: depends on object allocation sizes, usually 1 or 2.

Symbol type: macro

#FIO_MEMORY_ENABLE_BIG_ALLOC

c
#define FIO_MEMORY_ENABLE_BIG_ALLOC 1

Uses a whole system allocation to support bigger allocations.

Could increase fragmentation costs.

Symbol type: macro

#FIO_MEMORY_ARENA_COUNT

c
#define FIO_MEMORY_ARENA_COUNT -1

Memory arenas mitigate thread contention while using more memory.

Note that at some point arenas are statistically irrelevant... except when benchmarking contention in multi-core machines.

Negative values will result in dynamic selection based on CPU core count.

Symbol type: macro

#FIO_MEMORY_ARENA_COUNT_FALLBACK

c
#define FIO_MEMORY_ARENA_COUNT_FALLBACK 24

Symbol type: macro

#FIO_MEMORY_ARENA_COUNT_MAX

c
#define FIO_MEMORY_ARENA_COUNT_MAX 64

Symbol type: macro

#FIO_MEMORY_USE_THREAD_MUTEX

c
#define FIO_MEMORY_USE_THREAD_MUTEX 1

If arena count isn't linked to the CPU count, threads might busy-spin. It is better to slow wait than fast busy spin when the work in the lock is longer... and system allocations are performed inside arena locks.

Symbol type: macro

#FIO_MEMORY_BLOCKS_PER_ALLOCATION

c
#define FIO_MEMORY_BLOCKS_PER_ALLOCATION   \
  (1UL << FIO_MEMORY_BLOCKS_PER_ALLOCATION_LOG)

the number of allocation blocks per system allocation.

Symbol type: macro

#FIO_MEMORY_SYS_ALLOCATION_SIZE

c
#define FIO_MEMORY_SYS_ALLOCATION_SIZE   \
  (1UL << FIO_MEMORY_SYS_ALLOCATION_SIZE_LOG)

the total number of bytes consumed per system allocation.

Symbol type: macro

#FIO_MEMORY_BLOCK_ALLOC_LIMIT

c
#define FIO_MEMORY_BLOCK_ALLOC_LIMIT   \
  (FIO_MEMORY_SYS_ALLOCATION_SIZE >> (FIO_MEMORY_BLOCKS_PER_ALLOCATION_LOG + 2))

The maximum allocation size, after which a big/system allocation is used.

Symbol type: macro

#FIO_MEMORY_BIG_ALLOC_LIMIT

c
#define FIO_MEMORY_BIG_ALLOC_LIMIT   \
  (FIO_MEMORY_SYS_ALLOCATION_SIZE >>   \
   (FIO_MEMORY_BLOCKS_PER_ALLOCATION_LOG > 3   \
        ? 3   \
        : FIO_MEMORY_BLOCKS_PER_ALLOCATION_LOG))

the limit of a big allocation, if enabled

Symbol type: macro

#FIO_MEMORY_ALLOC_LIMIT

c
#define FIO_MEMORY_ALLOC_LIMIT FIO_MEMORY_BIG_ALLOC_LIMIT

Symbol type: macro

#FIO_MEM_BYTES2PAGES

c
#define FIO_MEM_BYTES2PAGES(size)   \
  ((size > (SIZE_MAX - ((1UL << FIO_MEM_PAGE_SIZE_LOG) - 1)))   \
       ? size   \
       : (((size_t)(size) + ((1UL << FIO_MEM_PAGE_SIZE_LOG) - 1)) &   \
          ((~(size_t)0) << FIO_MEM_PAGE_SIZE_LOG)))

Symbol type: macro

#FIO_PAGE_ALIGN

c
#define FIO_PAGE_ALIGN   \
  __attribute__((assume_aligned((1UL << FIO_MEM_PAGE_SIZE_LOG))))

Symbol type: macro

#FIO_PAGE_ALIGN_NEW

c
#define FIO_PAGE_ALIGN_NEW   \
  __attribute__((malloc, assume_aligned((1UL << FIO_MEM_PAGE_SIZE_LOG))))

Symbol type: macro

#FIO_MEMORY_HEADER_SIZE

c
#define FIO_MEMORY_HEADER_SIZE   \
  ((sizeof(FIO_NAME(FIO_MEMORY_NAME, __mem_chunk_s)) +   \
    (FIO_MEMORY_ALIGN_SIZE - 1)) &   \
   (~(FIO_MEMORY_ALIGN_SIZE - 1)))

Symbol type: macro

#FIO_MEMORY_UNITS_PER_BLOCK

c
#define FIO_MEMORY_UNITS_PER_BLOCK   \
  (FIO_MEMORY_BLOCK_SIZE / FIO_MEMORY_ALIGN_SIZE)

Symbol type: macro

#FIO_MEMORY_STATE_SIZE

c
#define FIO_MEMORY_STATE_SIZE(arena_count)   \
  FIO_MEM_BYTES2PAGES(   \
      (sizeof(*FIO_NAME(FIO_MEMORY_NAME, __mem_state)) +   \
       (sizeof(FIO_NAME(FIO_MEMORY_NAME, __mem_arena_s)) * (arena_count))))

Symbol type: macro

#FIO_MEMORY_BIG_BLOCK_MARKER

c
#define FIO_MEMORY_BIG_BLOCK_MARKER ((~(uint32_t)0) << 2)

Symbol type: macro

#FIO_MEMORY_BIG_BLOCK_HEADER_SIZE

c
#define FIO_MEMORY_BIG_BLOCK_HEADER_SIZE   \
  (((sizeof(FIO_NAME(FIO_MEMORY_NAME, __mem_big_block_s)) +   \
     ((FIO_MEMORY_ALIGN_SIZE - 1))) &   \
    ((~(0UL)) << FIO_MEMORY_ALIGN_LOG)))

Symbol type: macro

#FIO_MEMORY_BIG_BLOCK_SIZE

c
#define FIO_MEMORY_BIG_BLOCK_SIZE   \
  (FIO_MEMORY_SYS_ALLOCATION_SIZE - FIO_MEMORY_BIG_BLOCK_HEADER_SIZE)

Symbol type: macro

#FIO_MEMORY_UNITS_PER_BIG_BLOCK

c
#define FIO_MEMORY_UNITS_PER_BIG_BLOCK   \
  (FIO_MEMORY_BIG_BLOCK_SIZE / FIO_MEMORY_ALIGN_SIZE)

Symbol type: macro

#FIO_TEST_MULTI_THREADED

c
#define FIO_TEST_MULTI_THREADED 0

Symbol type: macro

#Functions

#fio_malloc

c
void *FIO_MEM_ALIGN_NEW fio_malloc(size_t size)

Allocates memory using a per-CPU core block memory pool. Memory is zeroed out.

Allocations above FIO_MEMORY_BLOCK_ALLOC_LIMIT will be redirected to mmap, as if mempool_mmap was called.

mempool_malloc promises a best attempt at providing locality between consecutive calls, but locality can't be guaranteed.

Symbol type: function

#fio_calloc

c
void *FIO_MEM_ALIGN_NEW fio_calloc(size_t size_per_unit, size_t unit_count)

same as calling fio_malloc(size_per_unit * unit_count);

Allocations above FIO_MEMORY_BLOCK_ALLOC_LIMIT will be redirected to mmap, as if mempool_mmap was called.

Symbol type: function

#fio_free

c
void fio_free(void *ptr)

Frees memory that was allocated using this library.

Symbol type: function

#fio_realloc

c
void *FIO_MEM_ALIGN fio_realloc(void *ptr, size_t new_size)

Re-allocates memory. An attempt to avoid copying the data is made only for big memory allocations (larger than FIO_MEMORY_BLOCK_ALLOC_LIMIT).

Symbol type: function

#fio_realloc2

c
void *FIO_MEM_ALIGN fio_realloc2(void *ptr, size_t new_size, size_t copy_len)

Re-allocates memory. An attempt to avoid copying the data is made only for big memory allocations (larger than FIO_MEMORY_BLOCK_ALLOC_LIMIT).

This variation can perform better, as it might copy less data.

Symbol type: function

#fio_realloc_aligned

c
void *FIO_MEM_ALIGN fio_realloc_aligned(void *ptr, size_t new_size, size_t copy_len, size_t alignment)

Re-allocates memory, enforcing a minimum pointer alignment.

This is the core of the aligned allocation API: malloc_aligned and calloc_aligned simply route to this function with a NULL ptr.

The alignment argument is normalized as follows:

  • 0 is treated as the allocator's default (FIO_MEMORY_ALIGN_SIZE);
  • non power-of-2 values are rounded DOWN to the nearest power of 2;
  • the effective alignment is the maximum of the requested alignment, the current alignment of ptr (if any) and FIO_MEMORY_ALIGN_SIZE;
  • effective values above FIO_MEMORY_SYS_ALLOCATION_SIZE fail (returns NULL, sets errno to EINVAL).

Data preservation semantics are identical to realloc2 (copy_len bytes). In-place growth may only occur when the existing pointer already satisfies the requested alignment.

Symbol type: function

#fio_malloc_aligned

c
void *FIO_MEM_ALIGN_NEW fio_malloc_aligned(size_t size, size_t alignment)

Allocates size bytes, returning a pointer aligned to (at least) alignment. Same semantics as realloc_aligned(NULL, size, 0, alignment).

Symbol type: function

#fio_calloc_aligned

c
void *FIO_MEM_ALIGN_NEW fio_calloc_aligned(size_t size_per_unit, size_t unit_count, size_t alignment)

Same as malloc_aligned(size_per_unit * unit_count, alignment), except that the allocated memory is zeroed out.

Symbol type: function

#fio_mmap

c
void *FIO_MEM_ALIGN_NEW fio_mmap(size_t size)

Allocates memory directly using mmap, this is preferred for objects that both require almost a page of memory (or more) and expect a long lifetime.

However, since this allocation will invoke the system call (mmap), it will be inherently slower.

mempoll_free can be used for deallocating the memory.

Symbol type: function

#fio_malloc_after_fork

c
void fio_malloc_after_fork(void)

When forking is called manually, call this function to reset the facil.io memory allocator's locks.

This is provided in case of forking in a multi-threaded environment, which some people do even though it's bad.

Symbol type: function

#fio_malloc_arenas

c
size_t fio_malloc_arenas(void)

Arena count for the allocator.

Symbol type: function

#fio_malloc_block_size

c
size_t fio_malloc_block_size(void)

Symbol type: function

#fio_malloc_print_state

c
void fio_malloc_print_state(void)

Prints the allocator's data structure. May be used for debugging.

Symbol type: function

#fio_malloc_print_free_block_list

c
void fio_malloc_print_free_block_list(void)

Prints the allocator's free block list. May be used for debugging.

Symbol type: function

#fio_malloc_print_settings

c
void fio_malloc_print_settings(void)

Prints the settings used to define the allocator.

Symbol type: function

#fio_malloc_sys_alloc_size

c
inline size_t fio_malloc_sys_alloc_size(void)

System allocation sizes (bytes per system allocation).

Symbol type: function

#fio_malloc_cache_slots

c
inline size_t fio_malloc_cache_slots(void)

Cached system allocations (free, but held on to).

Symbol type: function

#fio_malloc_alignment

c
inline size_t fio_malloc_alignment(void)

Allocations alignment.

Symbol type: function

#fio_malloc_alignment_log

c
inline size_t fio_malloc_alignment_log(void)

Allocations alignment log (base 2).

Symbol type: function

#fio_malloc_alloc_limit

c
inline size_t fio_malloc_alloc_limit(void)

Allocation limit (at which point do we switch to system allocations?).

Symbol type: function

#fio_malloc_arena_alloc_limit

c
inline size_t fio_malloc_arena_alloc_limit(void)

Allocation limit for arena based allocation (vs. big allocations).

Symbol type: function

#fio_realloc_is_safe

c
inline size_t fio_realloc_is_safe(void)

Symbol type: function

#fio_alloc_size

c
inline size_t fio_alloc_size(size_t minimum_bytes)

Returns the number of usable bytes the allocator will actually reserve for an allocation request of minimum_bytes (the allocation's size class).

The result is alignment agnostic - alignment padding, if any, is reserved in addition to the returned value.

NOTE: when the custom allocator is bypassed (FIO_MEMORY_DISABLE), the system allocator exposes no rounding guarantee and minimum_bytes is returned unchanged.

Symbol type: function