emq-sys 1.0.0-beta.2

Raw FFI bindings to EmbeddedMQ (bundles C sources by default)
Documentation
#include "core/emq_pool.h"
#include "core/emq_atomic.h"
#include "platform/emq_platform.h"

#include <stdlib.h>
#include <string.h>

#define EMQ_POOL_NUM_CLASSES 11u
#define EMQ_POOL_SLAB_BYTES (2u * 1024u * 1024u)
#define EMQ_POOL_MAX_CLASS 65536u
#define EMQ_POOL_HDR_SIZE 8u
#define EMQ_POOL_MAG_DEPTH 32u

static const uint32_t emq_pool_class_sizes[EMQ_POOL_NUM_CLASSES] = {
    64u,   128u,  256u,   512u,   1024u, 2048u,
    4096u, 8192u, 16384u, 32768u, 65536u};

typedef struct emq_pool_hdr {
  uint32_t class_idx;
  uint32_t block_bytes;
} emq_pool_hdr;

typedef struct emq_pool_block {
  struct emq_pool_block *next;
} emq_pool_block;

typedef struct emq_pool_depot {
  emq_pool_block *freelist;
  emq_mutex *mu;
} emq_pool_depot;

typedef struct emq_pool_slab {
  struct emq_pool_slab *next;
  void *base;
  size_t size;
} emq_pool_slab;

typedef struct emq_pool_magazine {
  emq_pool_block *freelist[EMQ_POOL_NUM_CLASSES];
  uint32_t count[EMQ_POOL_NUM_CLASSES];
  int initialized;
} emq_pool_magazine;

struct emq_pool {
  emq_pool_depot depots[EMQ_POOL_NUM_CLASSES];
  emq_pool_slab *slabs;
  emq_mutex *slab_mu;
  emq_atomic_u64 allocs;
  emq_atomic_u64 frees;
  emq_atomic_u64 hits;
  emq_atomic_u64 misses;
  emq_atomic_u64 live_bytes;
  emq_atomic_u64 mmap_bytes;
  emq_atomic_u64 malloc_fallbacks;
};

#if defined(_MSC_VER)
static __declspec(thread) emq_pool_magazine emq_pool_tls_mag;
#else
static __thread emq_pool_magazine emq_pool_tls_mag;
#endif

static void emq_pool_mag_init(void) {
  if (!emq_pool_tls_mag.initialized) {
    memset(&emq_pool_tls_mag, 0, sizeof(emq_pool_tls_mag));
    emq_pool_tls_mag.initialized = 1;
  }
}

static uint32_t emq_pool_class_for_size(size_t size) {
  uint32_t i;
  if (size <= emq_pool_class_sizes[0]) return 0;
  for (i = 1; i < EMQ_POOL_NUM_CLASSES; ++i) {
    if (size <= emq_pool_class_sizes[i]) return i;
  }
  return EMQ_POOL_NUM_CLASSES;
}

static uint32_t emq_pool_block_bytes(uint32_t class_idx) {
  return EMQ_POOL_HDR_SIZE + emq_pool_class_sizes[class_idx];
}

static void *emq_pool_user_ptr(void *block) {
  return (uint8_t *)block + EMQ_POOL_HDR_SIZE;
}

static emq_pool_hdr *emq_pool_hdr_from_user(void *ptr) {
  return (emq_pool_hdr *)((uint8_t *)ptr - EMQ_POOL_HDR_SIZE);
}

static void emq_pool_stat_add(emq_atomic_u64 *counter, uint64_t delta) {
  (void)emq_atomic_fetch_add_u64(counter, delta);
}

static void emq_pool_depot_push(emq_pool *pool, uint32_t class_idx,
                                emq_pool_block *blk);

static int emq_pool_carve_slab(emq_pool *pool, uint32_t class_idx) {
  void *base;
  emq_pool_slab *slab;
  uint32_t block_bytes;
  uint32_t count;
  uint32_t i;
  uint8_t *cursor;

  block_bytes = emq_pool_block_bytes(class_idx);
  count = (uint32_t)(EMQ_POOL_SLAB_BYTES / block_bytes);
  if (count == 0) return -1;

  base = emq_os_alloc_pages(EMQ_POOL_SLAB_BYTES);
  if (!base) return -2;

  emq_mutex_lock(pool->slab_mu);
  slab = (emq_pool_slab *)malloc(sizeof(*slab));
  if (!slab) {
    emq_os_free_pages(base, EMQ_POOL_SLAB_BYTES);
    emq_mutex_unlock(pool->slab_mu);
    return -2;
  }
  slab->base = base;
  slab->size = EMQ_POOL_SLAB_BYTES;
  slab->next = pool->slabs;
  pool->slabs = slab;
  emq_mutex_unlock(pool->slab_mu);

  emq_pool_stat_add(&pool->mmap_bytes, EMQ_POOL_SLAB_BYTES);
  cursor = (uint8_t *)base;
  for (i = 0; i < count; ++i) {
    emq_pool_block *blk = (emq_pool_block *)cursor;
    emq_pool_hdr *hdr = (emq_pool_hdr *)cursor;
    hdr->class_idx = class_idx;
    hdr->block_bytes = block_bytes;
    emq_pool_depot_push(pool, class_idx, blk);
    cursor += block_bytes;
  }
  return 0;
}

static emq_pool_block *emq_pool_depot_pop(emq_pool *pool, uint32_t class_idx) {
  emq_pool_depot *depot = &pool->depots[class_idx];
  emq_pool_block *blk;
  emq_mutex_lock(depot->mu);
  blk = depot->freelist;
  if (blk) depot->freelist = blk->next;
  emq_mutex_unlock(depot->mu);
  return blk;
}

static void emq_pool_depot_push(emq_pool *pool, uint32_t class_idx,
                                emq_pool_block *blk) {
  emq_pool_depot *depot = &pool->depots[class_idx];
  emq_mutex_lock(depot->mu);
  blk->next = depot->freelist;
  depot->freelist = blk;
  emq_mutex_unlock(depot->mu);
}

static void emq_pool_mag_flush(emq_pool *pool, uint32_t class_idx) {
  emq_pool_magazine *mag = &emq_pool_tls_mag;
  while (mag->count[class_idx] > EMQ_POOL_MAG_DEPTH / 2u) {
    emq_pool_block *blk = mag->freelist[class_idx];
    if (!blk) break;
    mag->freelist[class_idx] = blk->next;
    mag->count[class_idx]--;
    emq_pool_depot_push(pool, class_idx, blk);
  }
}

static void emq_pool_restore_hdr(emq_pool_block *blk, uint32_t class_idx) {
  emq_pool_hdr *hdr = (emq_pool_hdr *)blk;
  hdr->class_idx = class_idx;
  hdr->block_bytes = emq_pool_block_bytes(class_idx);
}

static emq_pool_block *emq_pool_mag_pop(emq_pool *pool, uint32_t class_idx) {
  emq_pool_magazine *mag = &emq_pool_tls_mag;
  emq_pool_block *blk = mag->freelist[class_idx];
  if (blk) {
    mag->freelist[class_idx] = blk->next;
    mag->count[class_idx]--;
    emq_pool_restore_hdr(blk, class_idx);
    emq_pool_stat_add(&pool->hits, 1);
    return blk;
  }
  emq_pool_stat_add(&pool->misses, 1);
  while (mag->count[class_idx] < EMQ_POOL_MAG_DEPTH) {
    emq_pool_block *refill = emq_pool_depot_pop(pool, class_idx);
    if (!refill) break;
    refill->next = mag->freelist[class_idx];
    mag->freelist[class_idx] = refill;
    mag->count[class_idx]++;
  }
  blk = mag->freelist[class_idx];
  if (blk) {
    mag->freelist[class_idx] = blk->next;
    mag->count[class_idx]--;
    emq_pool_restore_hdr(blk, class_idx);
    emq_pool_stat_add(&pool->hits, 1);
    return blk;
  }
  if (emq_pool_carve_slab(pool, class_idx) != 0) return NULL;
  return emq_pool_mag_pop(pool, class_idx);
}

emq_pool *emq_pool_create(void) {
  emq_pool *pool;
  uint32_t i;
  pool = (emq_pool *)calloc(1, sizeof(*pool));
  if (!pool) return NULL;
  pool->slab_mu = emq_mutex_create();
  if (!pool->slab_mu) {
    free(pool);
    return NULL;
  }
  for (i = 0; i < EMQ_POOL_NUM_CLASSES; ++i) {
    pool->depots[i].mu = emq_mutex_create();
    if (!pool->depots[i].mu) {
      emq_pool_destroy(pool);
      return NULL;
    }
  }
  emq_atomic_init_u64(&pool->allocs, 0);
  emq_atomic_init_u64(&pool->frees, 0);
  emq_atomic_init_u64(&pool->hits, 0);
  emq_atomic_init_u64(&pool->misses, 0);
  emq_atomic_init_u64(&pool->live_bytes, 0);
  emq_atomic_init_u64(&pool->mmap_bytes, 0);
  emq_atomic_init_u64(&pool->malloc_fallbacks, 0);
  return pool;
}

void emq_pool_destroy(emq_pool *pool) {
  emq_pool_slab *slab;
  uint32_t i;
  if (!pool) return;
  for (slab = pool->slabs; slab;) {
    emq_pool_slab *next = slab->next;
    emq_os_free_pages(slab->base, slab->size);
    free(slab);
    slab = next;
  }
  for (i = 0; i < EMQ_POOL_NUM_CLASSES; ++i) {
    emq_mutex_destroy(pool->depots[i].mu);
  }
  emq_mutex_destroy(pool->slab_mu);
  free(pool);
}

void *emq_pool_alloc(emq_pool *pool, size_t size) {
  uint32_t class_idx;
  emq_pool_block *blk;
  if (!pool || size == 0) return NULL;
  emq_pool_mag_init();
  class_idx = emq_pool_class_for_size(size);
  if (class_idx >= EMQ_POOL_NUM_CLASSES) {
    emq_pool_hdr *hdr;
    void *p = malloc(EMQ_POOL_HDR_SIZE + size);
    if (!p) return NULL;
    hdr = (emq_pool_hdr *)p;
    hdr->class_idx = UINT32_MAX;
    hdr->block_bytes = (uint32_t)(EMQ_POOL_HDR_SIZE + size);
    emq_pool_stat_add(&pool->malloc_fallbacks, 1);
    emq_pool_stat_add(&pool->allocs, 1);
    emq_pool_stat_add(&pool->live_bytes, size);
    return emq_pool_user_ptr(p);
  }
  blk = emq_pool_mag_pop(pool, class_idx);
  if (!blk) return NULL;
  emq_pool_stat_add(&pool->allocs, 1);
  emq_pool_stat_add(&pool->live_bytes, emq_pool_class_sizes[class_idx]);
  return emq_pool_user_ptr(blk);
}

void emq_pool_free(emq_pool *pool, void *ptr, size_t size) {
  emq_pool_hdr *hdr;
  uint32_t class_idx;
  emq_pool_block *blk;
  if (!pool || !ptr) return;
  (void)size;
  emq_pool_mag_init();
  hdr = emq_pool_hdr_from_user(ptr);
  class_idx = hdr->class_idx;
  if (class_idx >= EMQ_POOL_NUM_CLASSES) {
    uint64_t user_bytes = hdr->block_bytes > EMQ_POOL_HDR_SIZE
                              ? (uint64_t)(hdr->block_bytes - EMQ_POOL_HDR_SIZE)
                              : 0;
    emq_pool_stat_add(&pool->live_bytes, 0u - user_bytes);
    emq_pool_stat_add(&pool->frees, 1);
    free(hdr);
    return;
  }
  blk = (emq_pool_block *)hdr;
  emq_pool_stat_add(&pool->live_bytes,
                    0u - (uint64_t)emq_pool_class_sizes[class_idx]);
  emq_pool_stat_add(&pool->frees, 1);
  {
    emq_pool_magazine *mag = &emq_pool_tls_mag;
    if (mag->count[class_idx] < EMQ_POOL_MAG_DEPTH) {
      blk->next = mag->freelist[class_idx];
      mag->freelist[class_idx] = blk;
      mag->count[class_idx]++;
      emq_pool_stat_add(&pool->hits, 1);
      return;
    }
  }
  emq_pool_mag_flush(pool, class_idx);
  emq_pool_depot_push(pool, class_idx, blk);
}

void emq_pool_get_stats(const emq_pool *pool, emq_pool_stats *out) {
  if (!pool || !out) return;
  memset(out, 0, sizeof(*out));
  out->allocs = emq_atomic_load_u64(&pool->allocs);
  out->frees = emq_atomic_load_u64(&pool->frees);
  out->hits = emq_atomic_load_u64(&pool->hits);
  out->misses = emq_atomic_load_u64(&pool->misses);
  out->live_bytes = emq_atomic_load_u64(&pool->live_bytes);
  out->mmap_bytes = emq_atomic_load_u64(&pool->mmap_bytes);
  out->malloc_fallbacks = emq_atomic_load_u64(&pool->malloc_fallbacks);
}