"Fossies" - the Fresh Open Source Software Archive

Member "glusterfs-8.2/libglusterfs/src/mem-pool.c" (16 Sep 2020, 25654 Bytes) of package /linux/misc/glusterfs-8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "mem-pool.c" see the Fossies "Dox" file reference documentation.

    1 /*
    2   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    3   This file is part of GlusterFS.
    4 
    5   This file is licensed to you under your choice of the GNU Lesser
    6   General Public License, version 3 or any later version (LGPLv3 or
    7   later), or the GNU General Public License, version 2 (GPLv2), in all
    8   cases as published by the Free Software Foundation.
    9 */
   10 
   11 #include "glusterfs/mem-pool.h"
   12 #include "glusterfs/common-utils.h"  // for GF_ASSERT, gf_thread_cr...
   13 #include "glusterfs/globals.h"       // for xlator_t, THIS
   14 #include <stdlib.h>
   15 #include <stdarg.h>
   16 
   17 #include "unittest/unittest.h"
   18 #include "glusterfs/libglusterfs-messages.h"
   19 
   20 void
   21 gf_mem_acct_enable_set(void *data)
   22 {
   23     glusterfs_ctx_t *ctx = NULL;
   24 
   25     REQUIRE(data != NULL);
   26 
   27     ctx = data;
   28 
   29     GF_ASSERT(ctx != NULL);
   30 
   31     ctx->mem_acct_enable = 1;
   32 
   33     ENSURE(1 == ctx->mem_acct_enable);
   34 
   35     return;
   36 }
   37 
   38 static void *
   39 gf_mem_header_prepare(struct mem_header *header, size_t size)
   40 {
   41     void *ptr;
   42 
   43     header->size = size;
   44 
   45     ptr = header + 1;
   46 
   47     /* data follows in this gap of 'size' bytes */
   48     *(uint32_t *)(ptr + size) = GF_MEM_TRAILER_MAGIC;
   49 
   50     return ptr;
   51 }
   52 
   53 static void *
   54 gf_mem_set_acct_info(struct mem_acct *mem_acct, struct mem_header *header,
   55                      size_t size, uint32_t type, const char *typestr)
   56 {
   57     struct mem_acct_rec *rec = NULL;
   58     bool new_ref = false;
   59 
   60     if (mem_acct != NULL) {
   61         GF_ASSERT(type <= mem_acct->num_types);
   62 
   63         rec = &mem_acct->rec[type];
   64         LOCK(&rec->lock);
   65         {
   66             if (!rec->typestr) {
   67                 rec->typestr = typestr;
   68             }
   69             rec->size += size;
   70             new_ref = (rec->num_allocs == 0);
   71             rec->num_allocs++;
   72             rec->total_allocs++;
   73             rec->max_size = max(rec->max_size, rec->size);
   74             rec->max_num_allocs = max(rec->max_num_allocs, rec->num_allocs);
   75 
   76 #ifdef DEBUG
   77             list_add(&header->acct_list, &rec->obj_list);
   78 #endif
   79         }
   80         UNLOCK(&rec->lock);
   81 
   82         /* We only take a reference for each memory type used, not for each
   83          * allocation. This minimizes the use of atomic operations. */
   84         if (new_ref) {
   85             GF_ATOMIC_INC(mem_acct->refcnt);
   86         }
   87     }
   88 
   89     header->type = type;
   90     header->mem_acct = mem_acct;
   91     header->magic = GF_MEM_HEADER_MAGIC;
   92 
   93     return gf_mem_header_prepare(header, size);
   94 }
   95 
   96 static void *
   97 gf_mem_update_acct_info(struct mem_acct *mem_acct, struct mem_header *header,
   98                         size_t size)
   99 {
  100     struct mem_acct_rec *rec = NULL;
  101 
  102     if (mem_acct != NULL) {
  103         rec = &mem_acct->rec[header->type];
  104         LOCK(&rec->lock);
  105         {
  106             rec->size += size - header->size;
  107             rec->total_allocs++;
  108             rec->max_size = max(rec->max_size, rec->size);
  109 
  110 #ifdef DEBUG
  111             /* The old 'header' already was present in 'obj_list', but
  112              * realloc() could have changed its address. We need to remove
  113              * the old item from the list and add the new one. This can be
  114              * done this way because list_move() doesn't use the pointers
  115              * to the old location (which are not valid anymore) already
  116              * present in the list, it simply overwrites them. */
  117             list_move(&header->acct_list, &rec->obj_list);
  118 #endif
  119         }
  120         UNLOCK(&rec->lock);
  121     }
  122 
  123     return gf_mem_header_prepare(header, size);
  124 }
  125 
  126 static bool
  127 gf_mem_acct_enabled(void)
  128 {
  129     xlator_t *x = THIS;
  130     /* Low-level __gf_xxx() may be called
  131        before ctx is initialized. */
  132     return x->ctx && x->ctx->mem_acct_enable;
  133 }
  134 
  135 void *
  136 __gf_calloc(size_t nmemb, size_t size, uint32_t type, const char *typestr)
  137 {
  138     size_t tot_size = 0;
  139     size_t req_size = 0;
  140     void *ptr = NULL;
  141     xlator_t *xl = NULL;
  142 
  143     if (!gf_mem_acct_enabled())
  144         return CALLOC(nmemb, size);
  145 
  146     xl = THIS;
  147 
  148     req_size = nmemb * size;
  149     tot_size = req_size + GF_MEM_HEADER_SIZE + GF_MEM_TRAILER_SIZE;
  150 
  151     ptr = calloc(1, tot_size);
  152 
  153     if (!ptr) {
  154         gf_msg_nomem("", GF_LOG_ALERT, tot_size);
  155         return NULL;
  156     }
  157 
  158     return gf_mem_set_acct_info(xl->mem_acct, ptr, req_size, type, typestr);
  159 }
  160 
  161 void *
  162 __gf_malloc(size_t size, uint32_t type, const char *typestr)
  163 {
  164     size_t tot_size = 0;
  165     void *ptr = NULL;
  166     xlator_t *xl = NULL;
  167 
  168     if (!gf_mem_acct_enabled())
  169         return MALLOC(size);
  170 
  171     xl = THIS;
  172 
  173     tot_size = size + GF_MEM_HEADER_SIZE + GF_MEM_TRAILER_SIZE;
  174 
  175     ptr = malloc(tot_size);
  176     if (!ptr) {
  177         gf_msg_nomem("", GF_LOG_ALERT, tot_size);
  178         return NULL;
  179     }
  180 
  181     return gf_mem_set_acct_info(xl->mem_acct, ptr, size, type, typestr);
  182 }
  183 
  184 void *
  185 __gf_realloc(void *ptr, size_t size)
  186 {
  187     size_t tot_size = 0;
  188     struct mem_header *header = NULL;
  189 
  190     if (!gf_mem_acct_enabled())
  191         return REALLOC(ptr, size);
  192 
  193     REQUIRE(NULL != ptr);
  194 
  195     header = (struct mem_header *)(ptr - GF_MEM_HEADER_SIZE);
  196     GF_ASSERT(header->magic == GF_MEM_HEADER_MAGIC);
  197 
  198     tot_size = size + GF_MEM_HEADER_SIZE + GF_MEM_TRAILER_SIZE;
  199     header = realloc(header, tot_size);
  200     if (!header) {
  201         gf_msg_nomem("", GF_LOG_ALERT, tot_size);
  202         return NULL;
  203     }
  204 
  205     return gf_mem_update_acct_info(header->mem_acct, header, size);
  206 }
  207 
  208 int
  209 gf_vasprintf(char **string_ptr, const char *format, va_list arg)
  210 {
  211     va_list arg_save;
  212     char *str = NULL;
  213     int size = 0;
  214     int rv = 0;
  215 
  216     if (!string_ptr || !format)
  217         return -1;
  218 
  219     va_copy(arg_save, arg);
  220 
  221     size = vsnprintf(NULL, 0, format, arg);
  222     size++;
  223     str = GF_MALLOC(size, gf_common_mt_asprintf);
  224     if (str == NULL) {
  225         /* log is done in GF_MALLOC itself */
  226         va_end(arg_save);
  227         return -1;
  228     }
  229     rv = vsnprintf(str, size, format, arg_save);
  230 
  231     *string_ptr = str;
  232     va_end(arg_save);
  233     return (rv);
  234 }
  235 
  236 int
  237 gf_asprintf(char **string_ptr, const char *format, ...)
  238 {
  239     va_list arg;
  240     int rv = 0;
  241 
  242     va_start(arg, format);
  243     rv = gf_vasprintf(string_ptr, format, arg);
  244     va_end(arg);
  245 
  246     return rv;
  247 }
  248 
  249 #ifdef DEBUG
  250 void
  251 __gf_mem_invalidate(void *ptr)
  252 {
  253     struct mem_header *header = ptr;
  254     void *end = NULL;
  255 
  256     struct mem_invalid inval = {
  257         .magic = GF_MEM_INVALID_MAGIC,
  258         .mem_acct = header->mem_acct,
  259         .type = header->type,
  260         .size = header->size,
  261         .baseaddr = ptr + GF_MEM_HEADER_SIZE,
  262     };
  263 
  264     /* calculate the last byte of the allocated area */
  265     end = ptr + GF_MEM_HEADER_SIZE + inval.size + GF_MEM_TRAILER_SIZE;
  266 
  267     /* overwrite the old mem_header */
  268     memcpy(ptr, &inval, sizeof(inval));
  269     ptr += sizeof(inval);
  270 
  271     /* zero out remaining (old) mem_header bytes) */
  272     memset(ptr, 0x00, sizeof(*header) - sizeof(inval));
  273     ptr += sizeof(*header) - sizeof(inval);
  274 
  275     /* zero out the first byte of data */
  276     *(uint32_t *)(ptr) = 0x00;
  277     ptr += 1;
  278 
  279     /* repeated writes of invalid structurein data area */
  280     while ((ptr + (sizeof(inval))) < (end - 1)) {
  281         memcpy(ptr, &inval, sizeof(inval));
  282         ptr += sizeof(inval);
  283     }
  284 
  285     /* fill out remaining data area with 0xff */
  286     memset(ptr, 0xff, end - ptr);
  287 }
  288 #endif /* DEBUG */
  289 
  290 /* Coverity taint NOTE: pointers passed to free, would operate on
  291 pointer-GF_MEM_HEADER_SIZE content and if the pointer was used for any IO
  292 related purpose, the pointer stands tainted, and hence coverity would consider
  293 access to the said region as tainted. The following directive to coverity hence
  294 sanitizes the pointer, thus removing any taint to the same within this function.
  295 If the pointer is accessed outside the scope of this function without any
  296 checks on content read from an IO operation, taints will still be reported, and
  297 needs appropriate addressing. */
  298 
  299 /* coverity[ +tainted_data_sanitize : arg-0 ] */
  300 static void
  301 gf_free_sanitize(void *s)
  302 {
  303 }
  304 
  305 void
  306 __gf_free(void *free_ptr)
  307 {
  308     void *ptr = NULL;
  309     struct mem_acct *mem_acct;
  310     struct mem_header *header = NULL;
  311     bool last_ref = false;
  312 
  313     if (!gf_mem_acct_enabled()) {
  314         FREE(free_ptr);
  315         return;
  316     }
  317 
  318     if (!free_ptr)
  319         return;
  320 
  321     gf_free_sanitize(free_ptr);
  322     ptr = free_ptr - GF_MEM_HEADER_SIZE;
  323     header = (struct mem_header *)ptr;
  324 
  325     // Possible corruption, assert here
  326     GF_ASSERT(GF_MEM_HEADER_MAGIC == header->magic);
  327 
  328     mem_acct = header->mem_acct;
  329     if (!mem_acct) {
  330         goto free;
  331     }
  332 
  333     // This points to a memory overrun
  334     GF_ASSERT(GF_MEM_TRAILER_MAGIC ==
  335               *(uint32_t *)((char *)free_ptr + header->size));
  336 
  337     LOCK(&mem_acct->rec[header->type].lock);
  338     {
  339         mem_acct->rec[header->type].size -= header->size;
  340         mem_acct->rec[header->type].num_allocs--;
  341         /* If all the instances are freed up then ensure typestr is set
  342          * to NULL */
  343         if (!mem_acct->rec[header->type].num_allocs) {
  344             last_ref = true;
  345             mem_acct->rec[header->type].typestr = NULL;
  346         }
  347 #ifdef DEBUG
  348         list_del(&header->acct_list);
  349 #endif
  350     }
  351     UNLOCK(&mem_acct->rec[header->type].lock);
  352 
  353     if (last_ref) {
  354         xlator_mem_acct_unref(mem_acct);
  355     }
  356 
  357 free:
  358 #ifdef DEBUG
  359     __gf_mem_invalidate(ptr);
  360 #endif
  361 
  362     FREE(ptr);
  363 }
  364 
  365 static pthread_mutex_t pool_lock = PTHREAD_MUTEX_INITIALIZER;
  366 static struct list_head pool_threads;
  367 static pthread_mutex_t pool_free_lock = PTHREAD_MUTEX_INITIALIZER;
  368 static struct list_head pool_free_threads;
  369 static struct mem_pool_shared pools[NPOOLS];
  370 static size_t pool_list_size;
  371 
  372 static __thread per_thread_pool_list_t *thread_pool_list = NULL;
  373 
  374 #if !defined(GF_DISABLE_MEMPOOL)
  375 #define N_COLD_LISTS 1024
  376 #define POOL_SWEEP_SECS 30
  377 
  378 typedef struct {
  379     pooled_obj_hdr_t *cold_lists[N_COLD_LISTS];
  380     unsigned int n_cold_lists;
  381 } sweep_state_t;
  382 
  383 enum init_state {
  384     GF_MEMPOOL_INIT_NONE = 0,
  385     GF_MEMPOOL_INIT_EARLY,
  386     GF_MEMPOOL_INIT_LATE,
  387     GF_MEMPOOL_INIT_DESTROY
  388 };
  389 
  390 static enum init_state init_done = GF_MEMPOOL_INIT_NONE;
  391 static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
  392 static unsigned int init_count = 0;
  393 static pthread_t sweeper_tid;
  394 
  395 static bool
  396 collect_garbage(sweep_state_t *state, per_thread_pool_list_t *pool_list)
  397 {
  398     unsigned int i;
  399     per_thread_pool_t *pt_pool;
  400 
  401     (void)pthread_spin_lock(&pool_list->lock);
  402 
  403     for (i = 0; i < NPOOLS; ++i) {
  404         pt_pool = &pool_list->pools[i];
  405         if (pt_pool->cold_list) {
  406             if (state->n_cold_lists >= N_COLD_LISTS) {
  407                 (void)pthread_spin_unlock(&pool_list->lock);
  408                 return true;
  409             }
  410             state->cold_lists[state->n_cold_lists++] = pt_pool->cold_list;
  411         }
  412         pt_pool->cold_list = pt_pool->hot_list;
  413         pt_pool->hot_list = NULL;
  414     }
  415 
  416     (void)pthread_spin_unlock(&pool_list->lock);
  417 
  418     return false;
  419 }
  420 
  421 static void
  422 free_obj_list(pooled_obj_hdr_t *victim)
  423 {
  424     pooled_obj_hdr_t *next;
  425 
  426     while (victim) {
  427         next = victim->next;
  428         free(victim);
  429         victim = next;
  430     }
  431 }
  432 
  433 static void *
  434 pool_sweeper(void *arg)
  435 {
  436     sweep_state_t state;
  437     per_thread_pool_list_t *pool_list;
  438     uint32_t i;
  439     bool pending;
  440 
  441     /*
  442      * This is all a bit inelegant, but the point is to avoid doing
  443      * expensive things (like freeing thousands of objects) while holding a
  444      * global lock.  Thus, we split each iteration into two passes, with
  445      * only the first and fastest holding the lock.
  446      */
  447 
  448     pending = true;
  449 
  450     for (;;) {
  451         /* If we know there's pending work to do (or it's the first run), we
  452          * do collect garbage more often. */
  453         sleep(pending ? POOL_SWEEP_SECS / 5 : POOL_SWEEP_SECS);
  454 
  455         (void)pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
  456         state.n_cold_lists = 0;
  457         pending = false;
  458 
  459         /* First pass: collect stuff that needs our attention. */
  460         (void)pthread_mutex_lock(&pool_lock);
  461         list_for_each_entry(pool_list, &pool_threads, thr_list)
  462         {
  463             if (collect_garbage(&state, pool_list)) {
  464                 pending = true;
  465             }
  466         }
  467         (void)pthread_mutex_unlock(&pool_lock);
  468 
  469         /* Second pass: free cold objects from live pools. */
  470         for (i = 0; i < state.n_cold_lists; ++i) {
  471             free_obj_list(state.cold_lists[i]);
  472         }
  473         (void)pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
  474     }
  475 
  476     return NULL;
  477 }
  478 
  479 void
  480 mem_pool_thread_destructor(per_thread_pool_list_t *pool_list)
  481 {
  482     per_thread_pool_t *pt_pool;
  483     uint32_t i;
  484 
  485     if (pool_list == NULL) {
  486         pool_list = thread_pool_list;
  487     }
  488 
  489     /* The current thread is terminating. None of the allocated objects will
  490      * be used again. We can directly destroy them here instead of delaying
  491      * it until the next sweeper loop. */
  492     if (pool_list != NULL) {
  493         /* Remove pool_list from the global list to avoid that sweeper
  494          * could touch it. */
  495         pthread_mutex_lock(&pool_lock);
  496         list_del(&pool_list->thr_list);
  497         pthread_mutex_unlock(&pool_lock);
  498 
  499         /* We need to protect hot/cold changes from potential mem_put() calls
  500          * that reference this pool_list. Once poison is set to true, we are
  501          * sure that no one else will touch hot/cold lists. The only possible
  502          * race is when at the same moment a mem_put() is adding a new item
  503          * to the hot list. We protect from that by taking pool_list->lock.
  504          * After that we don't need the lock to destroy the hot/cold lists. */
  505         pthread_spin_lock(&pool_list->lock);
  506         pool_list->poison = true;
  507         pthread_spin_unlock(&pool_list->lock);
  508 
  509         for (i = 0; i < NPOOLS; i++) {
  510             pt_pool = &pool_list->pools[i];
  511 
  512             free_obj_list(pt_pool->hot_list);
  513             pt_pool->hot_list = NULL;
  514 
  515             free_obj_list(pt_pool->cold_list);
  516             pt_pool->cold_list = NULL;
  517         }
  518 
  519         pthread_mutex_lock(&pool_free_lock);
  520         list_add(&pool_list->thr_list, &pool_free_threads);
  521         pthread_mutex_unlock(&pool_free_lock);
  522 
  523         thread_pool_list = NULL;
  524     }
  525 }
  526 
  527 static __attribute__((constructor)) void
  528 mem_pools_preinit(void)
  529 {
  530     unsigned int i;
  531 
  532     INIT_LIST_HEAD(&pool_threads);
  533     INIT_LIST_HEAD(&pool_free_threads);
  534 
  535     for (i = 0; i < NPOOLS; ++i) {
  536         pools[i].power_of_two = POOL_SMALLEST + i;
  537 
  538         GF_ATOMIC_INIT(pools[i].allocs_hot, 0);
  539         GF_ATOMIC_INIT(pools[i].allocs_cold, 0);
  540         GF_ATOMIC_INIT(pools[i].allocs_stdc, 0);
  541         GF_ATOMIC_INIT(pools[i].frees_to_list, 0);
  542     }
  543 
  544     pool_list_size = sizeof(per_thread_pool_list_t) +
  545                      sizeof(per_thread_pool_t) * (NPOOLS - 1);
  546 
  547     init_done = GF_MEMPOOL_INIT_EARLY;
  548 }
  549 
  550 static __attribute__((destructor)) void
  551 mem_pools_postfini(void)
  552 {
  553     /* TODO: This function should destroy all per thread memory pools that
  554      *       are still alive, but this is not possible right now because glibc
  555      *       starts calling destructors as soon as exit() is called, and
  556      *       gluster doesn't ensure that all threads have been stopped before
  557      *       calling exit(). Existing threads would crash when they try to use
  558      *       memory or they terminate if we destroy things here.
  559      *
  560      *       When we propertly terminate all threads, we can add the needed
  561      *       code here. Till then we need to leave the memory allocated. Most
  562      *       probably this function will be executed on process termination,
  563      *       so the memory will be released anyway by the system. */
  564 }
  565 
  566 /* Call mem_pools_init() once threading has been configured completely. This
  567  * prevent the pool_sweeper thread from getting killed once the main() thread
  568  * exits during deamonizing. */
  569 void
  570 mem_pools_init(void)
  571 {
  572     pthread_mutex_lock(&init_mutex);
  573     if ((init_count++) == 0) {
  574         (void)gf_thread_create(&sweeper_tid, NULL, pool_sweeper, NULL,
  575                                "memsweep");
  576 
  577         init_done = GF_MEMPOOL_INIT_LATE;
  578     }
  579     pthread_mutex_unlock(&init_mutex);
  580 }
  581 
  582 void
  583 mem_pools_fini(void)
  584 {
  585     pthread_mutex_lock(&init_mutex);
  586     switch (init_count) {
  587         case 0:
  588             /*
  589              * If init_count is already zero (as e.g. if somebody called this
  590              * before mem_pools_init) then the sweeper was probably never even
  591              * started so we don't need to stop it. Even if there's some crazy
  592              * circumstance where there is a sweeper but init_count is still
  593              * zero, that just means we'll leave it running. Not perfect, but
  594              * far better than any known alternative.
  595              */
  596             break;
  597         case 1: {
  598             /* if mem_pools_init() was not called, sweeper_tid will be invalid
  599              * and the functions will error out. That is not critical. In all
  600              * other cases, the sweeper_tid will be valid and the thread gets
  601              * stopped. */
  602             (void)pthread_cancel(sweeper_tid);
  603             (void)pthread_join(sweeper_tid, NULL);
  604 
  605             /* There could be threads still running in some cases, so we can't
  606              * destroy pool_lists in use. We can also not destroy unused
  607              * pool_lists because some allocated objects may still be pointing
  608              * to them. */
  609             mem_pool_thread_destructor(NULL);
  610 
  611             init_done = GF_MEMPOOL_INIT_DESTROY;
  612             /* Fall through. */
  613         }
  614         default:
  615             --init_count;
  616     }
  617     pthread_mutex_unlock(&init_mutex);
  618 }
  619 
  620 #else
  621 void
  622 mem_pools_init(void)
  623 {
  624 }
  625 void
  626 mem_pools_fini(void)
  627 {
  628 }
  629 void
  630 mem_pool_thread_destructor(per_thread_pool_list_t *pool_list)
  631 {
  632 }
  633 
  634 #endif
  635 
  636 struct mem_pool *
  637 mem_pool_new_fn(glusterfs_ctx_t *ctx, unsigned long sizeof_type,
  638                 unsigned long count, char *name)
  639 {
  640     unsigned long extra_size, size;
  641     unsigned int power;
  642     struct mem_pool *new = NULL;
  643     struct mem_pool_shared *pool = NULL;
  644 
  645     if (!sizeof_type) {
  646         gf_msg_callingfn("mem-pool", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
  647                          "invalid argument");
  648         return NULL;
  649     }
  650 
  651     /* This is the overhead we'll have because of memory accounting for each
  652      * memory block. */
  653     extra_size = sizeof(pooled_obj_hdr_t);
  654 
  655     /* We need to compute the total space needed to hold the data type and
  656      * the header. Given that the smallest block size we have in the pools
  657      * is 2^POOL_SMALLEST, we need to take the MAX(size, 2^POOL_SMALLEST).
  658      * However, since this value is only needed to compute its rounded
  659      * logarithm in base 2, and this only depends on the highest bit set,
  660      * we can simply do a bitwise or with the minimum size. We need to
  661      * subtract 1 for correct handling of sizes that are exactly a power
  662      * of 2. */
  663     size = (sizeof_type + extra_size - 1UL) | ((1UL << POOL_SMALLEST) - 1UL);
  664 
  665     /* We compute the logarithm in base 2 rounded up of the resulting size.
  666      * This value will identify which pool we need to use from the pools of
  667      * powers of 2. This is equivalent to finding the position of the highest
  668      * bit set. */
  669     power = sizeof(size) * 8 - __builtin_clzl(size);
  670     if (power > POOL_LARGEST) {
  671         gf_msg_callingfn("mem-pool", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
  672                          "invalid argument");
  673         return NULL;
  674     }
  675     pool = &pools[power - POOL_SMALLEST];
  676 
  677     new = GF_MALLOC(sizeof(struct mem_pool), gf_common_mt_mem_pool);
  678     if (!new)
  679         return NULL;
  680 
  681     new->ctx = ctx;
  682     new->sizeof_type = sizeof_type;
  683     new->count = count;
  684     new->name = name;
  685     new->pool = pool;
  686     GF_ATOMIC_INIT(new->active, 0);
  687 #ifdef DEBUG
  688     GF_ATOMIC_INIT(new->hit, 0);
  689     GF_ATOMIC_INIT(new->miss, 0);
  690 #endif
  691     INIT_LIST_HEAD(&new->owner);
  692 
  693     LOCK(&ctx->lock);
  694     {
  695         list_add(&new->owner, &ctx->mempool_list);
  696     }
  697     UNLOCK(&ctx->lock);
  698 
  699     return new;
  700 }
  701 
  702 void *
  703 mem_get0(struct mem_pool *mem_pool)
  704 {
  705     void *ptr = mem_get(mem_pool);
  706     if (ptr) {
  707 #if defined(GF_DISABLE_MEMPOOL)
  708         memset(ptr, 0, mem_pool->sizeof_type);
  709 #else
  710         memset(ptr, 0, AVAILABLE_SIZE(mem_pool->pool->power_of_two));
  711 #endif
  712     }
  713 
  714     return ptr;
  715 }
  716 
  717 per_thread_pool_list_t *
  718 mem_get_pool_list(void)
  719 {
  720     per_thread_pool_list_t *pool_list;
  721     unsigned int i;
  722 
  723     pool_list = thread_pool_list;
  724     if (pool_list) {
  725         return pool_list;
  726     }
  727 
  728     (void)pthread_mutex_lock(&pool_free_lock);
  729     if (!list_empty(&pool_free_threads)) {
  730         pool_list = list_entry(pool_free_threads.next, per_thread_pool_list_t,
  731                                thr_list);
  732         list_del(&pool_list->thr_list);
  733     }
  734     (void)pthread_mutex_unlock(&pool_free_lock);
  735 
  736     if (!pool_list) {
  737         pool_list = MALLOC(pool_list_size);
  738         if (!pool_list) {
  739             return NULL;
  740         }
  741 
  742         INIT_LIST_HEAD(&pool_list->thr_list);
  743         (void)pthread_spin_init(&pool_list->lock, PTHREAD_PROCESS_PRIVATE);
  744         for (i = 0; i < NPOOLS; ++i) {
  745             pool_list->pools[i].parent = &pools[i];
  746             pool_list->pools[i].hot_list = NULL;
  747             pool_list->pools[i].cold_list = NULL;
  748         }
  749     }
  750 
  751     /* There's no need to take pool_list->lock, because this is already an
  752      * atomic operation and we don't need to synchronize it with any change
  753      * in hot/cold lists. */
  754     pool_list->poison = false;
  755 
  756     (void)pthread_mutex_lock(&pool_lock);
  757     list_add(&pool_list->thr_list, &pool_threads);
  758     (void)pthread_mutex_unlock(&pool_lock);
  759 
  760     thread_pool_list = pool_list;
  761 
  762     /* Ensure that all memory objects associated to the new pool_list are
  763      * destroyed when the thread terminates. */
  764     gf_thread_needs_cleanup();
  765 
  766     return pool_list;
  767 }
  768 
  769 static pooled_obj_hdr_t *
  770 mem_get_from_pool(struct mem_pool *mem_pool)
  771 {
  772     per_thread_pool_list_t *pool_list;
  773     per_thread_pool_t *pt_pool;
  774     pooled_obj_hdr_t *retval;
  775 #ifdef DEBUG
  776     gf_boolean_t hit = _gf_true;
  777 #endif
  778 
  779     pool_list = mem_get_pool_list();
  780     if (!pool_list || pool_list->poison) {
  781         return NULL;
  782     }
  783 
  784     pt_pool = &pool_list->pools[mem_pool->pool->power_of_two - POOL_SMALLEST];
  785 
  786     (void)pthread_spin_lock(&pool_list->lock);
  787 
  788     retval = pt_pool->hot_list;
  789     if (retval) {
  790         pt_pool->hot_list = retval->next;
  791         (void)pthread_spin_unlock(&pool_list->lock);
  792         GF_ATOMIC_INC(pt_pool->parent->allocs_hot);
  793     } else {
  794         retval = pt_pool->cold_list;
  795         if (retval) {
  796             pt_pool->cold_list = retval->next;
  797             (void)pthread_spin_unlock(&pool_list->lock);
  798             GF_ATOMIC_INC(pt_pool->parent->allocs_cold);
  799         } else {
  800             (void)pthread_spin_unlock(&pool_list->lock);
  801             GF_ATOMIC_INC(pt_pool->parent->allocs_stdc);
  802             retval = malloc(1 << pt_pool->parent->power_of_two);
  803 #ifdef DEBUG
  804             hit = _gf_false;
  805 #endif
  806         }
  807     }
  808 
  809     if (retval != NULL) {
  810         retval->pool = mem_pool;
  811         retval->power_of_two = mem_pool->pool->power_of_two;
  812 #ifdef DEBUG
  813         if (hit == _gf_true)
  814             GF_ATOMIC_INC(mem_pool->hit);
  815         else
  816             GF_ATOMIC_INC(mem_pool->miss);
  817 #endif
  818         retval->magic = GF_MEM_HEADER_MAGIC;
  819         retval->pool_list = pool_list;
  820     }
  821 
  822     return retval;
  823 }
  824 
  825 void *
  826 mem_get(struct mem_pool *mem_pool)
  827 {
  828     if (!mem_pool) {
  829         gf_msg_callingfn("mem-pool", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
  830                          "invalid argument");
  831         return NULL;
  832     }
  833 
  834 #if defined(GF_DISABLE_MEMPOOL)
  835     return GF_MALLOC(mem_pool->sizeof_type, gf_common_mt_mem_pool);
  836 #else
  837     pooled_obj_hdr_t *retval = mem_get_from_pool(mem_pool);
  838     if (!retval) {
  839         return NULL;
  840     }
  841 
  842     GF_ATOMIC_INC(mem_pool->active);
  843 
  844     return retval + 1;
  845 #endif /* GF_DISABLE_MEMPOOL */
  846 }
  847 
  848 void
  849 mem_put(void *ptr)
  850 {
  851 #if defined(GF_DISABLE_MEMPOOL)
  852     GF_FREE(ptr);
  853 #else
  854     pooled_obj_hdr_t *hdr;
  855     per_thread_pool_list_t *pool_list;
  856     per_thread_pool_t *pt_pool;
  857 
  858     if (!ptr) {
  859         gf_msg_callingfn("mem-pool", GF_LOG_ERROR, EINVAL, LG_MSG_INVALID_ARG,
  860                          "invalid argument");
  861         return;
  862     }
  863 
  864     hdr = ((pooled_obj_hdr_t *)ptr) - 1;
  865     if (hdr->magic != GF_MEM_HEADER_MAGIC) {
  866         /* Not one of ours; don't touch it. */
  867         return;
  868     }
  869 
  870     if (!hdr->pool_list) {
  871         gf_msg_callingfn("mem-pool", GF_LOG_CRITICAL, EINVAL,
  872                          LG_MSG_INVALID_ARG,
  873                          "invalid argument hdr->pool_list NULL");
  874         return;
  875     }
  876 
  877     pool_list = hdr->pool_list;
  878     pt_pool = &pool_list->pools[hdr->power_of_two - POOL_SMALLEST];
  879 
  880     if (hdr->pool)
  881         GF_ATOMIC_DEC(hdr->pool->active);
  882 
  883     hdr->magic = GF_MEM_INVALID_MAGIC;
  884 
  885     (void)pthread_spin_lock(&pool_list->lock);
  886     if (!pool_list->poison) {
  887         hdr->next = pt_pool->hot_list;
  888         pt_pool->hot_list = hdr;
  889         (void)pthread_spin_unlock(&pool_list->lock);
  890         GF_ATOMIC_INC(pt_pool->parent->frees_to_list);
  891     } else {
  892         /* If the owner thread of this element has terminated, we simply
  893          * release its memory. */
  894         (void)pthread_spin_unlock(&pool_list->lock);
  895         free(hdr);
  896     }
  897 #endif /* GF_DISABLE_MEMPOOL */
  898 }
  899 
  900 void
  901 mem_pool_destroy(struct mem_pool *pool)
  902 {
  903     if (!pool)
  904         return;
  905 
  906     /* remove this pool from the owner (glusterfs_ctx_t) */
  907     LOCK(&pool->ctx->lock);
  908     {
  909         list_del(&pool->owner);
  910     }
  911     UNLOCK(&pool->ctx->lock);
  912 
  913     /* free this pool, but keep the mem_pool_shared */
  914     GF_FREE(pool);
  915 
  916     /*
  917      * Pools are now permanent, so the mem_pool->pool is kept around. All
  918      * of the objects *in* the pool will eventually be freed via the
  919      * pool-sweeper thread, and this way we don't have to add a lot of
  920      * reference-counting complexity.
  921      */
  922 }