"Fossies" - the Fresh Open Source Software Archive

Member "glibc-2.36/malloc/malloc.c" (29 Jul 2022, 193335 Bytes) of package /linux/misc/glibc-2.36.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "malloc.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.35_vs_2.36.

    1 /* Malloc implementation for multiple threads without lock contention.
    2    Copyright (C) 1996-2022 Free Software Foundation, Inc.
    3    Copyright The GNU Toolchain Authors.
    4    This file is part of the GNU C Library.
    5 
    6    The GNU C Library is free software; you can redistribute it and/or
    7    modify it under the terms of the GNU Lesser General Public License as
    8    published by the Free Software Foundation; either version 2.1 of the
    9    License, or (at your option) any later version.
   10 
   11    The GNU C Library is distributed in the hope that it will be useful,
   12    but WITHOUT ANY WARRANTY; without even the implied warranty of
   13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   14    Lesser General Public License for more details.
   15 
   16    You should have received a copy of the GNU Lesser General Public
   17    License along with the GNU C Library; see the file COPYING.LIB.  If
   18    not, see <https://www.gnu.org/licenses/>.  */
   19 
   20 /*
   21   This is a version (aka ptmalloc2) of malloc/free/realloc written by
   22   Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
   23 
   24   There have been substantial changes made after the integration into
   25   glibc in all parts of the code.  Do not look for much commonality
   26   with the ptmalloc2 version.
   27 
   28 * Version ptmalloc2-20011215
   29   based on:
   30   VERSION 2.7.0 Sun Mar 11 14:14:06 2001  Doug Lea  (dl at gee)
   31 
   32 * Quickstart
   33 
   34   In order to compile this implementation, a Makefile is provided with
   35   the ptmalloc2 distribution, which has pre-defined targets for some
   36   popular systems (e.g. "make posix" for Posix threads).  All that is
   37   typically required with regard to compiler flags is the selection of
   38   the thread package via defining one out of USE_PTHREADS, USE_THR or
   39   USE_SPROC.  Check the thread-m.h file for what effects this has.
   40   Many/most systems will additionally require USE_TSD_DATA_HACK to be
   41   defined, so this is the default for "make posix".
   42 
   43 * Why use this malloc?
   44 
   45   This is not the fastest, most space-conserving, most portable, or
   46   most tunable malloc ever written. However it is among the fastest
   47   while also being among the most space-conserving, portable and tunable.
   48   Consistent balance across these factors results in a good general-purpose
   49   allocator for malloc-intensive programs.
   50 
   51   The main properties of the algorithms are:
   52   * For large (>= 512 bytes) requests, it is a pure best-fit allocator,
   53     with ties normally decided via FIFO (i.e. least recently used).
   54   * For small (<= 64 bytes by default) requests, it is a caching
   55     allocator, that maintains pools of quickly recycled chunks.
   56   * In between, and for combinations of large and small requests, it does
   57     the best it can trying to meet both goals at once.
   58   * For very large requests (>= 128KB by default), it relies on system
   59     memory mapping facilities, if supported.
   60 
   61   For a longer but slightly out of date high-level description, see
   62      http://gee.cs.oswego.edu/dl/html/malloc.html
   63 
   64   You may already by default be using a C library containing a malloc
   65   that is  based on some version of this malloc (for example in
   66   linux). You might still want to use the one in this file in order to
   67   customize settings or to avoid overheads associated with library
   68   versions.
   69 
   70 * Contents, described in more detail in "description of public routines" below.
   71 
   72   Standard (ANSI/SVID/...)  functions:
   73     malloc(size_t n);
   74     calloc(size_t n_elements, size_t element_size);
   75     free(void* p);
   76     realloc(void* p, size_t n);
   77     memalign(size_t alignment, size_t n);
   78     valloc(size_t n);
   79     mallinfo()
   80     mallopt(int parameter_number, int parameter_value)
   81 
   82   Additional functions:
   83     independent_calloc(size_t n_elements, size_t size, void* chunks[]);
   84     independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
   85     pvalloc(size_t n);
   86     malloc_trim(size_t pad);
   87     malloc_usable_size(void* p);
   88     malloc_stats();
   89 
   90 * Vital statistics:
   91 
   92   Supported pointer representation:       4 or 8 bytes
   93   Supported size_t  representation:       4 or 8 bytes
   94        Note that size_t is allowed to be 4 bytes even if pointers are 8.
   95        You can adjust this by defining INTERNAL_SIZE_T
   96 
   97   Alignment:                              2 * sizeof(size_t) (default)
   98        (i.e., 8 byte alignment with 4byte size_t). This suffices for
   99        nearly all current machines and C compilers. However, you can
  100        define MALLOC_ALIGNMENT to be wider than this if necessary.
  101 
  102   Minimum overhead per allocated chunk:   4 or 8 bytes
  103        Each malloced chunk has a hidden word of overhead holding size
  104        and status information.
  105 
  106   Minimum allocated size: 4-byte ptrs:  16 bytes    (including 4 overhead)
  107               8-byte ptrs:  24/32 bytes (including, 4/8 overhead)
  108 
  109        When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
  110        ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
  111        needed; 4 (8) for a trailing size field and 8 (16) bytes for
  112        free list pointers. Thus, the minimum allocatable size is
  113        16/24/32 bytes.
  114 
  115        Even a request for zero bytes (i.e., malloc(0)) returns a
  116        pointer to something of the minimum allocatable size.
  117 
  118        The maximum overhead wastage (i.e., number of extra bytes
  119        allocated than were requested in malloc) is less than or equal
  120        to the minimum size, except for requests >= mmap_threshold that
  121        are serviced via mmap(), where the worst case wastage is 2 *
  122        sizeof(size_t) bytes plus the remainder from a system page (the
  123        minimal mmap unit); typically 4096 or 8192 bytes.
  124 
  125   Maximum allocated size:  4-byte size_t: 2^32 minus about two pages
  126                8-byte size_t: 2^64 minus about two pages
  127 
  128        It is assumed that (possibly signed) size_t values suffice to
  129        represent chunk sizes. `Possibly signed' is due to the fact
  130        that `size_t' may be defined on a system as either a signed or
  131        an unsigned type. The ISO C standard says that it must be
  132        unsigned, but a few systems are known not to adhere to this.
  133        Additionally, even when size_t is unsigned, sbrk (which is by
  134        default used to obtain memory from system) accepts signed
  135        arguments, and may not be able to handle size_t-wide arguments
  136        with negative sign bit.  Generally, values that would
  137        appear as negative after accounting for overhead and alignment
  138        are supported only via mmap(), which does not have this
  139        limitation.
  140 
  141        Requests for sizes outside the allowed range will perform an optional
  142        failure action and then return null. (Requests may also
  143        also fail because a system is out of memory.)
  144 
  145   Thread-safety: thread-safe
  146 
  147   Compliance: I believe it is compliant with the 1997 Single Unix Specification
  148        Also SVID/XPG, ANSI C, and probably others as well.
  149 
  150 * Synopsis of compile-time options:
  151 
  152     People have reported using previous versions of this malloc on all
  153     versions of Unix, sometimes by tweaking some of the defines
  154     below. It has been tested most extensively on Solaris and Linux.
  155     People also report using it in stand-alone embedded systems.
  156 
  157     The implementation is in straight, hand-tuned ANSI C.  It is not
  158     at all modular. (Sorry!)  It uses a lot of macros.  To be at all
  159     usable, this code should be compiled using an optimizing compiler
  160     (for example gcc -O3) that can simplify expressions and control
  161     paths. (FAQ: some macros import variables as arguments rather than
  162     declare locals because people reported that some debuggers
  163     otherwise get confused.)
  164 
  165     OPTION                     DEFAULT VALUE
  166 
  167     Compilation Environment options:
  168 
  169     HAVE_MREMAP                0
  170 
  171     Changing default word sizes:
  172 
  173     INTERNAL_SIZE_T            size_t
  174 
  175     Configuration and functionality options:
  176 
  177     USE_PUBLIC_MALLOC_WRAPPERS NOT defined
  178     USE_MALLOC_LOCK            NOT defined
  179     MALLOC_DEBUG               NOT defined
  180     REALLOC_ZERO_BYTES_FREES   1
  181     TRIM_FASTBINS              0
  182 
  183     Options for customizing MORECORE:
  184 
  185     MORECORE                   sbrk
  186     MORECORE_FAILURE           -1
  187     MORECORE_CONTIGUOUS        1
  188     MORECORE_CANNOT_TRIM       NOT defined
  189     MORECORE_CLEARS            1
  190     MMAP_AS_MORECORE_SIZE      (1024 * 1024)
  191 
  192     Tuning options that are also dynamically changeable via mallopt:
  193 
  194     DEFAULT_MXFAST             64 (for 32bit), 128 (for 64bit)
  195     DEFAULT_TRIM_THRESHOLD     128 * 1024
  196     DEFAULT_TOP_PAD            0
  197     DEFAULT_MMAP_THRESHOLD     128 * 1024
  198     DEFAULT_MMAP_MAX           65536
  199 
  200     There are several other #defined constants and macros that you
  201     probably don't want to touch unless you are extending or adapting malloc.  */
  202 
  203 /*
  204   void* is the pointer type that malloc should say it returns
  205 */
  206 
  207 #ifndef void
  208 #define void      void
  209 #endif /*void*/
  210 
  211 #include <stddef.h>   /* for size_t */
  212 #include <stdlib.h>   /* for getenv(), abort() */
  213 #include <unistd.h>   /* for __libc_enable_secure */
  214 
  215 #include <atomic.h>
  216 #include <_itoa.h>
  217 #include <bits/wordsize.h>
  218 #include <sys/sysinfo.h>
  219 
  220 #include <ldsodefs.h>
  221 
  222 #include <unistd.h>
  223 #include <stdio.h>    /* needed for malloc_stats */
  224 #include <errno.h>
  225 #include <assert.h>
  226 
  227 #include <shlib-compat.h>
  228 
  229 /* For uintptr_t.  */
  230 #include <stdint.h>
  231 
  232 /* For va_arg, va_start, va_end.  */
  233 #include <stdarg.h>
  234 
  235 /* For MIN, MAX, powerof2.  */
  236 #include <sys/param.h>
  237 
  238 /* For ALIGN_UP et. al.  */
  239 #include <libc-pointer-arith.h>
  240 
  241 /* For DIAG_PUSH/POP_NEEDS_COMMENT et al.  */
  242 #include <libc-diag.h>
  243 
  244 /* For memory tagging.  */
  245 #include <libc-mtag.h>
  246 
  247 #include <malloc/malloc-internal.h>
  248 
  249 /* For SINGLE_THREAD_P.  */
  250 #include <sysdep-cancel.h>
  251 
  252 #include <libc-internal.h>
  253 
  254 /* For tcache double-free check.  */
  255 #include <random-bits.h>
  256 #include <sys/random.h>
  257 
  258 /*
  259   Debugging:
  260 
  261   Because freed chunks may be overwritten with bookkeeping fields, this
  262   malloc will often die when freed memory is overwritten by user
  263   programs.  This can be very effective (albeit in an annoying way)
  264   in helping track down dangling pointers.
  265 
  266   If you compile with -DMALLOC_DEBUG, a number of assertion checks are
  267   enabled that will catch more memory errors. You probably won't be
  268   able to make much sense of the actual assertion errors, but they
  269   should help you locate incorrectly overwritten memory.  The checking
  270   is fairly extensive, and will slow down execution
  271   noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
  272   will attempt to check every non-mmapped allocated and free chunk in
  273   the course of computing the summmaries. (By nature, mmapped regions
  274   cannot be checked very much automatically.)
  275 
  276   Setting MALLOC_DEBUG may also be helpful if you are trying to modify
  277   this code. The assertions in the check routines spell out in more
  278   detail the assumptions and invariants underlying the algorithms.
  279 
  280   Setting MALLOC_DEBUG does NOT provide an automated mechanism for
  281   checking that all accesses to malloced memory stay within their
  282   bounds. However, there are several add-ons and adaptations of this
  283   or other mallocs available that do this.
  284 */
  285 
  286 #ifndef MALLOC_DEBUG
  287 #define MALLOC_DEBUG 0
  288 #endif
  289 
  290 #if IS_IN (libc)
  291 #ifndef NDEBUG
  292 # define __assert_fail(assertion, file, line, function)         \
  293      __malloc_assert(assertion, file, line, function)
  294 
  295 _Noreturn static void
  296 __malloc_assert (const char *assertion, const char *file, unsigned int line,
  297          const char *function)
  298 {
  299   __libc_message (do_abort, "\
  300 Fatal glibc error: malloc assertion failure in %s: %s\n",
  301           function, assertion);
  302   __builtin_unreachable ();
  303 }
  304 #endif
  305 #endif
  306 
  307 #if USE_TCACHE
  308 /* We want 64 entries.  This is an arbitrary limit, which tunables can reduce.  */
  309 # define TCACHE_MAX_BINS        64
  310 # define MAX_TCACHE_SIZE    tidx2usize (TCACHE_MAX_BINS-1)
  311 
  312 /* Only used to pre-fill the tunables.  */
  313 # define tidx2usize(idx)    (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
  314 
  315 /* When "x" is from chunksize().  */
  316 # define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT)
  317 /* When "x" is a user-provided size.  */
  318 # define usize2tidx(x) csize2tidx (request2size (x))
  319 
  320 /* With rounding and alignment, the bins are...
  321    idx 0   bytes 0..24 (64-bit) or 0..12 (32-bit)
  322    idx 1   bytes 25..40 or 13..20
  323    idx 2   bytes 41..56 or 21..28
  324    etc.  */
  325 
  326 /* This is another arbitrary limit, which tunables can change.  Each
  327    tcache bin will hold at most this number of chunks.  */
  328 # define TCACHE_FILL_COUNT 7
  329 
  330 /* Maximum chunks in tcache bins for tunables.  This value must fit the range
  331    of tcache->counts[] entries, else they may overflow.  */
  332 # define MAX_TCACHE_COUNT UINT16_MAX
  333 #endif
  334 
  335 /* Safe-Linking:
  336    Use randomness from ASLR (mmap_base) to protect single-linked lists
  337    of Fast-Bins and TCache.  That is, mask the "next" pointers of the
  338    lists' chunks, and also perform allocation alignment checks on them.
  339    This mechanism reduces the risk of pointer hijacking, as was done with
  340    Safe-Unlinking in the double-linked lists of Small-Bins.
  341    It assumes a minimum page size of 4096 bytes (12 bits).  Systems with
  342    larger pages provide less entropy, although the pointer mangling
  343    still works.  */
  344 #define PROTECT_PTR(pos, ptr) \
  345   ((__typeof (ptr)) ((((size_t) pos) >> 12) ^ ((size_t) ptr)))
  346 #define REVEAL_PTR(ptr)  PROTECT_PTR (&ptr, ptr)
  347 
  348 /*
  349   The REALLOC_ZERO_BYTES_FREES macro controls the behavior of realloc (p, 0)
  350   when p is nonnull.  If the macro is nonzero, the realloc call returns NULL;
  351   otherwise, the call returns what malloc (0) would.  In either case,
  352   p is freed.  Glibc uses a nonzero REALLOC_ZERO_BYTES_FREES, which
  353   implements common historical practice.
  354 
  355   ISO C17 says the realloc call has implementation-defined behavior,
  356   and it might not even free p.
  357 */
  358 
  359 #ifndef REALLOC_ZERO_BYTES_FREES
  360 #define REALLOC_ZERO_BYTES_FREES 1
  361 #endif
  362 
  363 /*
  364   TRIM_FASTBINS controls whether free() of a very small chunk can
  365   immediately lead to trimming. Setting to true (1) can reduce memory
  366   footprint, but will almost always slow down programs that use a lot
  367   of small chunks.
  368 
  369   Define this only if you are willing to give up some speed to more
  370   aggressively reduce system-level memory footprint when releasing
  371   memory in programs that use many small chunks.  You can get
  372   essentially the same effect by setting MXFAST to 0, but this can
  373   lead to even greater slowdowns in programs using many small chunks.
  374   TRIM_FASTBINS is an in-between compile-time option, that disables
  375   only those chunks bordering topmost memory from being placed in
  376   fastbins.
  377 */
  378 
  379 #ifndef TRIM_FASTBINS
  380 #define TRIM_FASTBINS  0
  381 #endif
  382 
  383 /* Definition for getting more memory from the OS.  */
  384 #include "morecore.c"
  385 
  386 #define MORECORE         (*__glibc_morecore)
  387 #define MORECORE_FAILURE 0
  388 
  389 /* Memory tagging.  */
  390 
  391 /* Some systems support the concept of tagging (sometimes known as
  392    coloring) memory locations on a fine grained basis.  Each memory
  393    location is given a color (normally allocated randomly) and
  394    pointers are also colored.  When the pointer is dereferenced, the
  395    pointer's color is checked against the memory's color and if they
  396    differ the access is faulted (sometimes lazily).
  397 
  398    We use this in glibc by maintaining a single color for the malloc
  399    data structures that are interleaved with the user data and then
  400    assigning separate colors for each block allocation handed out.  In
  401    this way simple buffer overruns will be rapidly detected.  When
  402    memory is freed, the memory is recolored back to the glibc default
  403    so that simple use-after-free errors can also be detected.
  404 
  405    If memory is reallocated the buffer is recolored even if the
  406    address remains the same.  This has a performance impact, but
  407    guarantees that the old pointer cannot mistakenly be reused (code
  408    that compares old against new will see a mismatch and will then
  409    need to behave as though realloc moved the data to a new location).
  410 
  411    Internal API for memory tagging support.
  412 
  413    The aim is to keep the code for memory tagging support as close to
  414    the normal APIs in glibc as possible, so that if tagging is not
  415    enabled in the library, or is disabled at runtime then standard
  416    operations can continue to be used.  Support macros are used to do
  417    this:
  418 
  419    void *tag_new_zero_region (void *ptr, size_t size)
  420 
  421    Allocates a new tag, colors the memory with that tag, zeros the
  422    memory and returns a pointer that is correctly colored for that
  423    location.  The non-tagging version will simply call memset with 0.
  424 
  425    void *tag_region (void *ptr, size_t size)
  426 
  427    Color the region of memory pointed to by PTR and size SIZE with
  428    the color of PTR.  Returns the original pointer.
  429 
  430    void *tag_new_usable (void *ptr)
  431 
  432    Allocate a new random color and use it to color the user region of
  433    a chunk; this may include data from the subsequent chunk's header
  434    if tagging is sufficiently fine grained.  Returns PTR suitably
  435    recolored for accessing the memory there.
  436 
  437    void *tag_at (void *ptr)
  438 
  439    Read the current color of the memory at the address pointed to by
  440    PTR (ignoring it's current color) and return PTR recolored to that
  441    color.  PTR must be valid address in all other respects.  When
  442    tagging is not enabled, it simply returns the original pointer.
  443 */
  444 
  445 #ifdef USE_MTAG
  446 static bool mtag_enabled = false;
  447 static int mtag_mmap_flags = 0;
  448 #else
  449 # define mtag_enabled false
  450 # define mtag_mmap_flags 0
  451 #endif
  452 
  453 static __always_inline void *
  454 tag_region (void *ptr, size_t size)
  455 {
  456   if (__glibc_unlikely (mtag_enabled))
  457     return __libc_mtag_tag_region (ptr, size);
  458   return ptr;
  459 }
  460 
  461 static __always_inline void *
  462 tag_new_zero_region (void *ptr, size_t size)
  463 {
  464   if (__glibc_unlikely (mtag_enabled))
  465     return __libc_mtag_tag_zero_region (__libc_mtag_new_tag (ptr), size);
  466   return memset (ptr, 0, size);
  467 }
  468 
  469 /* Defined later.  */
  470 static void *
  471 tag_new_usable (void *ptr);
  472 
  473 static __always_inline void *
  474 tag_at (void *ptr)
  475 {
  476   if (__glibc_unlikely (mtag_enabled))
  477     return __libc_mtag_address_get_tag (ptr);
  478   return ptr;
  479 }
  480 
  481 #include <string.h>
  482 
  483 /*
  484   MORECORE-related declarations. By default, rely on sbrk
  485 */
  486 
  487 
  488 /*
  489   MORECORE is the name of the routine to call to obtain more memory
  490   from the system.  See below for general guidance on writing
  491   alternative MORECORE functions, as well as a version for WIN32 and a
  492   sample version for pre-OSX macos.
  493 */
  494 
  495 #ifndef MORECORE
  496 #define MORECORE sbrk
  497 #endif
  498 
  499 /*
  500   MORECORE_FAILURE is the value returned upon failure of MORECORE
  501   as well as mmap. Since it cannot be an otherwise valid memory address,
  502   and must reflect values of standard sys calls, you probably ought not
  503   try to redefine it.
  504 */
  505 
  506 #ifndef MORECORE_FAILURE
  507 #define MORECORE_FAILURE (-1)
  508 #endif
  509 
  510 /*
  511   If MORECORE_CONTIGUOUS is true, take advantage of fact that
  512   consecutive calls to MORECORE with positive arguments always return
  513   contiguous increasing addresses.  This is true of unix sbrk.  Even
  514   if not defined, when regions happen to be contiguous, malloc will
  515   permit allocations spanning regions obtained from different
  516   calls. But defining this when applicable enables some stronger
  517   consistency checks and space efficiencies.
  518 */
  519 
  520 #ifndef MORECORE_CONTIGUOUS
  521 #define MORECORE_CONTIGUOUS 1
  522 #endif
  523 
  524 /*
  525   Define MORECORE_CANNOT_TRIM if your version of MORECORE
  526   cannot release space back to the system when given negative
  527   arguments. This is generally necessary only if you are using
  528   a hand-crafted MORECORE function that cannot handle negative arguments.
  529 */
  530 
  531 /* #define MORECORE_CANNOT_TRIM */
  532 
  533 /*  MORECORE_CLEARS           (default 1)
  534      The degree to which the routine mapped to MORECORE zeroes out
  535      memory: never (0), only for newly allocated space (1) or always
  536      (2).  The distinction between (1) and (2) is necessary because on
  537      some systems, if the application first decrements and then
  538      increments the break value, the contents of the reallocated space
  539      are unspecified.
  540  */
  541 
  542 #ifndef MORECORE_CLEARS
  543 # define MORECORE_CLEARS 1
  544 #endif
  545 
  546 
  547 /*
  548    MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
  549    sbrk fails, and mmap is used as a backup.  The value must be a
  550    multiple of page size.  This backup strategy generally applies only
  551    when systems have "holes" in address space, so sbrk cannot perform
  552    contiguous expansion, but there is still space available on system.
  553    On systems for which this is known to be useful (i.e. most linux
  554    kernels), this occurs only when programs allocate huge amounts of
  555    memory.  Between this, and the fact that mmap regions tend to be
  556    limited, the size should be large, to avoid too many mmap calls and
  557    thus avoid running out of kernel resources.  */
  558 
  559 #ifndef MMAP_AS_MORECORE_SIZE
  560 #define MMAP_AS_MORECORE_SIZE (1024 * 1024)
  561 #endif
  562 
  563 /*
  564   Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
  565   large blocks.
  566 */
  567 
  568 #ifndef HAVE_MREMAP
  569 #define HAVE_MREMAP 0
  570 #endif
  571 
  572 /*
  573   This version of malloc supports the standard SVID/XPG mallinfo
  574   routine that returns a struct containing usage properties and
  575   statistics. It should work on any SVID/XPG compliant system that has
  576   a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
  577   install such a thing yourself, cut out the preliminary declarations
  578   as described above and below and save them in a malloc.h file. But
  579   there's no compelling reason to bother to do this.)
  580 
  581   The main declaration needed is the mallinfo struct that is returned
  582   (by-copy) by mallinfo().  The SVID/XPG malloinfo struct contains a
  583   bunch of fields that are not even meaningful in this version of
  584   malloc.  These fields are are instead filled by mallinfo() with
  585   other numbers that might be of interest.
  586 */
  587 
  588 
  589 /* ---------- description of public routines ------------ */
  590 
  591 #if IS_IN (libc)
  592 /*
  593   malloc(size_t n)
  594   Returns a pointer to a newly allocated chunk of at least n bytes, or null
  595   if no space is available. Additionally, on failure, errno is
  596   set to ENOMEM on ANSI C systems.
  597 
  598   If n is zero, malloc returns a minimum-sized chunk. (The minimum
  599   size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
  600   systems.)  On most systems, size_t is an unsigned type, so calls
  601   with negative arguments are interpreted as requests for huge amounts
  602   of space, which will often fail. The maximum supported value of n
  603   differs across systems, but is in all cases less than the maximum
  604   representable value of a size_t.
  605 */
  606 void*  __libc_malloc(size_t);
  607 libc_hidden_proto (__libc_malloc)
  608 
  609 /*
  610   free(void* p)
  611   Releases the chunk of memory pointed to by p, that had been previously
  612   allocated using malloc or a related routine such as realloc.
  613   It has no effect if p is null. It can have arbitrary (i.e., bad!)
  614   effects if p has already been freed.
  615 
  616   Unless disabled (using mallopt), freeing very large spaces will
  617   when possible, automatically trigger operations that give
  618   back unused memory to the system, thus reducing program footprint.
  619 */
  620 void     __libc_free(void*);
  621 libc_hidden_proto (__libc_free)
  622 
  623 /*
  624   calloc(size_t n_elements, size_t element_size);
  625   Returns a pointer to n_elements * element_size bytes, with all locations
  626   set to zero.
  627 */
  628 void*  __libc_calloc(size_t, size_t);
  629 
  630 /*
  631   realloc(void* p, size_t n)
  632   Returns a pointer to a chunk of size n that contains the same data
  633   as does chunk p up to the minimum of (n, p's size) bytes, or null
  634   if no space is available.
  635 
  636   The returned pointer may or may not be the same as p. The algorithm
  637   prefers extending p when possible, otherwise it employs the
  638   equivalent of a malloc-copy-free sequence.
  639 
  640   If p is null, realloc is equivalent to malloc.
  641 
  642   If space is not available, realloc returns null, errno is set (if on
  643   ANSI) and p is NOT freed.
  644 
  645   if n is for fewer bytes than already held by p, the newly unused
  646   space is lopped off and freed if possible.  Unless the #define
  647   REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
  648   zero (re)allocates a minimum-sized chunk.
  649 
  650   Large chunks that were internally obtained via mmap will always be
  651   grown using malloc-copy-free sequences unless the system supports
  652   MREMAP (currently only linux).
  653 
  654   The old unix realloc convention of allowing the last-free'd chunk
  655   to be used as an argument to realloc is not supported.
  656 */
  657 void*  __libc_realloc(void*, size_t);
  658 libc_hidden_proto (__libc_realloc)
  659 
  660 /*
  661   memalign(size_t alignment, size_t n);
  662   Returns a pointer to a newly allocated chunk of n bytes, aligned
  663   in accord with the alignment argument.
  664 
  665   The alignment argument should be a power of two. If the argument is
  666   not a power of two, the nearest greater power is used.
  667   8-byte alignment is guaranteed by normal malloc calls, so don't
  668   bother calling memalign with an argument of 8 or less.
  669 
  670   Overreliance on memalign is a sure way to fragment space.
  671 */
  672 void*  __libc_memalign(size_t, size_t);
  673 libc_hidden_proto (__libc_memalign)
  674 
  675 /*
  676   valloc(size_t n);
  677   Equivalent to memalign(pagesize, n), where pagesize is the page
  678   size of the system. If the pagesize is unknown, 4096 is used.
  679 */
  680 void*  __libc_valloc(size_t);
  681 
  682 
  683 
  684 /*
  685   mallinfo()
  686   Returns (by copy) a struct containing various summary statistics:
  687 
  688   arena:     current total non-mmapped bytes allocated from system
  689   ordblks:   the number of free chunks
  690   smblks:    the number of fastbin blocks (i.e., small chunks that
  691            have been freed but not use resused or consolidated)
  692   hblks:     current number of mmapped regions
  693   hblkhd:    total bytes held in mmapped regions
  694   usmblks:   always 0
  695   fsmblks:   total bytes held in fastbin blocks
  696   uordblks:  current total allocated space (normal or mmapped)
  697   fordblks:  total free space
  698   keepcost:  the maximum number of bytes that could ideally be released
  699            back to system via malloc_trim. ("ideally" means that
  700            it ignores page restrictions etc.)
  701 
  702   Because these fields are ints, but internal bookkeeping may
  703   be kept as longs, the reported values may wrap around zero and
  704   thus be inaccurate.
  705 */
  706 struct mallinfo2 __libc_mallinfo2(void);
  707 libc_hidden_proto (__libc_mallinfo2)
  708 
  709 struct mallinfo __libc_mallinfo(void);
  710 
  711 
  712 /*
  713   pvalloc(size_t n);
  714   Equivalent to valloc(minimum-page-that-holds(n)), that is,
  715   round up n to nearest pagesize.
  716  */
  717 void*  __libc_pvalloc(size_t);
  718 
  719 /*
  720   malloc_trim(size_t pad);
  721 
  722   If possible, gives memory back to the system (via negative
  723   arguments to sbrk) if there is unused memory at the `high' end of
  724   the malloc pool. You can call this after freeing large blocks of
  725   memory to potentially reduce the system-level memory requirements
  726   of a program. However, it cannot guarantee to reduce memory. Under
  727   some allocation patterns, some large free blocks of memory will be
  728   locked between two used chunks, so they cannot be given back to
  729   the system.
  730 
  731   The `pad' argument to malloc_trim represents the amount of free
  732   trailing space to leave untrimmed. If this argument is zero,
  733   only the minimum amount of memory to maintain internal data
  734   structures will be left (one page or less). Non-zero arguments
  735   can be supplied to maintain enough trailing space to service
  736   future expected allocations without having to re-obtain memory
  737   from the system.
  738 
  739   Malloc_trim returns 1 if it actually released any memory, else 0.
  740   On systems that do not support "negative sbrks", it will always
  741   return 0.
  742 */
  743 int      __malloc_trim(size_t);
  744 
  745 /*
  746   malloc_usable_size(void* p);
  747 
  748   Returns the number of bytes you can actually use in
  749   an allocated chunk, which may be more than you requested (although
  750   often not) due to alignment and minimum size constraints.
  751   You can use this many bytes without worrying about
  752   overwriting other allocated objects. This is not a particularly great
  753   programming practice. malloc_usable_size can be more useful in
  754   debugging and assertions, for example:
  755 
  756   p = malloc(n);
  757   assert(malloc_usable_size(p) >= 256);
  758 
  759 */
  760 size_t   __malloc_usable_size(void*);
  761 
  762 /*
  763   malloc_stats();
  764   Prints on stderr the amount of space obtained from the system (both
  765   via sbrk and mmap), the maximum amount (which may be more than
  766   current if malloc_trim and/or munmap got called), and the current
  767   number of bytes allocated via malloc (or realloc, etc) but not yet
  768   freed. Note that this is the number of bytes allocated, not the
  769   number requested. It will be larger than the number requested
  770   because of alignment and bookkeeping overhead. Because it includes
  771   alignment wastage as being in use, this figure may be greater than
  772   zero even when no user-level chunks are allocated.
  773 
  774   The reported current and maximum system memory can be inaccurate if
  775   a program makes other calls to system memory allocation functions
  776   (normally sbrk) outside of malloc.
  777 
  778   malloc_stats prints only the most commonly interesting statistics.
  779   More information can be obtained by calling mallinfo.
  780 
  781 */
  782 void     __malloc_stats(void);
  783 
  784 /*
  785   posix_memalign(void **memptr, size_t alignment, size_t size);
  786 
  787   POSIX wrapper like memalign(), checking for validity of size.
  788 */
  789 int      __posix_memalign(void **, size_t, size_t);
  790 #endif /* IS_IN (libc) */
  791 
  792 /*
  793   mallopt(int parameter_number, int parameter_value)
  794   Sets tunable parameters The format is to provide a
  795   (parameter-number, parameter-value) pair.  mallopt then sets the
  796   corresponding parameter to the argument value if it can (i.e., so
  797   long as the value is meaningful), and returns 1 if successful else
  798   0.  SVID/XPG/ANSI defines four standard param numbers for mallopt,
  799   normally defined in malloc.h.  Only one of these (M_MXFAST) is used
  800   in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
  801   so setting them has no effect. But this malloc also supports four
  802   other options in mallopt. See below for details.  Briefly, supported
  803   parameters are as follows (listed defaults are for "typical"
  804   configurations).
  805 
  806   Symbol            param #   default    allowed param values
  807   M_MXFAST          1         64         0-80  (0 disables fastbins)
  808   M_TRIM_THRESHOLD -1         128*1024   any   (-1U disables trimming)
  809   M_TOP_PAD        -2         0          any
  810   M_MMAP_THRESHOLD -3         128*1024   any   (or 0 if no MMAP support)
  811   M_MMAP_MAX       -4         65536      any   (0 disables use of mmap)
  812 */
  813 int      __libc_mallopt(int, int);
  814 #if IS_IN (libc)
  815 libc_hidden_proto (__libc_mallopt)
  816 #endif
  817 
  818 /* mallopt tuning options */
  819 
  820 /*
  821   M_MXFAST is the maximum request size used for "fastbins", special bins
  822   that hold returned chunks without consolidating their spaces. This
  823   enables future requests for chunks of the same size to be handled
  824   very quickly, but can increase fragmentation, and thus increase the
  825   overall memory footprint of a program.
  826 
  827   This malloc manages fastbins very conservatively yet still
  828   efficiently, so fragmentation is rarely a problem for values less
  829   than or equal to the default.  The maximum supported value of MXFAST
  830   is 80. You wouldn't want it any higher than this anyway.  Fastbins
  831   are designed especially for use with many small structs, objects or
  832   strings -- the default handles structs/objects/arrays with sizes up
  833   to 8 4byte fields, or small strings representing words, tokens,
  834   etc. Using fastbins for larger objects normally worsens
  835   fragmentation without improving speed.
  836 
  837   M_MXFAST is set in REQUEST size units. It is internally used in
  838   chunksize units, which adds padding and alignment.  You can reduce
  839   M_MXFAST to 0 to disable all use of fastbins.  This causes the malloc
  840   algorithm to be a closer approximation of fifo-best-fit in all cases,
  841   not just for larger requests, but will generally cause it to be
  842   slower.
  843 */
  844 
  845 
  846 /* M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h */
  847 #ifndef M_MXFAST
  848 #define M_MXFAST            1
  849 #endif
  850 
  851 #ifndef DEFAULT_MXFAST
  852 #define DEFAULT_MXFAST     (64 * SIZE_SZ / 4)
  853 #endif
  854 
  855 
  856 /*
  857   M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
  858   to keep before releasing via malloc_trim in free().
  859 
  860   Automatic trimming is mainly useful in long-lived programs.
  861   Because trimming via sbrk can be slow on some systems, and can
  862   sometimes be wasteful (in cases where programs immediately
  863   afterward allocate more large chunks) the value should be high
  864   enough so that your overall system performance would improve by
  865   releasing this much memory.
  866 
  867   The trim threshold and the mmap control parameters (see below)
  868   can be traded off with one another. Trimming and mmapping are
  869   two different ways of releasing unused memory back to the
  870   system. Between these two, it is often possible to keep
  871   system-level demands of a long-lived program down to a bare
  872   minimum. For example, in one test suite of sessions measuring
  873   the XF86 X server on Linux, using a trim threshold of 128K and a
  874   mmap threshold of 192K led to near-minimal long term resource
  875   consumption.
  876 
  877   If you are using this malloc in a long-lived program, it should
  878   pay to experiment with these values.  As a rough guide, you
  879   might set to a value close to the average size of a process
  880   (program) running on your system.  Releasing this much memory
  881   would allow such a process to run in memory.  Generally, it's
  882   worth it to tune for trimming rather tham memory mapping when a
  883   program undergoes phases where several large chunks are
  884   allocated and released in ways that can reuse each other's
  885   storage, perhaps mixed with phases where there are no such
  886   chunks at all.  And in well-behaved long-lived programs,
  887   controlling release of large blocks via trimming versus mapping
  888   is usually faster.
  889 
  890   However, in most programs, these parameters serve mainly as
  891   protection against the system-level effects of carrying around
  892   massive amounts of unneeded memory. Since frequent calls to
  893   sbrk, mmap, and munmap otherwise degrade performance, the default
  894   parameters are set to relatively high values that serve only as
  895   safeguards.
  896 
  897   The trim value It must be greater than page size to have any useful
  898   effect.  To disable trimming completely, you can set to
  899   (unsigned long)(-1)
  900 
  901   Trim settings interact with fastbin (MXFAST) settings: Unless
  902   TRIM_FASTBINS is defined, automatic trimming never takes place upon
  903   freeing a chunk with size less than or equal to MXFAST. Trimming is
  904   instead delayed until subsequent freeing of larger chunks. However,
  905   you can still force an attempted trim by calling malloc_trim.
  906 
  907   Also, trimming is not generally possible in cases where
  908   the main arena is obtained via mmap.
  909 
  910   Note that the trick some people use of mallocing a huge space and
  911   then freeing it at program startup, in an attempt to reserve system
  912   memory, doesn't have the intended effect under automatic trimming,
  913   since that memory will immediately be returned to the system.
  914 */
  915 
  916 #define M_TRIM_THRESHOLD       -1
  917 
  918 #ifndef DEFAULT_TRIM_THRESHOLD
  919 #define DEFAULT_TRIM_THRESHOLD (128 * 1024)
  920 #endif
  921 
  922 /*
  923   M_TOP_PAD is the amount of extra `padding' space to allocate or
  924   retain whenever sbrk is called. It is used in two ways internally:
  925 
  926   * When sbrk is called to extend the top of the arena to satisfy
  927   a new malloc request, this much padding is added to the sbrk
  928   request.
  929 
  930   * When malloc_trim is called automatically from free(),
  931   it is used as the `pad' argument.
  932 
  933   In both cases, the actual amount of padding is rounded
  934   so that the end of the arena is always a system page boundary.
  935 
  936   The main reason for using padding is to avoid calling sbrk so
  937   often. Having even a small pad greatly reduces the likelihood
  938   that nearly every malloc request during program start-up (or
  939   after trimming) will invoke sbrk, which needlessly wastes
  940   time.
  941 
  942   Automatic rounding-up to page-size units is normally sufficient
  943   to avoid measurable overhead, so the default is 0.  However, in
  944   systems where sbrk is relatively slow, it can pay to increase
  945   this value, at the expense of carrying around more memory than
  946   the program needs.
  947 */
  948 
  949 #define M_TOP_PAD              -2
  950 
  951 #ifndef DEFAULT_TOP_PAD
  952 #define DEFAULT_TOP_PAD        (0)
  953 #endif
  954 
  955 /*
  956   MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
  957   adjusted MMAP_THRESHOLD.
  958 */
  959 
  960 #ifndef DEFAULT_MMAP_THRESHOLD_MIN
  961 #define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
  962 #endif
  963 
  964 #ifndef DEFAULT_MMAP_THRESHOLD_MAX
  965   /* For 32-bit platforms we cannot increase the maximum mmap
  966      threshold much because it is also the minimum value for the
  967      maximum heap size and its alignment.  Going above 512k (i.e., 1M
  968      for new heaps) wastes too much address space.  */
  969 # if __WORDSIZE == 32
  970 #  define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
  971 # else
  972 #  define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
  973 # endif
  974 #endif
  975 
  976 /*
  977   M_MMAP_THRESHOLD is the request size threshold for using mmap()
  978   to service a request. Requests of at least this size that cannot
  979   be allocated using already-existing space will be serviced via mmap.
  980   (If enough normal freed space already exists it is used instead.)
  981 
  982   Using mmap segregates relatively large chunks of memory so that
  983   they can be individually obtained and released from the host
  984   system. A request serviced through mmap is never reused by any
  985   other request (at least not directly; the system may just so
  986   happen to remap successive requests to the same locations).
  987 
  988   Segregating space in this way has the benefits that:
  989 
  990    1. Mmapped space can ALWAYS be individually released back
  991       to the system, which helps keep the system level memory
  992       demands of a long-lived program low.
  993    2. Mapped memory can never become `locked' between
  994       other chunks, as can happen with normally allocated chunks, which
  995       means that even trimming via malloc_trim would not release them.
  996    3. On some systems with "holes" in address spaces, mmap can obtain
  997       memory that sbrk cannot.
  998 
  999   However, it has the disadvantages that:
 1000 
 1001    1. The space cannot be reclaimed, consolidated, and then
 1002       used to service later requests, as happens with normal chunks.
 1003    2. It can lead to more wastage because of mmap page alignment
 1004       requirements
 1005    3. It causes malloc performance to be more dependent on host
 1006       system memory management support routines which may vary in
 1007       implementation quality and may impose arbitrary
 1008       limitations. Generally, servicing a request via normal
 1009       malloc steps is faster than going through a system's mmap.
 1010 
 1011   The advantages of mmap nearly always outweigh disadvantages for
 1012   "large" chunks, but the value of "large" varies across systems.  The
 1013   default is an empirically derived value that works well in most
 1014   systems.
 1015 
 1016 
 1017   Update in 2006:
 1018   The above was written in 2001. Since then the world has changed a lot.
 1019   Memory got bigger. Applications got bigger. The virtual address space
 1020   layout in 32 bit linux changed.
 1021 
 1022   In the new situation, brk() and mmap space is shared and there are no
 1023   artificial limits on brk size imposed by the kernel. What is more,
 1024   applications have started using transient allocations larger than the
 1025   128Kb as was imagined in 2001.
 1026 
 1027   The price for mmap is also high now; each time glibc mmaps from the
 1028   kernel, the kernel is forced to zero out the memory it gives to the
 1029   application. Zeroing memory is expensive and eats a lot of cache and
 1030   memory bandwidth. This has nothing to do with the efficiency of the
 1031   virtual memory system, by doing mmap the kernel just has no choice but
 1032   to zero.
 1033 
 1034   In 2001, the kernel had a maximum size for brk() which was about 800
 1035   megabytes on 32 bit x86, at that point brk() would hit the first
 1036   mmaped shared libaries and couldn't expand anymore. With current 2.6
 1037   kernels, the VA space layout is different and brk() and mmap
 1038   both can span the entire heap at will.
 1039 
 1040   Rather than using a static threshold for the brk/mmap tradeoff,
 1041   we are now using a simple dynamic one. The goal is still to avoid
 1042   fragmentation. The old goals we kept are
 1043   1) try to get the long lived large allocations to use mmap()
 1044   2) really large allocations should always use mmap()
 1045   and we're adding now:
 1046   3) transient allocations should use brk() to avoid forcing the kernel
 1047      having to zero memory over and over again
 1048 
 1049   The implementation works with a sliding threshold, which is by default
 1050   limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
 1051   out at 128Kb as per the 2001 default.
 1052 
 1053   This allows us to satisfy requirement 1) under the assumption that long
 1054   lived allocations are made early in the process' lifespan, before it has
 1055   started doing dynamic allocations of the same size (which will
 1056   increase the threshold).
 1057 
 1058   The upperbound on the threshold satisfies requirement 2)
 1059 
 1060   The threshold goes up in value when the application frees memory that was
 1061   allocated with the mmap allocator. The idea is that once the application
 1062   starts freeing memory of a certain size, it's highly probable that this is
 1063   a size the application uses for transient allocations. This estimator
 1064   is there to satisfy the new third requirement.
 1065 
 1066 */
 1067 
 1068 #define M_MMAP_THRESHOLD      -3
 1069 
 1070 #ifndef DEFAULT_MMAP_THRESHOLD
 1071 #define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
 1072 #endif
 1073 
 1074 /*
 1075   M_MMAP_MAX is the maximum number of requests to simultaneously
 1076   service using mmap. This parameter exists because
 1077   some systems have a limited number of internal tables for
 1078   use by mmap, and using more than a few of them may degrade
 1079   performance.
 1080 
 1081   The default is set to a value that serves only as a safeguard.
 1082   Setting to 0 disables use of mmap for servicing large requests.
 1083 */
 1084 
 1085 #define M_MMAP_MAX             -4
 1086 
 1087 #ifndef DEFAULT_MMAP_MAX
 1088 #define DEFAULT_MMAP_MAX       (65536)
 1089 #endif
 1090 
 1091 #include <malloc.h>
 1092 
 1093 #ifndef RETURN_ADDRESS
 1094 #define RETURN_ADDRESS(X_) (NULL)
 1095 #endif
 1096 
 1097 /* Forward declarations.  */
 1098 struct malloc_chunk;
 1099 typedef struct malloc_chunk* mchunkptr;
 1100 
 1101 /* Internal routines.  */
 1102 
 1103 static void*  _int_malloc(mstate, size_t);
 1104 static void     _int_free(mstate, mchunkptr, int);
 1105 static void*  _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T,
 1106                INTERNAL_SIZE_T);
 1107 static void*  _int_memalign(mstate, size_t, size_t);
 1108 #if IS_IN (libc)
 1109 static void*  _mid_memalign(size_t, size_t, void *);
 1110 #endif
 1111 
 1112 static void malloc_printerr(const char *str) __attribute__ ((noreturn));
 1113 
 1114 static void munmap_chunk(mchunkptr p);
 1115 #if HAVE_MREMAP
 1116 static mchunkptr mremap_chunk(mchunkptr p, size_t new_size);
 1117 #endif
 1118 
 1119 /* ------------------ MMAP support ------------------  */
 1120 
 1121 
 1122 #include <fcntl.h>
 1123 #include <sys/mman.h>
 1124 
 1125 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
 1126 # define MAP_ANONYMOUS MAP_ANON
 1127 #endif
 1128 
 1129 #ifndef MAP_NORESERVE
 1130 # define MAP_NORESERVE 0
 1131 #endif
 1132 
 1133 #define MMAP(addr, size, prot, flags) \
 1134  __mmap((addr), (size), (prot), (flags)|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0)
 1135 
 1136 
 1137 /*
 1138   -----------------------  Chunk representations -----------------------
 1139 */
 1140 
 1141 
 1142 /*
 1143   This struct declaration is misleading (but accurate and necessary).
 1144   It declares a "view" into memory allowing access to necessary
 1145   fields at known offsets from a given base. See explanation below.
 1146 */
 1147 
 1148 struct malloc_chunk {
 1149 
 1150   INTERNAL_SIZE_T      mchunk_prev_size;  /* Size of previous chunk (if free).  */
 1151   INTERNAL_SIZE_T      mchunk_size;       /* Size in bytes, including overhead. */
 1152 
 1153   struct malloc_chunk* fd;         /* double links -- used only if free. */
 1154   struct malloc_chunk* bk;
 1155 
 1156   /* Only used for large blocks: pointer to next larger size.  */
 1157   struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
 1158   struct malloc_chunk* bk_nextsize;
 1159 };
 1160 
 1161 
 1162 /*
 1163    malloc_chunk details:
 1164 
 1165     (The following includes lightly edited explanations by Colin Plumb.)
 1166 
 1167     Chunks of memory are maintained using a `boundary tag' method as
 1168     described in e.g., Knuth or Standish.  (See the paper by Paul
 1169     Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
 1170     survey of such techniques.)  Sizes of free chunks are stored both
 1171     in the front of each chunk and at the end.  This makes
 1172     consolidating fragmented chunks into bigger chunks very fast.  The
 1173     size fields also hold bits representing whether chunks are free or
 1174     in use.
 1175 
 1176     An allocated chunk looks like this:
 1177 
 1178 
 1179     chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1180         |             Size of previous chunk, if unallocated (P clear)  |
 1181         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1182         |             Size of chunk, in bytes                     |A|M|P|
 1183       mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1184         |             User data starts here...                          .
 1185         .                                                               .
 1186         .             (malloc_usable_size() bytes)                      .
 1187         .                                                               |
 1188 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1189         |             (size of chunk, but used for application data)    |
 1190         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1191         |             Size of next chunk, in bytes                |A|0|1|
 1192         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1193 
 1194     Where "chunk" is the front of the chunk for the purpose of most of
 1195     the malloc code, but "mem" is the pointer that is returned to the
 1196     user.  "Nextchunk" is the beginning of the next contiguous chunk.
 1197 
 1198     Chunks always begin on even word boundaries, so the mem portion
 1199     (which is returned to the user) is also on an even word boundary, and
 1200     thus at least double-word aligned.
 1201 
 1202     Free chunks are stored in circular doubly-linked lists, and look like this:
 1203 
 1204     chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1205         |             Size of previous chunk, if unallocated (P clear)  |
 1206         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1207     `head:' |             Size of chunk, in bytes                     |A|0|P|
 1208       mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1209         |             Forward pointer to next chunk in list             |
 1210         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1211         |             Back pointer to previous chunk in list            |
 1212         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1213         |             Unused space (may be 0 bytes long)                .
 1214         .                                                               .
 1215         .                                                               |
 1216 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1217     `foot:' |             Size of chunk, in bytes                           |
 1218         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1219         |             Size of next chunk, in bytes                |A|0|0|
 1220         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1221 
 1222     The P (PREV_INUSE) bit, stored in the unused low-order bit of the
 1223     chunk size (which is always a multiple of two words), is an in-use
 1224     bit for the *previous* chunk.  If that bit is *clear*, then the
 1225     word before the current chunk size contains the previous chunk
 1226     size, and can be used to find the front of the previous chunk.
 1227     The very first chunk allocated always has this bit set,
 1228     preventing access to non-existent (or non-owned) memory. If
 1229     prev_inuse is set for any given chunk, then you CANNOT determine
 1230     the size of the previous chunk, and might even get a memory
 1231     addressing fault when trying to do so.
 1232 
 1233     The A (NON_MAIN_ARENA) bit is cleared for chunks on the initial,
 1234     main arena, described by the main_arena variable.  When additional
 1235     threads are spawned, each thread receives its own arena (up to a
 1236     configurable limit, after which arenas are reused for multiple
 1237     threads), and the chunks in these arenas have the A bit set.  To
 1238     find the arena for a chunk on such a non-main arena, heap_for_ptr
 1239     performs a bit mask operation and indirection through the ar_ptr
 1240     member of the per-heap header heap_info (see arena.c).
 1241 
 1242     Note that the `foot' of the current chunk is actually represented
 1243     as the prev_size of the NEXT chunk. This makes it easier to
 1244     deal with alignments etc but can be very confusing when trying
 1245     to extend or adapt this code.
 1246 
 1247     The three exceptions to all this are:
 1248 
 1249      1. The special chunk `top' doesn't bother using the
 1250     trailing size field since there is no next contiguous chunk
 1251     that would have to index off it. After initialization, `top'
 1252     is forced to always exist.  If it would become less than
 1253     MINSIZE bytes long, it is replenished.
 1254 
 1255      2. Chunks allocated via mmap, which have the second-lowest-order
 1256     bit M (IS_MMAPPED) set in their size fields.  Because they are
 1257     allocated one-by-one, each must contain its own trailing size
 1258     field.  If the M bit is set, the other bits are ignored
 1259     (because mmapped chunks are neither in an arena, nor adjacent
 1260     to a freed chunk).  The M bit is also used for chunks which
 1261     originally came from a dumped heap via malloc_set_state in
 1262     hooks.c.
 1263 
 1264      3. Chunks in fastbins are treated as allocated chunks from the
 1265     point of view of the chunk allocator.  They are consolidated
 1266     with their neighbors only in bulk, in malloc_consolidate.
 1267 */
 1268 
 1269 /*
 1270   ---------- Size and alignment checks and conversions ----------
 1271 */
 1272 
 1273 /* Conversion from malloc headers to user pointers, and back.  When
 1274    using memory tagging the user data and the malloc data structure
 1275    headers have distinct tags.  Converting fully from one to the other
 1276    involves extracting the tag at the other address and creating a
 1277    suitable pointer using it.  That can be quite expensive.  There are
 1278    cases when the pointers are not dereferenced (for example only used
 1279    for alignment check) so the tags are not relevant, and there are
 1280    cases when user data is not tagged distinctly from malloc headers
 1281    (user data is untagged because tagging is done late in malloc and
 1282    early in free).  User memory tagging across internal interfaces:
 1283 
 1284       sysmalloc: Returns untagged memory.
 1285       _int_malloc: Returns untagged memory.
 1286       _int_free: Takes untagged memory.
 1287       _int_memalign: Returns untagged memory.
 1288       _int_memalign: Returns untagged memory.
 1289       _mid_memalign: Returns tagged memory.
 1290       _int_realloc: Takes and returns tagged memory.
 1291 */
 1292 
 1293 /* The chunk header is two SIZE_SZ elements, but this is used widely, so
 1294    we define it here for clarity later.  */
 1295 #define CHUNK_HDR_SZ (2 * SIZE_SZ)
 1296 
 1297 /* Convert a chunk address to a user mem pointer without correcting
 1298    the tag.  */
 1299 #define chunk2mem(p) ((void*)((char*)(p) + CHUNK_HDR_SZ))
 1300 
 1301 /* Convert a chunk address to a user mem pointer and extract the right tag.  */
 1302 #define chunk2mem_tag(p) ((void*)tag_at ((char*)(p) + CHUNK_HDR_SZ))
 1303 
 1304 /* Convert a user mem pointer to a chunk address and extract the right tag.  */
 1305 #define mem2chunk(mem) ((mchunkptr)tag_at (((char*)(mem) - CHUNK_HDR_SZ)))
 1306 
 1307 /* The smallest possible chunk */
 1308 #define MIN_CHUNK_SIZE        (offsetof(struct malloc_chunk, fd_nextsize))
 1309 
 1310 /* The smallest size we can malloc is an aligned minimal chunk */
 1311 
 1312 #define MINSIZE  \
 1313   (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
 1314 
 1315 /* Check if m has acceptable alignment */
 1316 
 1317 #define aligned_OK(m)  (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
 1318 
 1319 #define misaligned_chunk(p) \
 1320   ((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \
 1321    & MALLOC_ALIGN_MASK)
 1322 
 1323 /* pad request bytes into a usable size -- internal version */
 1324 /* Note: This must be a macro that evaluates to a compile time constant
 1325    if passed a literal constant.  */
 1326 #define request2size(req)                                         \
 1327   (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE)  ?             \
 1328    MINSIZE :                                                      \
 1329    ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
 1330 
 1331 /* Check if REQ overflows when padded and aligned and if the resulting
 1332    value is less than PTRDIFF_T.  Returns the requested size or
 1333    MINSIZE in case the value is less than MINSIZE, or 0 if any of the
 1334    previous checks fail.  */
 1335 static inline size_t
 1336 checked_request2size (size_t req) __nonnull (1)
 1337 {
 1338   if (__glibc_unlikely (req > PTRDIFF_MAX))
 1339     return 0;
 1340 
 1341   /* When using tagged memory, we cannot share the end of the user
 1342      block with the header for the next chunk, so ensure that we
 1343      allocate blocks that are rounded up to the granule size.  Take
 1344      care not to overflow from close to MAX_SIZE_T to a small
 1345      number.  Ideally, this would be part of request2size(), but that
 1346      must be a macro that produces a compile time constant if passed
 1347      a constant literal.  */
 1348   if (__glibc_unlikely (mtag_enabled))
 1349     {
 1350       /* Ensure this is not evaluated if !mtag_enabled, see gcc PR 99551.  */
 1351       asm ("");
 1352 
 1353       req = (req + (__MTAG_GRANULE_SIZE - 1)) &
 1354         ~(size_t)(__MTAG_GRANULE_SIZE - 1);
 1355     }
 1356 
 1357   return request2size (req);
 1358 }
 1359 
 1360 /*
 1361    --------------- Physical chunk operations ---------------
 1362  */
 1363 
 1364 
 1365 /* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
 1366 #define PREV_INUSE 0x1
 1367 
 1368 /* extract inuse bit of previous chunk */
 1369 #define prev_inuse(p)       ((p)->mchunk_size & PREV_INUSE)
 1370 
 1371 
 1372 /* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
 1373 #define IS_MMAPPED 0x2
 1374 
 1375 /* check for mmap()'ed chunk */
 1376 #define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
 1377 
 1378 
 1379 /* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
 1380    from a non-main arena.  This is only set immediately before handing
 1381    the chunk to the user, if necessary.  */
 1382 #define NON_MAIN_ARENA 0x4
 1383 
 1384 /* Check for chunk from main arena.  */
 1385 #define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
 1386 
 1387 /* Mark a chunk as not being on the main arena.  */
 1388 #define set_non_main_arena(p) ((p)->mchunk_size |= NON_MAIN_ARENA)
 1389 
 1390 
 1391 /*
 1392    Bits to mask off when extracting size
 1393 
 1394    Note: IS_MMAPPED is intentionally not masked off from size field in
 1395    macros for which mmapped chunks should never be seen. This should
 1396    cause helpful core dumps to occur if it is tried by accident by
 1397    people extending or adapting this malloc.
 1398  */
 1399 #define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
 1400 
 1401 /* Get size, ignoring use bits */
 1402 #define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
 1403 
 1404 /* Like chunksize, but do not mask SIZE_BITS.  */
 1405 #define chunksize_nomask(p)         ((p)->mchunk_size)
 1406 
 1407 /* Ptr to next physical malloc_chunk. */
 1408 #define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
 1409 
 1410 /* Size of the chunk below P.  Only valid if !prev_inuse (P).  */
 1411 #define prev_size(p) ((p)->mchunk_prev_size)
 1412 
 1413 /* Set the size of the chunk below P.  Only valid if !prev_inuse (P).  */
 1414 #define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz))
 1415 
 1416 /* Ptr to previous physical malloc_chunk.  Only valid if !prev_inuse (P).  */
 1417 #define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p)))
 1418 
 1419 /* Treat space at ptr + offset as a chunk */
 1420 #define chunk_at_offset(p, s)  ((mchunkptr) (((char *) (p)) + (s)))
 1421 
 1422 /* extract p's inuse bit */
 1423 #define inuse(p)                                  \
 1424   ((((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size) & PREV_INUSE)
 1425 
 1426 /* set/clear chunk as being inuse without otherwise disturbing */
 1427 #define set_inuse(p)                                  \
 1428   ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size |= PREV_INUSE
 1429 
 1430 #define clear_inuse(p)                                \
 1431   ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size &= ~(PREV_INUSE)
 1432 
 1433 
 1434 /* check/set/clear inuse bits in known places */
 1435 #define inuse_bit_at_offset(p, s)                         \
 1436   (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE)
 1437 
 1438 #define set_inuse_bit_at_offset(p, s)                         \
 1439   (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size |= PREV_INUSE)
 1440 
 1441 #define clear_inuse_bit_at_offset(p, s)                       \
 1442   (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE))
 1443 
 1444 
 1445 /* Set size at head, without disturbing its use bit */
 1446 #define set_head_size(p, s)  ((p)->mchunk_size = (((p)->mchunk_size & SIZE_BITS) | (s)))
 1447 
 1448 /* Set size/use field */
 1449 #define set_head(p, s)       ((p)->mchunk_size = (s))
 1450 
 1451 /* Set size at footer (only when chunk is not in use) */
 1452 #define set_foot(p, s)       (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s))
 1453 
 1454 #pragma GCC poison mchunk_size
 1455 #pragma GCC poison mchunk_prev_size
 1456 
 1457 /* This is the size of the real usable data in the chunk.  Not valid for
 1458    dumped heap chunks.  */
 1459 #define memsize(p)                                                    \
 1460   (__MTAG_GRANULE_SIZE > SIZE_SZ && __glibc_unlikely (mtag_enabled) ? \
 1461     chunksize (p) - CHUNK_HDR_SZ :                                    \
 1462     chunksize (p) - CHUNK_HDR_SZ + (chunk_is_mmapped (p) ? 0 : SIZE_SZ))
 1463 
 1464 /* If memory tagging is enabled the layout changes to accommodate the granule
 1465    size, this is wasteful for small allocations so not done by default.
 1466    Both the chunk header and user data has to be granule aligned.  */
 1467 _Static_assert (__MTAG_GRANULE_SIZE <= CHUNK_HDR_SZ,
 1468         "memory tagging is not supported with large granule.");
 1469 
 1470 static __always_inline void *
 1471 tag_new_usable (void *ptr)
 1472 {
 1473   if (__glibc_unlikely (mtag_enabled) && ptr)
 1474     {
 1475       mchunkptr cp = mem2chunk(ptr);
 1476       ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), memsize (cp));
 1477     }
 1478   return ptr;
 1479 }
 1480 
 1481 /*
 1482    -------------------- Internal data structures --------------------
 1483 
 1484    All internal state is held in an instance of malloc_state defined
 1485    below. There are no other static variables, except in two optional
 1486    cases:
 1487  * If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
 1488  * If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
 1489      for mmap.
 1490 
 1491    Beware of lots of tricks that minimize the total bookkeeping space
 1492    requirements. The result is a little over 1K bytes (for 4byte
 1493    pointers and size_t.)
 1494  */
 1495 
 1496 /*
 1497    Bins
 1498 
 1499     An array of bin headers for free chunks. Each bin is doubly
 1500     linked.  The bins are approximately proportionally (log) spaced.
 1501     There are a lot of these bins (128). This may look excessive, but
 1502     works very well in practice.  Most bins hold sizes that are
 1503     unusual as malloc request sizes, but are more usual for fragments
 1504     and consolidated sets of chunks, which is what these bins hold, so
 1505     they can be found quickly.  All procedures maintain the invariant
 1506     that no consolidated chunk physically borders another one, so each
 1507     chunk in a list is known to be preceeded and followed by either
 1508     inuse chunks or the ends of memory.
 1509 
 1510     Chunks in bins are kept in size order, with ties going to the
 1511     approximately least recently used chunk. Ordering isn't needed
 1512     for the small bins, which all contain the same-sized chunks, but
 1513     facilitates best-fit allocation for larger chunks. These lists
 1514     are just sequential. Keeping them in order almost never requires
 1515     enough traversal to warrant using fancier ordered data
 1516     structures.
 1517 
 1518     Chunks of the same size are linked with the most
 1519     recently freed at the front, and allocations are taken from the
 1520     back.  This results in LRU (FIFO) allocation order, which tends
 1521     to give each chunk an equal opportunity to be consolidated with
 1522     adjacent freed chunks, resulting in larger free chunks and less
 1523     fragmentation.
 1524 
 1525     To simplify use in double-linked lists, each bin header acts
 1526     as a malloc_chunk. This avoids special-casing for headers.
 1527     But to conserve space and improve locality, we allocate
 1528     only the fd/bk pointers of bins, and then use repositioning tricks
 1529     to treat these as the fields of a malloc_chunk*.
 1530  */
 1531 
 1532 typedef struct malloc_chunk *mbinptr;
 1533 
 1534 /* addressing -- note that bin_at(0) does not exist */
 1535 #define bin_at(m, i) \
 1536   (mbinptr) (((char *) &((m)->bins[((i) - 1) * 2]))               \
 1537              - offsetof (struct malloc_chunk, fd))
 1538 
 1539 /* analog of ++bin */
 1540 #define next_bin(b)  ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1)))
 1541 
 1542 /* Reminders about list directionality within bins */
 1543 #define first(b)     ((b)->fd)
 1544 #define last(b)      ((b)->bk)
 1545 
 1546 /*
 1547    Indexing
 1548 
 1549     Bins for sizes < 512 bytes contain chunks of all the same size, spaced
 1550     8 bytes apart. Larger bins are approximately logarithmically spaced:
 1551 
 1552     64 bins of size       8
 1553     32 bins of size      64
 1554     16 bins of size     512
 1555      8 bins of size    4096
 1556      4 bins of size   32768
 1557      2 bins of size  262144
 1558      1 bin  of size what's left
 1559 
 1560     There is actually a little bit of slop in the numbers in bin_index
 1561     for the sake of speed. This makes no difference elsewhere.
 1562 
 1563     The bins top out around 1MB because we expect to service large
 1564     requests via mmap.
 1565 
 1566     Bin 0 does not exist.  Bin 1 is the unordered list; if that would be
 1567     a valid chunk size the small bins are bumped up one.
 1568  */
 1569 
 1570 #define NBINS             128
 1571 #define NSMALLBINS         64
 1572 #define SMALLBIN_WIDTH    MALLOC_ALIGNMENT
 1573 #define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ)
 1574 #define MIN_LARGE_SIZE    ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
 1575 
 1576 #define in_smallbin_range(sz)  \
 1577   ((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE)
 1578 
 1579 #define smallbin_index(sz) \
 1580   ((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\
 1581    + SMALLBIN_CORRECTION)
 1582 
 1583 #define largebin_index_32(sz)                                                \
 1584   (((((unsigned long) (sz)) >> 6) <= 38) ?  56 + (((unsigned long) (sz)) >> 6) :\
 1585    ((((unsigned long) (sz)) >> 9) <= 20) ?  91 + (((unsigned long) (sz)) >> 9) :\
 1586    ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
 1587    ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
 1588    ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
 1589    126)
 1590 
 1591 #define largebin_index_32_big(sz)                                            \
 1592   (((((unsigned long) (sz)) >> 6) <= 45) ?  49 + (((unsigned long) (sz)) >> 6) :\
 1593    ((((unsigned long) (sz)) >> 9) <= 20) ?  91 + (((unsigned long) (sz)) >> 9) :\
 1594    ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
 1595    ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
 1596    ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
 1597    126)
 1598 
 1599 // XXX It remains to be seen whether it is good to keep the widths of
 1600 // XXX the buckets the same or whether it should be scaled by a factor
 1601 // XXX of two as well.
 1602 #define largebin_index_64(sz)                                                \
 1603   (((((unsigned long) (sz)) >> 6) <= 48) ?  48 + (((unsigned long) (sz)) >> 6) :\
 1604    ((((unsigned long) (sz)) >> 9) <= 20) ?  91 + (((unsigned long) (sz)) >> 9) :\
 1605    ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
 1606    ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
 1607    ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
 1608    126)
 1609 
 1610 #define largebin_index(sz) \
 1611   (SIZE_SZ == 8 ? largebin_index_64 (sz)                                     \
 1612    : MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz)                     \
 1613    : largebin_index_32 (sz))
 1614 
 1615 #define bin_index(sz) \
 1616   ((in_smallbin_range (sz)) ? smallbin_index (sz) : largebin_index (sz))
 1617 
 1618 /* Take a chunk off a bin list.  */
 1619 static void
 1620 unlink_chunk (mstate av, mchunkptr p)
 1621 {
 1622   if (chunksize (p) != prev_size (next_chunk (p)))
 1623     malloc_printerr ("corrupted size vs. prev_size");
 1624 
 1625   mchunkptr fd = p->fd;
 1626   mchunkptr bk = p->bk;
 1627 
 1628   if (__builtin_expect (fd->bk != p || bk->fd != p, 0))
 1629     malloc_printerr ("corrupted double-linked list");
 1630 
 1631   fd->bk = bk;
 1632   bk->fd = fd;
 1633   if (!in_smallbin_range (chunksize_nomask (p)) && p->fd_nextsize != NULL)
 1634     {
 1635       if (p->fd_nextsize->bk_nextsize != p
 1636       || p->bk_nextsize->fd_nextsize != p)
 1637     malloc_printerr ("corrupted double-linked list (not small)");
 1638 
 1639       if (fd->fd_nextsize == NULL)
 1640     {
 1641       if (p->fd_nextsize == p)
 1642         fd->fd_nextsize = fd->bk_nextsize = fd;
 1643       else
 1644         {
 1645           fd->fd_nextsize = p->fd_nextsize;
 1646           fd->bk_nextsize = p->bk_nextsize;
 1647           p->fd_nextsize->bk_nextsize = fd;
 1648           p->bk_nextsize->fd_nextsize = fd;
 1649         }
 1650     }
 1651       else
 1652     {
 1653       p->fd_nextsize->bk_nextsize = p->bk_nextsize;
 1654       p->bk_nextsize->fd_nextsize = p->fd_nextsize;
 1655     }
 1656     }
 1657 }
 1658 
 1659 /*
 1660    Unsorted chunks
 1661 
 1662     All remainders from chunk splits, as well as all returned chunks,
 1663     are first placed in the "unsorted" bin. They are then placed
 1664     in regular bins after malloc gives them ONE chance to be used before
 1665     binning. So, basically, the unsorted_chunks list acts as a queue,
 1666     with chunks being placed on it in free (and malloc_consolidate),
 1667     and taken off (to be either used or placed in bins) in malloc.
 1668 
 1669     The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
 1670     does not have to be taken into account in size comparisons.
 1671  */
 1672 
 1673 /* The otherwise unindexable 1-bin is used to hold unsorted chunks. */
 1674 #define unsorted_chunks(M)          (bin_at (M, 1))
 1675 
 1676 /*
 1677    Top
 1678 
 1679     The top-most available chunk (i.e., the one bordering the end of
 1680     available memory) is treated specially. It is never included in
 1681     any bin, is used only if no other chunk is available, and is
 1682     released back to the system if it is very large (see
 1683     M_TRIM_THRESHOLD).  Because top initially
 1684     points to its own bin with initial zero size, thus forcing
 1685     extension on the first malloc request, we avoid having any special
 1686     code in malloc to check whether it even exists yet. But we still
 1687     need to do so when getting memory from system, so we make
 1688     initial_top treat the bin as a legal but unusable chunk during the
 1689     interval between initialization and the first call to
 1690     sysmalloc. (This is somewhat delicate, since it relies on
 1691     the 2 preceding words to be zero during this interval as well.)
 1692  */
 1693 
 1694 /* Conveniently, the unsorted bin can be used as dummy top on first call */
 1695 #define initial_top(M)              (unsorted_chunks (M))
 1696 
 1697 /*
 1698    Binmap
 1699 
 1700     To help compensate for the large number of bins, a one-level index
 1701     structure is used for bin-by-bin searching.  `binmap' is a
 1702     bitvector recording whether bins are definitely empty so they can
 1703     be skipped over during during traversals.  The bits are NOT always
 1704     cleared as soon as bins are empty, but instead only
 1705     when they are noticed to be empty during traversal in malloc.
 1706  */
 1707 
 1708 /* Conservatively use 32 bits per map word, even if on 64bit system */
 1709 #define BINMAPSHIFT      5
 1710 #define BITSPERMAP       (1U << BINMAPSHIFT)
 1711 #define BINMAPSIZE       (NBINS / BITSPERMAP)
 1712 
 1713 #define idx2block(i)     ((i) >> BINMAPSHIFT)
 1714 #define idx2bit(i)       ((1U << ((i) & ((1U << BINMAPSHIFT) - 1))))
 1715 
 1716 #define mark_bin(m, i)    ((m)->binmap[idx2block (i)] |= idx2bit (i))
 1717 #define unmark_bin(m, i)  ((m)->binmap[idx2block (i)] &= ~(idx2bit (i)))
 1718 #define get_binmap(m, i)  ((m)->binmap[idx2block (i)] & idx2bit (i))
 1719 
 1720 /*
 1721    Fastbins
 1722 
 1723     An array of lists holding recently freed small chunks.  Fastbins
 1724     are not doubly linked.  It is faster to single-link them, and
 1725     since chunks are never removed from the middles of these lists,
 1726     double linking is not necessary. Also, unlike regular bins, they
 1727     are not even processed in FIFO order (they use faster LIFO) since
 1728     ordering doesn't much matter in the transient contexts in which
 1729     fastbins are normally used.
 1730 
 1731     Chunks in fastbins keep their inuse bit set, so they cannot
 1732     be consolidated with other free chunks. malloc_consolidate
 1733     releases all chunks in fastbins and consolidates them with
 1734     other free chunks.
 1735  */
 1736 
 1737 typedef struct malloc_chunk *mfastbinptr;
 1738 #define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
 1739 
 1740 /* offset 2 to use otherwise unindexable first 2 bins */
 1741 #define fastbin_index(sz) \
 1742   ((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
 1743 
 1744 
 1745 /* The maximum fastbin request size we support */
 1746 #define MAX_FAST_SIZE     (80 * SIZE_SZ / 4)
 1747 
 1748 #define NFASTBINS  (fastbin_index (request2size (MAX_FAST_SIZE)) + 1)
 1749 
 1750 /*
 1751    FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
 1752    that triggers automatic consolidation of possibly-surrounding
 1753    fastbin chunks. This is a heuristic, so the exact value should not
 1754    matter too much. It is defined at half the default trim threshold as a
 1755    compromise heuristic to only attempt consolidation if it is likely
 1756    to lead to trimming. However, it is not dynamically tunable, since
 1757    consolidation reduces fragmentation surrounding large chunks even
 1758    if trimming is not used.
 1759  */
 1760 
 1761 #define FASTBIN_CONSOLIDATION_THRESHOLD  (65536UL)
 1762 
 1763 /*
 1764    NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
 1765    regions.  Otherwise, contiguity is exploited in merging together,
 1766    when possible, results from consecutive MORECORE calls.
 1767 
 1768    The initial value comes from MORECORE_CONTIGUOUS, but is
 1769    changed dynamically if mmap is ever used as an sbrk substitute.
 1770  */
 1771 
 1772 #define NONCONTIGUOUS_BIT     (2U)
 1773 
 1774 #define contiguous(M)          (((M)->flags & NONCONTIGUOUS_BIT) == 0)
 1775 #define noncontiguous(M)       (((M)->flags & NONCONTIGUOUS_BIT) != 0)
 1776 #define set_noncontiguous(M)   ((M)->flags |= NONCONTIGUOUS_BIT)
 1777 #define set_contiguous(M)      ((M)->flags &= ~NONCONTIGUOUS_BIT)
 1778 
 1779 /* Maximum size of memory handled in fastbins.  */
 1780 static INTERNAL_SIZE_T global_max_fast;
 1781 
 1782 /*
 1783    Set value of max_fast.
 1784    Use impossibly small value if 0.
 1785    Precondition: there are no existing fastbin chunks in the main arena.
 1786    Since do_check_malloc_state () checks this, we call malloc_consolidate ()
 1787    before changing max_fast.  Note other arenas will leak their fast bin
 1788    entries if max_fast is reduced.
 1789  */
 1790 
 1791 #define set_max_fast(s) \
 1792   global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ)  \
 1793                      ? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
 1794 
 1795 static inline INTERNAL_SIZE_T
 1796 get_max_fast (void)
 1797 {
 1798   /* Tell the GCC optimizers that global_max_fast is never larger
 1799      than MAX_FAST_SIZE.  This avoids out-of-bounds array accesses in
 1800      _int_malloc after constant propagation of the size parameter.
 1801      (The code never executes because malloc preserves the
 1802      global_max_fast invariant, but the optimizers may not recognize
 1803      this.)  */
 1804   if (global_max_fast > MAX_FAST_SIZE)
 1805     __builtin_unreachable ();
 1806   return global_max_fast;
 1807 }
 1808 
 1809 /*
 1810    ----------- Internal state representation and initialization -----------
 1811  */
 1812 
 1813 /*
 1814    have_fastchunks indicates that there are probably some fastbin chunks.
 1815    It is set true on entering a chunk into any fastbin, and cleared early in
 1816    malloc_consolidate.  The value is approximate since it may be set when there
 1817    are no fastbin chunks, or it may be clear even if there are fastbin chunks
 1818    available.  Given it's sole purpose is to reduce number of redundant calls to
 1819    malloc_consolidate, it does not affect correctness.  As a result we can safely
 1820    use relaxed atomic accesses.
 1821  */
 1822 
 1823 
 1824 struct malloc_state
 1825 {
 1826   /* Serialize access.  */
 1827   __libc_lock_define (, mutex);
 1828 
 1829   /* Flags (formerly in max_fast).  */
 1830   int flags;
 1831 
 1832   /* Set if the fastbin chunks contain recently inserted free blocks.  */
 1833   /* Note this is a bool but not all targets support atomics on booleans.  */
 1834   int have_fastchunks;
 1835 
 1836   /* Fastbins */
 1837   mfastbinptr fastbinsY[NFASTBINS];
 1838 
 1839   /* Base of the topmost chunk -- not otherwise kept in a bin */
 1840   mchunkptr top;
 1841 
 1842   /* The remainder from the most recent split of a small request */
 1843   mchunkptr last_remainder;
 1844 
 1845   /* Normal bins packed as described above */
 1846   mchunkptr bins[NBINS * 2 - 2];
 1847 
 1848   /* Bitmap of bins */
 1849   unsigned int binmap[BINMAPSIZE];
 1850 
 1851   /* Linked list */
 1852   struct malloc_state *next;
 1853 
 1854   /* Linked list for free arenas.  Access to this field is serialized
 1855      by free_list_lock in arena.c.  */
 1856   struct malloc_state *next_free;
 1857 
 1858   /* Number of threads attached to this arena.  0 if the arena is on
 1859      the free list.  Access to this field is serialized by
 1860      free_list_lock in arena.c.  */
 1861   INTERNAL_SIZE_T attached_threads;
 1862 
 1863   /* Memory allocated from the system in this arena.  */
 1864   INTERNAL_SIZE_T system_mem;
 1865   INTERNAL_SIZE_T max_system_mem;
 1866 };
 1867 
 1868 struct malloc_par
 1869 {
 1870   /* Tunable parameters */
 1871   unsigned long trim_threshold;
 1872   INTERNAL_SIZE_T top_pad;
 1873   INTERNAL_SIZE_T mmap_threshold;
 1874   INTERNAL_SIZE_T arena_test;
 1875   INTERNAL_SIZE_T arena_max;
 1876 
 1877 #if HAVE_TUNABLES
 1878   /* Transparent Large Page support.  */
 1879   INTERNAL_SIZE_T thp_pagesize;
 1880   /* A value different than 0 means to align mmap allocation to hp_pagesize
 1881      add hp_flags on flags.  */
 1882   INTERNAL_SIZE_T hp_pagesize;
 1883   int hp_flags;
 1884 #endif
 1885 
 1886   /* Memory map support */
 1887   int n_mmaps;
 1888   int n_mmaps_max;
 1889   int max_n_mmaps;
 1890   /* the mmap_threshold is dynamic, until the user sets
 1891      it manually, at which point we need to disable any
 1892      dynamic behavior. */
 1893   int no_dyn_threshold;
 1894 
 1895   /* Statistics */
 1896   INTERNAL_SIZE_T mmapped_mem;
 1897   INTERNAL_SIZE_T max_mmapped_mem;
 1898 
 1899   /* First address handed out by MORECORE/sbrk.  */
 1900   char *sbrk_base;
 1901 
 1902 #if USE_TCACHE
 1903   /* Maximum number of buckets to use.  */
 1904   size_t tcache_bins;
 1905   size_t tcache_max_bytes;
 1906   /* Maximum number of chunks in each bucket.  */
 1907   size_t tcache_count;
 1908   /* Maximum number of chunks to remove from the unsorted list, which
 1909      aren't used to prefill the cache.  */
 1910   size_t tcache_unsorted_limit;
 1911 #endif
 1912 };
 1913 
 1914 /* There are several instances of this struct ("arenas") in this
 1915    malloc.  If you are adapting this malloc in a way that does NOT use
 1916    a static or mmapped malloc_state, you MUST explicitly zero-fill it
 1917    before using. This malloc relies on the property that malloc_state
 1918    is initialized to all zeroes (as is true of C statics).  */
 1919 
 1920 static struct malloc_state main_arena =
 1921 {
 1922   .mutex = _LIBC_LOCK_INITIALIZER,
 1923   .next = &main_arena,
 1924   .attached_threads = 1
 1925 };
 1926 
 1927 /* There is only one instance of the malloc parameters.  */
 1928 
 1929 static struct malloc_par mp_ =
 1930 {
 1931   .top_pad = DEFAULT_TOP_PAD,
 1932   .n_mmaps_max = DEFAULT_MMAP_MAX,
 1933   .mmap_threshold = DEFAULT_MMAP_THRESHOLD,
 1934   .trim_threshold = DEFAULT_TRIM_THRESHOLD,
 1935 #define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8))
 1936   .arena_test = NARENAS_FROM_NCORES (1)
 1937 #if USE_TCACHE
 1938   ,
 1939   .tcache_count = TCACHE_FILL_COUNT,
 1940   .tcache_bins = TCACHE_MAX_BINS,
 1941   .tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-1),
 1942   .tcache_unsorted_limit = 0 /* No limit.  */
 1943 #endif
 1944 };
 1945 
 1946 /*
 1947    Initialize a malloc_state struct.
 1948 
 1949    This is called from ptmalloc_init () or from _int_new_arena ()
 1950    when creating a new arena.
 1951  */
 1952 
 1953 static void
 1954 malloc_init_state (mstate av)
 1955 {
 1956   int i;
 1957   mbinptr bin;
 1958 
 1959   /* Establish circular links for normal bins */
 1960   for (i = 1; i < NBINS; ++i)
 1961     {
 1962       bin = bin_at (av, i);
 1963       bin->fd = bin->bk = bin;
 1964     }
 1965 
 1966 #if MORECORE_CONTIGUOUS
 1967   if (av != &main_arena)
 1968 #endif
 1969   set_noncontiguous (av);
 1970   if (av == &main_arena)
 1971     set_max_fast (DEFAULT_MXFAST);
 1972   atomic_store_relaxed (&av->have_fastchunks, false);
 1973 
 1974   av->top = initial_top (av);
 1975 }
 1976 
 1977 /*
 1978    Other internal utilities operating on mstates
 1979  */
 1980 
 1981 static void *sysmalloc (INTERNAL_SIZE_T, mstate);
 1982 static int      systrim (size_t, mstate);
 1983 static void     malloc_consolidate (mstate);
 1984 
 1985 
 1986 /* -------------- Early definitions for debugging hooks ---------------- */
 1987 
 1988 /* This function is called from the arena shutdown hook, to free the
 1989    thread cache (if it exists).  */
 1990 static void tcache_thread_shutdown (void);
 1991 
 1992 /* ------------------ Testing support ----------------------------------*/
 1993 
 1994 static int perturb_byte;
 1995 
 1996 static void
 1997 alloc_perturb (char *p, size_t n)
 1998 {
 1999   if (__glibc_unlikely (perturb_byte))
 2000     memset (p, perturb_byte ^ 0xff, n);
 2001 }
 2002 
 2003 static void
 2004 free_perturb (char *p, size_t n)
 2005 {
 2006   if (__glibc_unlikely (perturb_byte))
 2007     memset (p, perturb_byte, n);
 2008 }
 2009 
 2010 
 2011 
 2012 #include <stap-probe.h>
 2013 
 2014 /* ----------- Routines dealing with transparent huge pages ----------- */
 2015 
 2016 static inline void
 2017 madvise_thp (void *p, INTERNAL_SIZE_T size)
 2018 {
 2019 #if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
 2020   /* Do not consider areas smaller than a huge page or if the tunable is
 2021      not active.  */
 2022   if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
 2023     return;
 2024 
 2025   /* Linux requires the input address to be page-aligned, and unaligned
 2026      inputs happens only for initial data segment.  */
 2027   if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
 2028     {
 2029       void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
 2030       size += PTR_DIFF (p, q);
 2031       p = q;
 2032     }
 2033 
 2034   __madvise (p, size, MADV_HUGEPAGE);
 2035 #endif
 2036 }
 2037 
 2038 /* ------------------- Support for multiple arenas -------------------- */
 2039 #include "arena.c"
 2040 
 2041 /*
 2042    Debugging support
 2043 
 2044    These routines make a number of assertions about the states
 2045    of data structures that should be true at all times. If any
 2046    are not true, it's very likely that a user program has somehow
 2047    trashed memory. (It's also possible that there is a coding error
 2048    in malloc. In which case, please report it!)
 2049  */
 2050 
 2051 #if !MALLOC_DEBUG
 2052 
 2053 # define check_chunk(A, P)
 2054 # define check_free_chunk(A, P)
 2055 # define check_inuse_chunk(A, P)
 2056 # define check_remalloced_chunk(A, P, N)
 2057 # define check_malloced_chunk(A, P, N)
 2058 # define check_malloc_state(A)
 2059 
 2060 #else
 2061 
 2062 # define check_chunk(A, P)              do_check_chunk (A, P)
 2063 # define check_free_chunk(A, P)         do_check_free_chunk (A, P)
 2064 # define check_inuse_chunk(A, P)        do_check_inuse_chunk (A, P)
 2065 # define check_remalloced_chunk(A, P, N) do_check_remalloced_chunk (A, P, N)
 2066 # define check_malloced_chunk(A, P, N)   do_check_malloced_chunk (A, P, N)
 2067 # define check_malloc_state(A)         do_check_malloc_state (A)
 2068 
 2069 /*
 2070    Properties of all chunks
 2071  */
 2072 
 2073 static void
 2074 do_check_chunk (mstate av, mchunkptr p)
 2075 {
 2076   unsigned long sz = chunksize (p);
 2077   /* min and max possible addresses assuming contiguous allocation */
 2078   char *max_address = (char *) (av->top) + chunksize (av->top);
 2079   char *min_address = max_address - av->system_mem;
 2080 
 2081   if (!chunk_is_mmapped (p))
 2082     {
 2083       /* Has legal address ... */
 2084       if (p != av->top)
 2085         {
 2086           if (contiguous (av))
 2087             {
 2088               assert (((char *) p) >= min_address);
 2089               assert (((char *) p + sz) <= ((char *) (av->top)));
 2090             }
 2091         }
 2092       else
 2093         {
 2094           /* top size is always at least MINSIZE */
 2095           assert ((unsigned long) (sz) >= MINSIZE);
 2096           /* top predecessor always marked inuse */
 2097           assert (prev_inuse (p));
 2098         }
 2099     }
 2100   else
 2101     {
 2102       /* address is outside main heap  */
 2103       if (contiguous (av) && av->top != initial_top (av))
 2104         {
 2105           assert (((char *) p) < min_address || ((char *) p) >= max_address);
 2106         }
 2107       /* chunk is page-aligned */
 2108       assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - 1)) == 0);
 2109       /* mem is aligned */
 2110       assert (aligned_OK (chunk2mem (p)));
 2111     }
 2112 }
 2113 
 2114 /*
 2115    Properties of free chunks
 2116  */
 2117 
 2118 static void
 2119 do_check_free_chunk (mstate av, mchunkptr p)
 2120 {
 2121   INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE | NON_MAIN_ARENA);
 2122   mchunkptr next = chunk_at_offset (p, sz);
 2123 
 2124   do_check_chunk (av, p);
 2125 
 2126   /* Chunk must claim to be free ... */
 2127   assert (!inuse (p));
 2128   assert (!chunk_is_mmapped (p));
 2129 
 2130   /* Unless a special marker, must have OK fields */
 2131   if ((unsigned long) (sz) >= MINSIZE)
 2132     {
 2133       assert ((sz & MALLOC_ALIGN_MASK) == 0);
 2134       assert (aligned_OK (chunk2mem (p)));
 2135       /* ... matching footer field */
 2136       assert (prev_size (next_chunk (p)) == sz);
 2137       /* ... and is fully consolidated */
 2138       assert (prev_inuse (p));
 2139       assert (next == av->top || inuse (next));
 2140 
 2141       /* ... and has minimally sane links */
 2142       assert (p->fd->bk == p);
 2143       assert (p->bk->fd == p);
 2144     }
 2145   else /* markers are always of size SIZE_SZ */
 2146     assert (sz == SIZE_SZ);
 2147 }
 2148 
 2149 /*
 2150    Properties of inuse chunks
 2151  */
 2152 
 2153 static void
 2154 do_check_inuse_chunk (mstate av, mchunkptr p)
 2155 {
 2156   mchunkptr next;
 2157 
 2158   do_check_chunk (av, p);
 2159 
 2160   if (chunk_is_mmapped (p))
 2161     return; /* mmapped chunks have no next/prev */
 2162 
 2163   /* Check whether it claims to be in use ... */
 2164   assert (inuse (p));
 2165 
 2166   next = next_chunk (p);
 2167 
 2168   /* ... and is surrounded by OK chunks.
 2169      Since more things can be checked with free chunks than inuse ones,
 2170      if an inuse chunk borders them and debug is on, it's worth doing them.
 2171    */
 2172   if (!prev_inuse (p))
 2173     {
 2174       /* Note that we cannot even look at prev unless it is not inuse */
 2175       mchunkptr prv = prev_chunk (p);
 2176       assert (next_chunk (prv) == p);
 2177       do_check_free_chunk (av, prv);
 2178     }
 2179 
 2180   if (next == av->top)
 2181     {
 2182       assert (prev_inuse (next));
 2183       assert (chunksize (next) >= MINSIZE);
 2184     }
 2185   else if (!inuse (next))
 2186     do_check_free_chunk (av, next);
 2187 }
 2188 
 2189 /*
 2190    Properties of chunks recycled from fastbins
 2191  */
 2192 
 2193 static void
 2194 do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
 2195 {
 2196   INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE | NON_MAIN_ARENA);
 2197 
 2198   if (!chunk_is_mmapped (p))
 2199     {
 2200       assert (av == arena_for_chunk (p));
 2201       if (chunk_main_arena (p))
 2202         assert (av == &main_arena);
 2203       else
 2204         assert (av != &main_arena);
 2205     }
 2206 
 2207   do_check_inuse_chunk (av, p);
 2208 
 2209   /* Legal size ... */
 2210   assert ((sz & MALLOC_ALIGN_MASK) == 0);
 2211   assert ((unsigned long) (sz) >= MINSIZE);
 2212   /* ... and alignment */
 2213   assert (aligned_OK (chunk2mem (p)));
 2214   /* chunk is less than MINSIZE more than request */
 2215   assert ((long) (sz) - (long) (s) >= 0);
 2216   assert ((long) (sz) - (long) (s + MINSIZE) < 0);
 2217 }
 2218 
 2219 /*
 2220    Properties of nonrecycled chunks at the point they are malloced
 2221  */
 2222 
 2223 static void
 2224 do_check_malloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
 2225 {
 2226   /* same as recycled case ... */
 2227   do_check_remalloced_chunk (av, p, s);
 2228 
 2229   /*
 2230      ... plus,  must obey implementation invariant that prev_inuse is
 2231      always true of any allocated chunk; i.e., that each allocated
 2232      chunk borders either a previously allocated and still in-use
 2233      chunk, or the base of its memory arena. This is ensured
 2234      by making all allocations from the `lowest' part of any found
 2235      chunk.  This does not necessarily hold however for chunks
 2236      recycled via fastbins.
 2237    */
 2238 
 2239   assert (prev_inuse (p));
 2240 }
 2241 
 2242 
 2243 /*
 2244    Properties of malloc_state.
 2245 
 2246    This may be useful for debugging malloc, as well as detecting user
 2247    programmer errors that somehow write into malloc_state.
 2248 
 2249    If you are extending or experimenting with this malloc, you can
 2250    probably figure out how to hack this routine to print out or
 2251    display chunk addresses, sizes, bins, and other instrumentation.
 2252  */
 2253 
 2254 static void
 2255 do_check_malloc_state (mstate av)
 2256 {
 2257   int i;
 2258   mchunkptr p;
 2259   mchunkptr q;
 2260   mbinptr b;
 2261   unsigned int idx;
 2262   INTERNAL_SIZE_T size;
 2263   unsigned long total = 0;
 2264   int max_fast_bin;
 2265 
 2266   /* internal size_t must be no wider than pointer type */
 2267   assert (sizeof (INTERNAL_SIZE_T) <= sizeof (char *));
 2268 
 2269   /* alignment is a power of 2 */
 2270   assert ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT - 1)) == 0);
 2271 
 2272   /* Check the arena is initialized. */
 2273   assert (av->top != 0);
 2274 
 2275   /* No memory has been allocated yet, so doing more tests is not possible.  */
 2276   if (av->top == initial_top (av))
 2277     return;
 2278 
 2279   /* pagesize is a power of 2 */
 2280   assert (powerof2(GLRO (dl_pagesize)));
 2281 
 2282   /* A contiguous main_arena is consistent with sbrk_base.  */
 2283   if (av == &main_arena && contiguous (av))
 2284     assert ((char *) mp_.sbrk_base + av->system_mem ==
 2285             (char *) av->top + chunksize (av->top));
 2286 
 2287   /* properties of fastbins */
 2288 
 2289   /* max_fast is in allowed range */
 2290   assert ((get_max_fast () & ~1) <= request2size (MAX_FAST_SIZE));
 2291 
 2292   max_fast_bin = fastbin_index (get_max_fast ());
 2293 
 2294   for (i = 0; i < NFASTBINS; ++i)
 2295     {
 2296       p = fastbin (av, i);
 2297 
 2298       /* The following test can only be performed for the main arena.
 2299          While mallopt calls malloc_consolidate to get rid of all fast
 2300          bins (especially those larger than the new maximum) this does
 2301          only happen for the main arena.  Trying to do this for any
 2302          other arena would mean those arenas have to be locked and
 2303          malloc_consolidate be called for them.  This is excessive.  And
 2304          even if this is acceptable to somebody it still cannot solve
 2305          the problem completely since if the arena is locked a
 2306          concurrent malloc call might create a new arena which then
 2307          could use the newly invalid fast bins.  */
 2308 
 2309       /* all bins past max_fast are empty */
 2310       if (av == &main_arena && i > max_fast_bin)
 2311         assert (p == 0);
 2312 
 2313       while (p != 0)
 2314         {
 2315       if (__glibc_unlikely (misaligned_chunk (p)))
 2316         malloc_printerr ("do_check_malloc_state(): "
 2317                  "unaligned fastbin chunk detected");
 2318           /* each chunk claims to be inuse */
 2319           do_check_inuse_chunk (av, p);
 2320           total += chunksize (p);
 2321           /* chunk belongs in this bin */
 2322           assert (fastbin_index (chunksize (p)) == i);
 2323       p = REVEAL_PTR (p->fd);
 2324         }
 2325     }
 2326 
 2327   /* check normal bins */
 2328   for (i = 1; i < NBINS; ++i)
 2329     {
 2330       b = bin_at (av, i);
 2331 
 2332       /* binmap is accurate (except for bin 1 == unsorted_chunks) */
 2333       if (i >= 2)
 2334         {
 2335           unsigned int binbit = get_binmap (av, i);
 2336           int empty = last (b) == b;
 2337           if (!binbit)
 2338             assert (empty);
 2339           else if (!empty)
 2340             assert (binbit);
 2341         }
 2342 
 2343       for (p = last (b); p != b; p = p->bk)
 2344         {
 2345           /* each chunk claims to be free */
 2346           do_check_free_chunk (av, p);
 2347           size = chunksize (p);
 2348           total += size;
 2349           if (i >= 2)
 2350             {
 2351               /* chunk belongs in bin */
 2352               idx = bin_index (size);
 2353               assert (idx == i);
 2354               /* lists are sorted */
 2355               assert (p->bk == b ||
 2356                       (unsigned long) chunksize (p->bk) >= (unsigned long) chunksize (p));
 2357 
 2358               if (!in_smallbin_range (size))
 2359                 {
 2360                   if (p->fd_nextsize != NULL)
 2361                     {
 2362                       if (p->fd_nextsize == p)
 2363                         assert (p->bk_nextsize == p);
 2364                       else
 2365                         {
 2366                           if (p->fd_nextsize == first (b))
 2367                             assert (chunksize (p) < chunksize (p->fd_nextsize));
 2368                           else
 2369                             assert (chunksize (p) > chunksize (p->fd_nextsize));
 2370 
 2371                           if (p == first (b))
 2372                             assert (chunksize (p) > chunksize (p->bk_nextsize));
 2373                           else
 2374                             assert (chunksize (p) < chunksize (p->bk_nextsize));
 2375                         }
 2376                     }
 2377                   else
 2378                     assert (p->bk_nextsize == NULL);
 2379                 }
 2380             }
 2381           else if (!in_smallbin_range (size))
 2382             assert (p->fd_nextsize == NULL && p->bk_nextsize == NULL);
 2383           /* chunk is followed by a legal chain of inuse chunks */
 2384           for (q = next_chunk (p);
 2385                (q != av->top && inuse (q) &&
 2386                 (unsigned long) (chunksize (q)) >= MINSIZE);
 2387                q = next_chunk (q))
 2388             do_check_inuse_chunk (av, q);
 2389         }
 2390     }
 2391 
 2392   /* top chunk is OK */
 2393   check_chunk (av, av->top);
 2394 }
 2395 #endif
 2396 
 2397 
 2398 /* ----------------- Support for debugging hooks -------------------- */
 2399 #if IS_IN (libc)
 2400 #include "hooks.c"
 2401 #endif
 2402 
 2403 
 2404 /* ----------- Routines dealing with system allocation -------------- */
 2405 
 2406 /*
 2407    sysmalloc handles malloc cases requiring more memory from the system.
 2408    On entry, it is assumed that av->top does not have enough
 2409    space to service request for nb bytes, thus requiring that av->top
 2410    be extended or replaced.
 2411  */
 2412 
 2413 static void *
 2414 sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av)
 2415 {
 2416   long int size;
 2417 
 2418   /*
 2419     Round up size to nearest page.  For mmapped chunks, the overhead is one
 2420     SIZE_SZ unit larger than for normal chunks, because there is no
 2421     following chunk whose prev_size field could be used.
 2422 
 2423     See the front_misalign handling below, for glibc there is no need for
 2424     further alignments unless we have have high alignment.
 2425    */
 2426   if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
 2427     size = ALIGN_UP (nb + SIZE_SZ, pagesize);
 2428   else
 2429     size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize);
 2430 
 2431   /* Don't try if size wraps around 0.  */
 2432   if ((unsigned long) (size) <= (unsigned long) (nb))
 2433     return MAP_FAILED;
 2434 
 2435   char *mm = (char *) MMAP (0, size,
 2436                 mtag_mmap_flags | PROT_READ | PROT_WRITE,
 2437                 extra_flags);
 2438   if (mm == MAP_FAILED)
 2439     return mm;
 2440 
 2441 #ifdef MAP_HUGETLB
 2442   if (!(extra_flags & MAP_HUGETLB))
 2443     madvise_thp (mm, size);
 2444 #endif
 2445 
 2446   /*
 2447     The offset to the start of the mmapped region is stored in the prev_size
 2448     field of the chunk.  This allows us to adjust returned start address to
 2449     meet alignment requirements here and in memalign(), and still be able to
 2450     compute proper address argument for later munmap in free() and realloc().
 2451    */
 2452 
 2453   INTERNAL_SIZE_T front_misalign; /* unusable bytes at front of new space */
 2454 
 2455   if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
 2456     {
 2457       /* For glibc, chunk2mem increases the address by CHUNK_HDR_SZ and
 2458      MALLOC_ALIGN_MASK is CHUNK_HDR_SZ-1.  Each mmap'ed area is page
 2459      aligned and therefore definitely MALLOC_ALIGN_MASK-aligned.  */
 2460       assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == 0);
 2461       front_misalign = 0;
 2462     }
 2463   else
 2464     front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK;
 2465 
 2466   mchunkptr p;                    /* the allocated/returned chunk */
 2467 
 2468   if (front_misalign > 0)
 2469     {
 2470       ptrdiff_t correction = MALLOC_ALIGNMENT - front_misalign;
 2471       p = (mchunkptr) (mm + correction);
 2472       set_prev_size (p, correction);
 2473       set_head (p, (size - correction) | IS_MMAPPED);
 2474     }
 2475   else
 2476     {
 2477       p = (mchunkptr) mm;
 2478       set_prev_size (p, 0);
 2479       set_head (p, size | IS_MMAPPED);
 2480     }
 2481 
 2482   /* update statistics */
 2483   int new = atomic_exchange_and_add (&mp_.n_mmaps, 1) + 1;
 2484   atomic_max (&mp_.max_n_mmaps, new);
 2485 
 2486   unsigned long sum;
 2487   sum = atomic_exchange_and_add (&mp_.mmapped_mem, size) + size;
 2488   atomic_max (&mp_.max_mmapped_mem, sum);
 2489 
 2490   check_chunk (av, p);
 2491 
 2492   return chunk2mem (p);
 2493 }
 2494 
 2495 /*
 2496    Allocate memory using mmap() based on S and NB requested size, aligning to
 2497    PAGESIZE if required.  The EXTRA_FLAGS is used on mmap() call.  If the call
 2498    succeedes S is updated with the allocated size.  This is used as a fallback
 2499    if MORECORE fails.
 2500  */
 2501 static void *
 2502 sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb,
 2503              INTERNAL_SIZE_T old_size, size_t minsize,
 2504              size_t pagesize, int extra_flags, mstate av)
 2505 {
 2506   long int size = *s;
 2507 
 2508   /* Cannot merge with old top, so add its size back in */
 2509   if (contiguous (av))
 2510     size = ALIGN_UP (size + old_size, pagesize);
 2511 
 2512   /* If we are relying on mmap as backup, then use larger units */
 2513   if ((unsigned long) (size) < minsize)
 2514     size = minsize;
 2515 
 2516   /* Don't try if size wraps around 0 */
 2517   if ((unsigned long) (size) <= (unsigned long) (nb))
 2518     return MORECORE_FAILURE;
 2519 
 2520   char *mbrk = (char *) (MMAP (0, size,
 2521                    mtag_mmap_flags | PROT_READ | PROT_WRITE,
 2522                    extra_flags));
 2523   if (mbrk == MAP_FAILED)
 2524     return MAP_FAILED;
 2525 
 2526 #ifdef MAP_HUGETLB
 2527   if (!(extra_flags & MAP_HUGETLB))
 2528     madvise_thp (mbrk, size);
 2529 #endif
 2530 
 2531   /* Record that we no longer have a contiguous sbrk region.  After the first
 2532      time mmap is used as backup, we do not ever rely on contiguous space
 2533      since this could incorrectly bridge regions.  */
 2534   set_noncontiguous (av);
 2535 
 2536   *s = size;
 2537   return mbrk;
 2538 }
 2539 
 2540 static void *
 2541 sysmalloc (INTERNAL_SIZE_T nb, mstate av)
 2542 {
 2543   mchunkptr old_top;              /* incoming value of av->top */
 2544   INTERNAL_SIZE_T old_size;       /* its size */
 2545   char *old_end;                  /* its end address */
 2546 
 2547   long size;                      /* arg to first MORECORE or mmap call */
 2548   char *brk;                      /* return value from MORECORE */
 2549 
 2550   long correction;                /* arg to 2nd MORECORE call */
 2551   char *snd_brk;                  /* 2nd return val */
 2552 
 2553   INTERNAL_SIZE_T front_misalign; /* unusable bytes at front of new space */
 2554   INTERNAL_SIZE_T end_misalign;   /* partial page left at end of new space */
 2555   char *aligned_brk;              /* aligned offset into brk */
 2556 
 2557   mchunkptr p;                    /* the allocated/returned chunk */
 2558   mchunkptr remainder;            /* remainder from allocation */
 2559   unsigned long remainder_size;   /* its size */
 2560 
 2561 
 2562   size_t pagesize = GLRO (dl_pagesize);
 2563   bool tried_mmap = false;
 2564 
 2565 
 2566   /*
 2567      If have mmap, and the request size meets the mmap threshold, and
 2568      the system supports mmap, and there are few enough currently
 2569      allocated mmapped regions, try to directly map this request
 2570      rather than expanding top.
 2571    */
 2572 
 2573   if (av == NULL
 2574       || ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold)
 2575       && (mp_.n_mmaps < mp_.n_mmaps_max)))
 2576     {
 2577       char *mm;
 2578 #if HAVE_TUNABLES
 2579       if (mp_.hp_pagesize > 0 && nb >= mp_.hp_pagesize)
 2580     {
 2581       /* There is no need to isse the THP madvise call if Huge Pages are
 2582          used directly.  */
 2583       mm = sysmalloc_mmap (nb, mp_.hp_pagesize, mp_.hp_flags, av);
 2584       if (mm != MAP_FAILED)
 2585         return mm;
 2586     }
 2587 #endif
 2588       mm = sysmalloc_mmap (nb, pagesize, 0, av);
 2589       if (mm != MAP_FAILED)
 2590     return mm;
 2591       tried_mmap = true;
 2592     }
 2593 
 2594   /* There are no usable arenas and mmap also failed.  */
 2595   if (av == NULL)
 2596     return 0;
 2597 
 2598   /* Record incoming configuration of top */
 2599 
 2600   old_top = av->top;
 2601   old_size = chunksize (old_top);
 2602   old_end = (char *) (chunk_at_offset (old_top, old_size));
 2603 
 2604   brk = snd_brk = (char *) (MORECORE_FAILURE);
 2605 
 2606   /*
 2607      If not the first time through, we require old_size to be
 2608      at least MINSIZE and to have prev_inuse set.
 2609    */
 2610 
 2611   assert ((old_top == initial_top (av) && old_size == 0) ||
 2612           ((unsigned long) (old_size) >= MINSIZE &&
 2613            prev_inuse (old_top) &&
 2614            ((unsigned long) old_end & (pagesize - 1)) == 0));
 2615 
 2616   /* Precondition: not enough current space to satisfy nb request */
 2617   assert ((unsigned long) (old_size) < (unsigned long) (nb + MINSIZE));
 2618 
 2619 
 2620   if (av != &main_arena)
 2621     {
 2622       heap_info *old_heap, *heap;
 2623       size_t old_heap_size;
 2624 
 2625       /* First try to extend the current heap. */
 2626       old_heap = heap_for_ptr (old_top);
 2627       old_heap_size = old_heap->size;
 2628       if ((long) (MINSIZE + nb - old_size) > 0
 2629           && grow_heap (old_heap, MINSIZE + nb - old_size) == 0)
 2630         {
 2631           av->system_mem += old_heap->size - old_heap_size;
 2632           set_head (old_top, (((char *) old_heap + old_heap->size) - (char *) old_top)
 2633                     | PREV_INUSE);
 2634         }
 2635       else if ((heap = new_heap (nb + (MINSIZE + sizeof (*heap)), mp_.top_pad)))
 2636         {
 2637           /* Use a newly allocated heap.  */
 2638           heap->ar_ptr = av;
 2639           heap->prev = old_heap;
 2640           av->system_mem += heap->size;
 2641           /* Set up the new top.  */
 2642           top (av) = chunk_at_offset (heap, sizeof (*heap));
 2643           set_head (top (av), (heap->size - sizeof (*heap)) | PREV_INUSE);
 2644 
 2645           /* Setup fencepost and free the old top chunk with a multiple of
 2646              MALLOC_ALIGNMENT in size. */
 2647           /* The fencepost takes at least MINSIZE bytes, because it might
 2648              become the top chunk again later.  Note that a footer is set
 2649              up, too, although the chunk is marked in use. */
 2650           old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK;
 2651           set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ),
 2652             0 | PREV_INUSE);
 2653           if (old_size >= MINSIZE)
 2654             {
 2655               set_head (chunk_at_offset (old_top, old_size),
 2656             CHUNK_HDR_SZ | PREV_INUSE);
 2657               set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ);
 2658               set_head (old_top, old_size | PREV_INUSE | NON_MAIN_ARENA);
 2659               _int_free (av, old_top, 1);
 2660             }
 2661           else
 2662             {
 2663               set_head (old_top, (old_size + CHUNK_HDR_SZ) | PREV_INUSE);
 2664               set_foot (old_top, (old_size + CHUNK_HDR_SZ));
 2665             }
 2666         }
 2667       else if (!tried_mmap)
 2668     {
 2669       /* We can at least try to use to mmap memory.  If new_heap fails
 2670          it is unlikely that trying to allocate huge pages will
 2671          succeed.  */
 2672       char *mm = sysmalloc_mmap (nb, pagesize, 0, av);
 2673       if (mm != MAP_FAILED)
 2674         return mm;
 2675     }
 2676     }
 2677   else     /* av == main_arena */
 2678 
 2679 
 2680     { /* Request enough space for nb + pad + overhead */
 2681       size = nb + mp_.top_pad + MINSIZE;
 2682 
 2683       /*
 2684          If contiguous, we can subtract out existing space that we hope to
 2685          combine with new space. We add it back later only if
 2686          we don't actually get contiguous space.
 2687        */
 2688 
 2689       if (contiguous (av))
 2690         size -= old_size;
 2691 
 2692       /*
 2693          Round to a multiple of page size or huge page size.
 2694          If MORECORE is not contiguous, this ensures that we only call it
 2695          with whole-page arguments.  And if MORECORE is contiguous and
 2696          this is not first time through, this preserves page-alignment of
 2697          previous calls. Otherwise, we correct to page-align below.
 2698        */
 2699 
 2700 #if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
 2701       /* Defined in brk.c.  */
 2702       extern void *__curbrk;
 2703       if (__glibc_unlikely (mp_.thp_pagesize != 0))
 2704     {
 2705       uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
 2706                     mp_.thp_pagesize);
 2707       size = top - (uintptr_t) __curbrk;
 2708     }
 2709       else
 2710 #endif
 2711     size = ALIGN_UP (size, GLRO(dl_pagesize));
 2712 
 2713       /*
 2714          Don't try to call MORECORE if argument is so big as to appear
 2715          negative. Note that since mmap takes size_t arg, it may succeed
 2716          below even if we cannot call MORECORE.
 2717        */
 2718 
 2719       if (size > 0)
 2720         {
 2721           brk = (char *) (MORECORE (size));
 2722       if (brk != (char *) (MORECORE_FAILURE))
 2723         madvise_thp (brk, size);
 2724           LIBC_PROBE (memory_sbrk_more, 2, brk, size);
 2725         }
 2726 
 2727       if (brk == (char *) (MORECORE_FAILURE))
 2728         {
 2729           /*
 2730              If have mmap, try using it as a backup when MORECORE fails or
 2731              cannot be used. This is worth doing on systems that have "holes" in
 2732              address space, so sbrk cannot extend to give contiguous space, but
 2733              space is available elsewhere.  Note that we ignore mmap max count
 2734              and threshold limits, since the space will not be used as a
 2735              segregated mmap region.
 2736            */
 2737 
 2738       char *mbrk = MAP_FAILED;
 2739 #if HAVE_TUNABLES
 2740       if (mp_.hp_pagesize > 0)
 2741         mbrk = sysmalloc_mmap_fallback (&size, nb, old_size,
 2742                         mp_.hp_pagesize, mp_.hp_pagesize,
 2743                         mp_.hp_flags, av);
 2744 #endif
 2745       if (mbrk == MAP_FAILED)
 2746         mbrk = sysmalloc_mmap_fallback (&size, nb, old_size, pagesize,
 2747                         MMAP_AS_MORECORE_SIZE, 0, av);
 2748       if (mbrk != MAP_FAILED)
 2749         {
 2750           /* We do not need, and cannot use, another sbrk call to find end */
 2751           brk = mbrk;
 2752           snd_brk = brk + size;
 2753         }
 2754         }
 2755 
 2756       if (brk != (char *) (MORECORE_FAILURE))
 2757         {
 2758           if (mp_.sbrk_base == 0)
 2759             mp_.sbrk_base = brk;
 2760           av->system_mem += size;
 2761 
 2762           /*
 2763              If MORECORE extends previous space, we can likewise extend top size.
 2764            */
 2765 
 2766           if (brk == old_end && snd_brk == (char *) (MORECORE_FAILURE))
 2767             set_head (old_top, (size + old_size) | PREV_INUSE);
 2768 
 2769           else if (contiguous (av) && old_size && brk < old_end)
 2770         /* Oops!  Someone else killed our space..  Can't touch anything.  */
 2771         malloc_printerr ("break adjusted to free malloc space");
 2772 
 2773           /*
 2774              Otherwise, make adjustments:
 2775 
 2776            * If the first time through or noncontiguous, we need to call sbrk
 2777               just to find out where the end of memory lies.
 2778 
 2779            * We need to ensure that all returned chunks from malloc will meet
 2780               MALLOC_ALIGNMENT
 2781 
 2782            * If there was an intervening foreign sbrk, we need to adjust sbrk
 2783               request size to account for fact that we will not be able to
 2784               combine new space with existing space in old_top.
 2785 
 2786            * Almost all systems internally allocate whole pages at a time, in
 2787               which case we might as well use the whole last page of request.
 2788               So we allocate enough more memory to hit a page boundary now,
 2789               which in turn causes future contiguous calls to page-align.
 2790            */
 2791 
 2792           else
 2793             {
 2794               front_misalign = 0;
 2795               end_misalign = 0;
 2796               correction = 0;
 2797               aligned_brk = brk;
 2798 
 2799               /* handle contiguous cases */
 2800               if (contiguous (av))
 2801                 {
 2802                   /* Count foreign sbrk as system_mem.  */
 2803                   if (old_size)
 2804                     av->system_mem += brk - old_end;
 2805 
 2806                   /* Guarantee alignment of first new chunk made from this space */
 2807 
 2808                   front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
 2809                   if (front_misalign > 0)
 2810                     {
 2811                       /*
 2812                          Skip over some bytes to arrive at an aligned position.
 2813                          We don't need to specially mark these wasted front bytes.
 2814                          They will never be accessed anyway because
 2815                          prev_inuse of av->top (and any chunk created from its start)
 2816                          is always true after initialization.
 2817                        */
 2818 
 2819                       correction = MALLOC_ALIGNMENT - front_misalign;
 2820                       aligned_brk += correction;
 2821                     }
 2822 
 2823                   /*
 2824                      If this isn't adjacent to existing space, then we will not
 2825                      be able to merge with old_top space, so must add to 2nd request.
 2826                    */
 2827 
 2828                   correction += old_size;
 2829 
 2830                   /* Extend the end address to hit a page boundary */
 2831                   end_misalign = (INTERNAL_SIZE_T) (brk + size + correction);
 2832                   correction += (ALIGN_UP (end_misalign, pagesize)) - end_misalign;
 2833 
 2834                   assert (correction >= 0);
 2835                   snd_brk = (char *) (MORECORE (correction));
 2836 
 2837                   /*
 2838                      If can't allocate correction, try to at least find out current
 2839                      brk.  It might be enough to proceed without failing.
 2840 
 2841                      Note that if second sbrk did NOT fail, we assume that space
 2842                      is contiguous with first sbrk. This is a safe assumption unless
 2843                      program is multithreaded but doesn't use locks and a foreign sbrk
 2844                      occurred between our first and second calls.
 2845                    */
 2846 
 2847                   if (snd_brk == (char *) (MORECORE_FAILURE))
 2848                     {
 2849                       correction = 0;
 2850                       snd_brk = (char *) (MORECORE (0));
 2851                     }
 2852           else
 2853             madvise_thp (snd_brk, correction);
 2854                 }
 2855 
 2856               /* handle non-contiguous cases */
 2857               else
 2858                 {
 2859                   if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
 2860                     /* MORECORE/mmap must correctly align */
 2861                     assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == 0);
 2862                   else
 2863                     {
 2864                       front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
 2865                       if (front_misalign > 0)
 2866                         {
 2867                           /*
 2868                              Skip over some bytes to arrive at an aligned position.
 2869                              We don't need to specially mark these wasted front bytes.
 2870                              They will never be accessed anyway because
 2871                              prev_inuse of av->top (and any chunk created from its start)
 2872                              is always true after initialization.
 2873                            */
 2874 
 2875                           aligned_brk += MALLOC_ALIGNMENT - front_misalign;
 2876                         }
 2877                     }
 2878 
 2879                   /* Find out current end of memory */
 2880                   if (snd_brk == (char *) (MORECORE_FAILURE))
 2881                     {
 2882                       snd_brk = (char *) (MORECORE (0));
 2883                     }
 2884                 }
 2885 
 2886               /* Adjust top based on results of second sbrk */
 2887               if (snd_brk != (char *) (MORECORE_FAILURE))
 2888                 {
 2889                   av->top = (mchunkptr) aligned_brk;
 2890                   set_head (av->top, (snd_brk - aligned_brk + correction) | PREV_INUSE);
 2891                   av->system_mem += correction;
 2892 
 2893                   /*
 2894                      If not the first time through, we either have a
 2895                      gap due to foreign sbrk or a non-contiguous region.  Insert a
 2896                      double fencepost at old_top to prevent consolidation with space
 2897                      we don't own. These fenceposts are artificial chunks that are
 2898                      marked as inuse and are in any case too small to use.  We need
 2899                      two to make sizes and alignments work out.
 2900                    */
 2901 
 2902                   if (old_size != 0)
 2903                     {
 2904                       /*
 2905                          Shrink old_top to insert fenceposts, keeping size a
 2906                          multiple of MALLOC_ALIGNMENT. We know there is at least
 2907                          enough space in old_top to do this.
 2908                        */
 2909                       old_size = (old_size - 2 * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK;
 2910                       set_head (old_top, old_size | PREV_INUSE);
 2911 
 2912                       /*
 2913                          Note that the following assignments completely overwrite
 2914                          old_top when old_size was previously MINSIZE.  This is
 2915                          intentional. We need the fencepost, even if old_top otherwise gets
 2916                          lost.
 2917                        */
 2918               set_head (chunk_at_offset (old_top, old_size),
 2919                 CHUNK_HDR_SZ | PREV_INUSE);
 2920               set_head (chunk_at_offset (old_top,
 2921                          old_size + CHUNK_HDR_SZ),
 2922                 CHUNK_HDR_SZ | PREV_INUSE);
 2923 
 2924                       /* If possible, release the rest. */
 2925                       if (old_size >= MINSIZE)
 2926                         {
 2927                           _int_free (av, old_top, 1);
 2928                         }
 2929                     }
 2930                 }
 2931             }
 2932         }
 2933     } /* if (av !=  &main_arena) */
 2934 
 2935   if ((unsigned long) av->system_mem > (unsigned long) (av->max_system_mem))
 2936     av->max_system_mem = av->system_mem;
 2937   check_malloc_state (av);
 2938 
 2939   /* finally, do the allocation */
 2940   p = av->top;
 2941   size = chunksize (p);
 2942 
 2943   /* check that one of the above allocation paths succeeded */
 2944   if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
 2945     {
 2946       remainder_size = size - nb;
 2947       remainder = chunk_at_offset (p, nb);
 2948       av->top = remainder;
 2949       set_head (p, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0));
 2950       set_head (remainder, remainder_size | PREV_INUSE);
 2951       check_malloced_chunk (av, p, nb);
 2952       return chunk2mem (p);
 2953     }
 2954 
 2955   /* catch all failure paths */
 2956   __set_errno (ENOMEM);
 2957   return 0;
 2958 }
 2959 
 2960 
 2961 /*
 2962    systrim is an inverse of sorts to sysmalloc.  It gives memory back
 2963    to the system (via negative arguments to sbrk) if there is unused
 2964    memory at the `high' end of the malloc pool. It is called
 2965    automatically by free() when top space exceeds the trim
 2966    threshold. It is also called by the public malloc_trim routine.  It
 2967    returns 1 if it actually released any memory, else 0.
 2968  */
 2969 
 2970 static int
 2971 systrim (size_t pad, mstate av)
 2972 {
 2973   long top_size;         /* Amount of top-most memory */
 2974   long extra;            /* Amount to release */
 2975   long released;         /* Amount actually released */
 2976   char *current_brk;     /* address returned by pre-check sbrk call */
 2977   char *new_brk;         /* address returned by post-check sbrk call */
 2978   long top_area;
 2979 
 2980   top_size = chunksize (av->top);
 2981 
 2982   top_area = top_size - MINSIZE - 1;
 2983   if (top_area <= pad)
 2984     return 0;
 2985 
 2986   /* Release in pagesize units and round down to the nearest page.  */
 2987 #if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
 2988   if (__glibc_unlikely (mp_.thp_pagesize != 0))
 2989     extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
 2990   else
 2991 #endif
 2992     extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
 2993 
 2994   if (extra == 0)
 2995     return 0;
 2996 
 2997   /*
 2998      Only proceed if end of memory is where we last set it.
 2999      This avoids problems if there were foreign sbrk calls.
 3000    */
 3001   current_brk = (char *) (MORECORE (0));
 3002   if (current_brk == (char *) (av->top) + top_size)
 3003     {
 3004       /*
 3005          Attempt to release memory. We ignore MORECORE return value,
 3006          and instead call again to find out where new end of memory is.
 3007          This avoids problems if first call releases less than we asked,
 3008          of if failure somehow altered brk value. (We could still
 3009          encounter problems if it altered brk in some very bad way,
 3010          but the only thing we can do is adjust anyway, which will cause
 3011          some downstream failure.)
 3012        */
 3013 
 3014       MORECORE (-extra);
 3015       new_brk = (char *) (MORECORE (0));
 3016 
 3017       LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra);
 3018 
 3019       if (new_brk != (char *) MORECORE_FAILURE)
 3020         {
 3021           released = (long) (current_brk - new_brk);
 3022 
 3023           if (released != 0)
 3024             {
 3025               /* Success. Adjust top. */
 3026               av->system_mem -= released;
 3027               set_head (av->top, (top_size - released) | PREV_INUSE);
 3028               check_malloc_state (av);
 3029               return 1;
 3030             }
 3031         }
 3032     }
 3033   return 0;
 3034 }
 3035 
 3036 static void
 3037 munmap_chunk (mchunkptr p)
 3038 {
 3039   size_t pagesize = GLRO (dl_pagesize);
 3040   INTERNAL_SIZE_T size = chunksize (p);
 3041 
 3042   assert (chunk_is_mmapped (p));
 3043 
 3044   uintptr_t mem = (uintptr_t) chunk2mem (p);
 3045   uintptr_t block = (uintptr_t) p - prev_size (p);
 3046   size_t total_size = prev_size (p) + size;
 3047   /* Unfortunately we have to do the compilers job by hand here.  Normally
 3048      we would test BLOCK and TOTAL-SIZE separately for compliance with the
 3049      page size.  But gcc does not recognize the optimization possibility
 3050      (in the moment at least) so we combine the two values into one before
 3051      the bit test.  */
 3052   if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
 3053       || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
 3054     malloc_printerr ("munmap_chunk(): invalid pointer");
 3055 
 3056   atomic_decrement (&mp_.n_mmaps);
 3057   atomic_add (&mp_.mmapped_mem, -total_size);
 3058 
 3059   /* If munmap failed the process virtual memory address space is in a
 3060      bad shape.  Just leave the block hanging around, the process will
 3061      terminate shortly anyway since not much can be done.  */
 3062   __munmap ((char *) block, total_size);
 3063 }
 3064 
 3065 #if HAVE_MREMAP
 3066 
 3067 static mchunkptr
 3068 mremap_chunk (mchunkptr p, size_t new_size)
 3069 {
 3070   size_t pagesize = GLRO (dl_pagesize);
 3071   INTERNAL_SIZE_T offset = prev_size (p);
 3072   INTERNAL_SIZE_T size = chunksize (p);
 3073   char *cp;
 3074 
 3075   assert (chunk_is_mmapped (p));
 3076 
 3077   uintptr_t block = (uintptr_t) p - offset;
 3078   uintptr_t mem = (uintptr_t) chunk2mem(p);
 3079   size_t total_size = offset + size;
 3080   if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
 3081       || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
 3082     malloc_printerr("mremap_chunk(): invalid pointer");
 3083 
 3084   /* Note the extra SIZE_SZ overhead as in mmap_chunk(). */
 3085   new_size = ALIGN_UP (new_size + offset + SIZE_SZ, pagesize);
 3086 
 3087   /* No need to remap if the number of pages does not change.  */
 3088   if (total_size == new_size)
 3089     return p;
 3090 
 3091   cp = (char *) __mremap ((char *) block, total_size, new_size,
 3092                           MREMAP_MAYMOVE);
 3093 
 3094   if (cp == MAP_FAILED)
 3095     return 0;
 3096 
 3097   madvise_thp (cp, new_size);
 3098 
 3099   p = (mchunkptr) (cp + offset);
 3100 
 3101   assert (aligned_OK (chunk2mem (p)));
 3102 
 3103   assert (prev_size (p) == offset);
 3104   set_head (p, (new_size - offset) | IS_MMAPPED);
 3105 
 3106   INTERNAL_SIZE_T new;
 3107   new = atomic_exchange_and_add (&mp_.mmapped_mem, new_size - size - offset)
 3108         + new_size - size - offset;
 3109   atomic_max (&mp_.max_mmapped_mem, new);
 3110   return p;
 3111 }
 3112 #endif /* HAVE_MREMAP */
 3113 
 3114 /*------------------------ Public wrappers. --------------------------------*/
 3115 
 3116 #if USE_TCACHE
 3117 
 3118 /* We overlay this structure on the user-data portion of a chunk when
 3119    the chunk is stored in the per-thread cache.  */
 3120 typedef struct tcache_entry
 3121 {
 3122   struct tcache_entry *next;
 3123   /* This field exists to detect double frees.  */
 3124   uintptr_t key;
 3125 } tcache_entry;
 3126 
 3127 /* There is one of these for each thread, which contains the
 3128    per-thread cache (hence "tcache_perthread_struct").  Keeping
 3129    overall size low is mildly important.  Note that COUNTS and ENTRIES
 3130    are redundant (we could have just counted the linked list each
 3131    time), this is for performance reasons.  */
 3132 typedef struct tcache_perthread_struct
 3133 {
 3134   uint16_t counts[TCACHE_MAX_BINS];
 3135   tcache_entry *entries[TCACHE_MAX_BINS];
 3136 } tcache_perthread_struct;
 3137 
 3138 static __thread bool tcache_shutting_down = false;
 3139 static __thread tcache_perthread_struct *tcache = NULL;
 3140 
 3141 /* Process-wide key to try and catch a double-free in the same thread.  */
 3142 static uintptr_t tcache_key;
 3143 
 3144 /* The value of tcache_key does not really have to be a cryptographically
 3145    secure random number.  It only needs to be arbitrary enough so that it does
 3146    not collide with values present in applications.  If a collision does happen
 3147    consistently enough, it could cause a degradation in performance since the
 3148    entire list is checked to check if the block indeed has been freed the
 3149    second time.  The odds of this happening are exceedingly low though, about 1
 3150    in 2^wordsize.  There is probably a higher chance of the performance
 3151    degradation being due to a double free where the first free happened in a
 3152    different thread; that's a case this check does not cover.  */
 3153 static void
 3154 tcache_key_initialize (void)
 3155 {
 3156   if (__getrandom (&tcache_key, sizeof(tcache_key), GRND_NONBLOCK)
 3157       != sizeof (tcache_key))
 3158     {
 3159       tcache_key = random_bits ();
 3160 #if __WORDSIZE == 64
 3161       tcache_key = (tcache_key << 32) | random_bits ();
 3162 #endif
 3163     }
 3164 }
 3165 
 3166 /* Caller must ensure that we know tc_idx is valid and there's room
 3167    for more chunks.  */
 3168 static __always_inline void
 3169 tcache_put (mchunkptr chunk, size_t tc_idx)
 3170 {
 3171   tcache_entry *e = (tcache_entry *) chunk2mem (chunk);
 3172 
 3173   /* Mark this chunk as "in the tcache" so the test in _int_free will
 3174      detect a double free.  */
 3175   e->key = tcache_key;
 3176 
 3177   e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]);
 3178   tcache->entries[tc_idx] = e;
 3179   ++(tcache->counts[tc_idx]);
 3180 }
 3181 
 3182 /* Caller must ensure that we know tc_idx is valid and there's
 3183    available chunks to remove.  */
 3184 static __always_inline void *
 3185 tcache_get (size_t tc_idx)
 3186 {
 3187   tcache_entry *e = tcache->entries[tc_idx];
 3188   if (__glibc_unlikely (!aligned_OK (e)))
 3189     malloc_printerr ("malloc(): unaligned tcache chunk detected");
 3190   tcache->entries[tc_idx] = REVEAL_PTR (e->next);
 3191   --(tcache->counts[tc_idx]);
 3192   e->key = 0;
 3193   return (void *) e;
 3194 }
 3195 
 3196 static void
 3197 tcache_thread_shutdown (void)
 3198 {
 3199   int i;
 3200   tcache_perthread_struct *tcache_tmp = tcache;
 3201 
 3202   tcache_shutting_down = true;
 3203 
 3204   if (!tcache)
 3205     return;
 3206 
 3207   /* Disable the tcache and prevent it from being reinitialized.  */
 3208   tcache = NULL;
 3209 
 3210   /* Free all of the entries and the tcache itself back to the arena
 3211      heap for coalescing.  */
 3212   for (i = 0; i < TCACHE_MAX_BINS; ++i)
 3213     {
 3214       while (tcache_tmp->entries[i])
 3215     {
 3216       tcache_entry *e = tcache_tmp->entries[i];
 3217       if (__glibc_unlikely (!aligned_OK (e)))
 3218         malloc_printerr ("tcache_thread_shutdown(): "
 3219                  "unaligned tcache chunk detected");
 3220       tcache_tmp->entries[i] = REVEAL_PTR (e->next);
 3221       __libc_free (e);
 3222     }
 3223     }
 3224 
 3225   __libc_free (tcache_tmp);
 3226 }
 3227 
 3228 static void
 3229 tcache_init(void)
 3230 {
 3231   mstate ar_ptr;
 3232   void *victim = 0;
 3233   const size_t bytes = sizeof (tcache_perthread_struct);
 3234 
 3235   if (tcache_shutting_down)
 3236     return;
 3237 
 3238   arena_get (ar_ptr, bytes);
 3239   victim = _int_malloc (ar_ptr, bytes);
 3240   if (!victim && ar_ptr != NULL)
 3241     {
 3242       ar_ptr = arena_get_retry (ar_ptr, bytes);
 3243       victim = _int_malloc (ar_ptr, bytes);
 3244     }
 3245 
 3246 
 3247   if (ar_ptr != NULL)
 3248     __libc_lock_unlock (ar_ptr->mutex);
 3249 
 3250   /* In a low memory situation, we may not be able to allocate memory
 3251      - in which case, we just keep trying later.  However, we
 3252      typically do this very early, so either there is sufficient
 3253      memory, or there isn't enough memory to do non-trivial
 3254      allocations anyway.  */
 3255   if (victim)
 3256     {
 3257       tcache = (tcache_perthread_struct *) victim;
 3258       memset (tcache, 0, sizeof (tcache_perthread_struct));
 3259     }
 3260 
 3261 }
 3262 
 3263 # define MAYBE_INIT_TCACHE() \
 3264   if (__glibc_unlikely (tcache == NULL)) \
 3265     tcache_init();
 3266 
 3267 #else  /* !USE_TCACHE */
 3268 # define MAYBE_INIT_TCACHE()
 3269 
 3270 static void
 3271 tcache_thread_shutdown (void)
 3272 {
 3273   /* Nothing to do if there is no thread cache.  */
 3274 }
 3275 
 3276 #endif /* !USE_TCACHE  */
 3277 
 3278 #if IS_IN (libc)
 3279 void *
 3280 __libc_malloc (size_t bytes)
 3281 {
 3282   mstate ar_ptr;
 3283   void *victim;
 3284 
 3285   _Static_assert (PTRDIFF_MAX <= SIZE_MAX / 2,
 3286                   "PTRDIFF_MAX is not more than half of SIZE_MAX");
 3287 
 3288   if (!__malloc_initialized)
 3289     ptmalloc_init ();
 3290 #if USE_TCACHE
 3291   /* int_free also calls request2size, be careful to not pad twice.  */
 3292   size_t tbytes = checked_request2size (bytes);
 3293   if (tbytes == 0)
 3294     {
 3295       __set_errno (ENOMEM);
 3296       return NULL;
 3297     }
 3298   size_t tc_idx = csize2tidx (tbytes);
 3299 
 3300   MAYBE_INIT_TCACHE ();
 3301 
 3302   DIAG_PUSH_NEEDS_COMMENT;
 3303   if (tc_idx < mp_.tcache_bins
 3304       && tcache
 3305       && tcache->counts[tc_idx] > 0)
 3306     {
 3307       victim = tcache_get (tc_idx);
 3308       return tag_new_usable (victim);
 3309     }
 3310   DIAG_POP_NEEDS_COMMENT;
 3311 #endif
 3312 
 3313   if (SINGLE_THREAD_P)
 3314     {
 3315       victim = tag_new_usable (_int_malloc (&main_arena, bytes));
 3316       assert (!victim || chunk_is_mmapped (mem2chunk (victim)) ||
 3317           &main_arena == arena_for_chunk (mem2chunk (victim)));
 3318       return victim;
 3319     }
 3320 
 3321   arena_get (ar_ptr, bytes);
 3322 
 3323   victim = _int_malloc (ar_ptr, bytes);
 3324   /* Retry with another arena only if we were able to find a usable arena
 3325      before.  */
 3326   if (!victim && ar_ptr != NULL)
 3327     {
 3328       LIBC_PROBE (memory_malloc_retry, 1, bytes);
 3329       ar_ptr = arena_get_retry (ar_ptr, bytes);
 3330       victim = _int_malloc (ar_ptr, bytes);
 3331     }
 3332 
 3333   if (ar_ptr != NULL)
 3334     __libc_lock_unlock (ar_ptr->mutex);
 3335 
 3336   victim = tag_new_usable (victim);
 3337 
 3338   assert (!victim || chunk_is_mmapped (mem2chunk (victim)) ||
 3339           ar_ptr == arena_for_chunk (mem2chunk (victim)));
 3340   return victim;
 3341 }
 3342 libc_hidden_def (__libc_malloc)
 3343 
 3344 void
 3345 __libc_free (void *mem)
 3346 {
 3347   mstate ar_ptr;
 3348   mchunkptr p;                          /* chunk corresponding to mem */
 3349 
 3350   if (mem == 0)                              /* free(0) has no effect */
 3351     return;
 3352 
 3353   /* Quickly check that the freed pointer matches the tag for the memory.
 3354      This gives a useful double-free detection.  */
 3355   if (__glibc_unlikely (mtag_enabled))
 3356     *(volatile char *)mem;
 3357 
 3358   int err = errno;
 3359 
 3360   p = mem2chunk (mem);
 3361 
 3362   if (chunk_is_mmapped (p))                       /* release mmapped memory. */
 3363     {
 3364       /* See if the dynamic brk/mmap threshold needs adjusting.
 3365      Dumped fake mmapped chunks do not affect the threshold.  */
 3366       if (!mp_.no_dyn_threshold
 3367           && chunksize_nomask (p) > mp_.mmap_threshold
 3368           && chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
 3369         {
 3370           mp_.mmap_threshold = chunksize (p);
 3371           mp_.trim_threshold = 2 * mp_.mmap_threshold;
 3372           LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2,
 3373                       mp_.mmap_threshold, mp_.trim_threshold);
 3374         }
 3375       munmap_chunk (p);
 3376     }
 3377   else
 3378     {
 3379       MAYBE_INIT_TCACHE ();
 3380 
 3381       /* Mark the chunk as belonging to the library again.  */
 3382       (void)tag_region (chunk2mem (p), memsize (p));
 3383 
 3384       ar_ptr = arena_for_chunk (p);
 3385       _int_free (ar_ptr, p, 0);
 3386     }
 3387 
 3388   __set_errno (err);
 3389 }
 3390 libc_hidden_def (__libc_free)
 3391 
 3392 void *
 3393 __libc_realloc (void *oldmem, size_t bytes)
 3394 {
 3395   mstate ar_ptr;
 3396   INTERNAL_SIZE_T nb;         /* padded request size */
 3397 
 3398   void *newp;             /* chunk to return */
 3399 
 3400   if (!__malloc_initialized)
 3401     ptmalloc_init ();
 3402 
 3403 #if REALLOC_ZERO_BYTES_FREES
 3404   if (bytes == 0 && oldmem != NULL)
 3405     {
 3406       __libc_free (oldmem); return 0;
 3407     }
 3408 #endif
 3409 
 3410   /* realloc of null is supposed to be same as malloc */
 3411   if (oldmem == 0)
 3412     return __libc_malloc (bytes);
 3413 
 3414   /* Perform a quick check to ensure that the pointer's tag matches the
 3415      memory's tag.  */
 3416   if (__glibc_unlikely (mtag_enabled))
 3417     *(volatile char*) oldmem;
 3418 
 3419   /* chunk corresponding to oldmem */
 3420   const mchunkptr oldp = mem2chunk (oldmem);
 3421   /* its size */
 3422   const INTERNAL_SIZE_T oldsize = chunksize (oldp);
 3423 
 3424   if (chunk_is_mmapped (oldp))
 3425     ar_ptr = NULL;
 3426   else
 3427     {
 3428       MAYBE_INIT_TCACHE ();
 3429       ar_ptr = arena_for_chunk (oldp);
 3430     }
 3431 
 3432   /* Little security check which won't hurt performance: the allocator
 3433      never wrapps around at the end of the address space.  Therefore
 3434      we can exclude some size values which might appear here by
 3435      accident or by "design" from some intruder.  */
 3436   if ((__builtin_expect ((uintptr_t) oldp > (uintptr_t) -oldsize, 0)
 3437        || __builtin_expect (misaligned_chunk (oldp), 0)))
 3438       malloc_printerr ("realloc(): invalid pointer");
 3439 
 3440   nb = checked_request2size (bytes);
 3441   if (nb == 0)
 3442     {
 3443       __set_errno (ENOMEM);
 3444       return NULL;
 3445     }
 3446 
 3447   if (chunk_is_mmapped (oldp))
 3448     {
 3449       void *newmem;
 3450 
 3451 #if HAVE_MREMAP
 3452       newp = mremap_chunk (oldp, nb);
 3453       if (newp)
 3454     {
 3455       void *newmem = chunk2mem_tag (newp);
 3456       /* Give the new block a different tag.  This helps to ensure
 3457          that stale handles to the previous mapping are not
 3458          reused.  There's a performance hit for both us and the
 3459          caller for doing this, so we might want to
 3460          reconsider.  */
 3461       return tag_new_usable (newmem);
 3462     }
 3463 #endif
 3464       /* Note the extra SIZE_SZ overhead. */
 3465       if (oldsize - SIZE_SZ >= nb)
 3466         return oldmem;                         /* do nothing */
 3467 
 3468       /* Must alloc, copy, free. */
 3469       newmem = __libc_malloc (bytes);
 3470       if (newmem == 0)
 3471         return 0;              /* propagate failure */
 3472 
 3473       memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ);
 3474       munmap_chunk (oldp);
 3475       return newmem;
 3476     }
 3477 
 3478   if (SINGLE_THREAD_P)
 3479     {
 3480       newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
 3481       assert (!newp || chunk_is_mmapped (mem2chunk (newp)) ||
 3482           ar_ptr == arena_for_chunk (mem2chunk (newp)));
 3483 
 3484       return newp;
 3485     }
 3486 
 3487   __libc_lock_lock (ar_ptr->mutex);
 3488 
 3489   newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
 3490 
 3491   __libc_lock_unlock (ar_ptr->mutex);
 3492   assert (!newp || chunk_is_mmapped (mem2chunk (newp)) ||
 3493           ar_ptr == arena_for_chunk (mem2chunk (newp)));
 3494 
 3495   if (newp == NULL)
 3496     {
 3497       /* Try harder to allocate memory in other arenas.  */
 3498       LIBC_PROBE (memory_realloc_retry, 2, bytes, oldmem);
 3499       newp = __libc_malloc (bytes);
 3500       if (newp != NULL)
 3501         {
 3502       size_t sz = memsize (oldp);
 3503       memcpy (newp, oldmem, sz);
 3504       (void) tag_region (chunk2mem (oldp), sz);
 3505           _int_free (ar_ptr, oldp, 0);
 3506         }
 3507     }
 3508 
 3509   return newp;
 3510 }
 3511 libc_hidden_def (__libc_realloc)
 3512 
 3513 void *
 3514 __libc_memalign (size_t alignment, size_t bytes)
 3515 {
 3516   if (!__malloc_initialized)
 3517     ptmalloc_init ();
 3518 
 3519   void *address = RETURN_ADDRESS (0);
 3520   return _mid_memalign (alignment, bytes, address);
 3521 }
 3522 
 3523 static void *
 3524 _mid_memalign (size_t alignment, size_t bytes, void *address)
 3525 {
 3526   mstate ar_ptr;
 3527   void *p;
 3528 
 3529   /* If we need less alignment than we give anyway, just relay to malloc.  */
 3530   if (alignment <= MALLOC_ALIGNMENT)
 3531     return __libc_malloc (bytes);
 3532 
 3533   /* Otherwise, ensure that it is at least a minimum chunk size */
 3534   if (alignment < MINSIZE)
 3535     alignment = MINSIZE;
 3536 
 3537   /* If the alignment is greater than SIZE_MAX / 2 + 1 it cannot be a
 3538      power of 2 and will cause overflow in the check below.  */
 3539   if (alignment > SIZE_MAX / 2 + 1)
 3540     {
 3541       __set_errno (EINVAL);
 3542       return 0;
 3543     }
 3544 
 3545 
 3546   /* Make sure alignment is power of 2.  */
 3547   if (!powerof2 (alignment))
 3548     {
 3549       size_t a = MALLOC_ALIGNMENT * 2;
 3550       while (a < alignment)
 3551         a <<= 1;
 3552       alignment = a;
 3553     }
 3554 
 3555   if (SINGLE_THREAD_P)
 3556     {
 3557       p = _int_memalign (&main_arena, alignment, bytes);
 3558       assert (!p || chunk_is_mmapped (mem2chunk (p)) ||
 3559           &main_arena == arena_for_chunk (mem2chunk (p)));
 3560       return tag_new_usable (p);
 3561     }
 3562 
 3563   arena_get (ar_ptr, bytes + alignment + MINSIZE);
 3564 
 3565   p = _int_memalign (ar_ptr, alignment, bytes);
 3566   if (!p && ar_ptr != NULL)
 3567     {
 3568       LIBC_PROBE (memory_memalign_retry, 2, bytes, alignment);
 3569       ar_ptr = arena_get_retry (ar_ptr, bytes);
 3570       p = _int_memalign (ar_ptr, alignment, bytes);
 3571     }
 3572 
 3573   if (ar_ptr != NULL)
 3574     __libc_lock_unlock (ar_ptr->mutex);
 3575 
 3576   assert (!p || chunk_is_mmapped (mem2chunk (p)) ||
 3577           ar_ptr == arena_for_chunk (mem2chunk (p)));
 3578   return tag_new_usable (p);
 3579 }
 3580 /* For ISO C11.  */
 3581 weak_alias (__libc_memalign, aligned_alloc)
 3582 libc_hidden_def (__libc_memalign)
 3583 
 3584 void *
 3585 __libc_valloc (size_t bytes)
 3586 {
 3587   if (!__malloc_initialized)
 3588     ptmalloc_init ();
 3589 
 3590   void *address = RETURN_ADDRESS (0);
 3591   size_t pagesize = GLRO (dl_pagesize);
 3592   return _mid_memalign (pagesize, bytes, address);
 3593 }
 3594 
 3595 void *
 3596 __libc_pvalloc (size_t bytes)
 3597 {
 3598   if (!__malloc_initialized)
 3599     ptmalloc_init ();
 3600 
 3601   void *address = RETURN_ADDRESS (0);
 3602   size_t pagesize = GLRO (dl_pagesize);
 3603   size_t rounded_bytes;
 3604   /* ALIGN_UP with overflow check.  */
 3605   if (__glibc_unlikely (__builtin_add_overflow (bytes,
 3606                         pagesize - 1,
 3607                         &rounded_bytes)))
 3608     {
 3609       __set_errno (ENOMEM);
 3610       return 0;
 3611     }
 3612   rounded_bytes = rounded_bytes & -(pagesize - 1);
 3613 
 3614   return _mid_memalign (pagesize, rounded_bytes, address);
 3615 }
 3616 
 3617 void *
 3618 __libc_calloc (size_t n, size_t elem_size)
 3619 {
 3620   mstate av;
 3621   mchunkptr oldtop;
 3622   INTERNAL_SIZE_T sz, oldtopsize;
 3623   void *mem;
 3624   unsigned long clearsize;
 3625   unsigned long nclears;
 3626   INTERNAL_SIZE_T *d;
 3627   ptrdiff_t bytes;
 3628 
 3629   if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
 3630     {
 3631        __set_errno (ENOMEM);
 3632        return NULL;
 3633     }
 3634 
 3635   sz = bytes;
 3636 
 3637   if (!__malloc_initialized)
 3638     ptmalloc_init ();
 3639 
 3640   MAYBE_INIT_TCACHE ();
 3641 
 3642   if (SINGLE_THREAD_P)
 3643     av = &main_arena;
 3644   else
 3645     arena_get (av, sz);
 3646 
 3647   if (av)
 3648     {
 3649       /* Check if we hand out the top chunk, in which case there may be no
 3650      need to clear. */
 3651 #if MORECORE_CLEARS
 3652       oldtop = top (av);
 3653       oldtopsize = chunksize (top (av));
 3654 # if MORECORE_CLEARS < 2
 3655       /* Only newly allocated memory is guaranteed to be cleared.  */
 3656       if (av == &main_arena &&
 3657       oldtopsize < mp_.sbrk_base + av->max_system_mem - (char *) oldtop)
 3658     oldtopsize = (mp_.sbrk_base + av->max_system_mem - (char *) oldtop);
 3659 # endif
 3660       if (av != &main_arena)
 3661     {
 3662       heap_info *heap = heap_for_ptr (oldtop);
 3663       if (oldtopsize < (char *) heap + heap->mprotect_size - (char *) oldtop)
 3664         oldtopsize = (char *) heap + heap->mprotect_size - (char *) oldtop;
 3665     }
 3666 #endif
 3667     }
 3668   else
 3669     {
 3670       /* No usable arenas.  */
 3671       oldtop = 0;
 3672       oldtopsize = 0;
 3673     }
 3674   mem = _int_malloc (av, sz);
 3675 
 3676   assert (!mem || chunk_is_mmapped (mem2chunk (mem)) ||
 3677           av == arena_for_chunk (mem2chunk (mem)));
 3678 
 3679   if (!SINGLE_THREAD_P)
 3680     {
 3681       if (mem == 0 && av != NULL)
 3682     {
 3683       LIBC_PROBE (memory_calloc_retry, 1, sz);
 3684       av = arena_get_retry (av, sz);
 3685       mem = _int_malloc (av, sz);
 3686     }
 3687 
 3688       if (av != NULL)
 3689     __libc_lock_unlock (av->mutex);
 3690     }
 3691 
 3692   /* Allocation failed even after a retry.  */
 3693   if (mem == 0)
 3694     return 0;
 3695 
 3696   mchunkptr p = mem2chunk (mem);
 3697 
 3698   /* If we are using memory tagging, then we need to set the tags
 3699      regardless of MORECORE_CLEARS, so we zero the whole block while
 3700      doing so.  */
 3701   if (__glibc_unlikely (mtag_enabled))
 3702     return tag_new_zero_region (mem, memsize (p));
 3703 
 3704   INTERNAL_SIZE_T csz = chunksize (p);
 3705 
 3706   /* Two optional cases in which clearing not necessary */
 3707   if (chunk_is_mmapped (p))
 3708     {
 3709       if (__builtin_expect (perturb_byte, 0))
 3710         return memset (mem, 0, sz);
 3711 
 3712       return mem;
 3713     }
 3714 
 3715 #if MORECORE_CLEARS
 3716   if (perturb_byte == 0 && (p == oldtop && csz > oldtopsize))
 3717     {
 3718       /* clear only the bytes from non-freshly-sbrked memory */
 3719       csz = oldtopsize;
 3720     }
 3721 #endif
 3722 
 3723   /* Unroll clear of <= 36 bytes (72 if 8byte sizes).  We know that
 3724      contents have an odd number of INTERNAL_SIZE_T-sized words;
 3725      minimally 3.  */
 3726   d = (INTERNAL_SIZE_T *) mem;
 3727   clearsize = csz - SIZE_SZ;
 3728   nclears = clearsize / sizeof (INTERNAL_SIZE_T);
 3729   assert (nclears >= 3);
 3730 
 3731   if (nclears > 9)
 3732     return memset (d, 0, clearsize);
 3733 
 3734   else
 3735     {
 3736       *(d + 0) = 0;
 3737       *(d + 1) = 0;
 3738       *(d + 2) = 0;
 3739       if (nclears > 4)
 3740         {
 3741           *(d + 3) = 0;
 3742           *(d + 4) = 0;
 3743           if (nclears > 6)
 3744             {
 3745               *(d + 5) = 0;
 3746               *(d + 6) = 0;
 3747               if (nclears > 8)
 3748                 {
 3749                   *(d + 7) = 0;
 3750                   *(d + 8) = 0;
 3751                 }
 3752             }
 3753         }
 3754     }
 3755 
 3756   return mem;
 3757 }
 3758 #endif /* IS_IN (libc) */
 3759 
 3760 /*
 3761    ------------------------------ malloc ------------------------------
 3762  */
 3763 
 3764 static void *
 3765 _int_malloc (mstate av, size_t bytes)
 3766 {
 3767   INTERNAL_SIZE_T nb;               /* normalized request size */
 3768   unsigned int idx;                 /* associated bin index */
 3769   mbinptr bin;                      /* associated bin */
 3770 
 3771   mchunkptr victim;                 /* inspected/selected chunk */
 3772   INTERNAL_SIZE_T size;             /* its size */
 3773   int victim_index;                 /* its bin index */
 3774 
 3775   mchunkptr remainder;              /* remainder from a split */
 3776   unsigned long remainder_size;     /* its size */
 3777 
 3778   unsigned int block;               /* bit map traverser */
 3779   unsigned int bit;                 /* bit map traverser */
 3780   unsigned int map;                 /* current word of binmap */
 3781 
 3782   mchunkptr fwd;                    /* misc temp for linking */
 3783   mchunkptr bck;                    /* misc temp for linking */
 3784 
 3785 #if USE_TCACHE
 3786   size_t tcache_unsorted_count;     /* count of unsorted chunks processed */
 3787 #endif
 3788 
 3789   /*
 3790      Convert request size to internal form by adding SIZE_SZ bytes
 3791      overhead plus possibly more to obtain necessary alignment and/or
 3792      to obtain a size of at least MINSIZE, the smallest allocatable
 3793      size. Also, checked_request2size returns false for request sizes
 3794      that are so large that they wrap around zero when padded and
 3795      aligned.
 3796    */
 3797 
 3798   nb = checked_request2size (bytes);
 3799   if (nb == 0)
 3800     {
 3801       __set_errno (ENOMEM);
 3802       return NULL;
 3803     }
 3804 
 3805   /* There are no usable arenas.  Fall back to sysmalloc to get a chunk from
 3806      mmap.  */
 3807   if (__glibc_unlikely (av == NULL))
 3808     {
 3809       void *p = sysmalloc (nb, av);
 3810       if (p != NULL)
 3811     alloc_perturb (p, bytes);
 3812       return p;
 3813     }
 3814 
 3815   /*
 3816      If the size qualifies as a fastbin, first check corresponding bin.
 3817      This code is safe to execute even if av is not yet initialized, so we
 3818      can try it without checking, which saves some time on this fast path.
 3819    */
 3820 
 3821 #define REMOVE_FB(fb, victim, pp)           \
 3822   do                            \
 3823     {                           \
 3824       victim = pp;                  \
 3825       if (victim == NULL)               \
 3826     break;                      \
 3827       pp = REVEAL_PTR (victim->fd);                                     \
 3828       if (__glibc_unlikely (pp != NULL && misaligned_chunk (pp)))       \
 3829     malloc_printerr ("malloc(): unaligned fastbin chunk detected"); \
 3830     }                           \
 3831   while ((pp = catomic_compare_and_exchange_val_acq (fb, pp, victim)) \
 3832      != victim);                    \
 3833 
 3834   if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
 3835     {
 3836       idx = fastbin_index (nb);
 3837       mfastbinptr *fb = &fastbin (av, idx);
 3838       mchunkptr pp;
 3839       victim = *fb;
 3840 
 3841       if (victim != NULL)
 3842     {
 3843       if (__glibc_unlikely (misaligned_chunk (victim)))
 3844         malloc_printerr ("malloc(): unaligned fastbin chunk detected 2");
 3845 
 3846       if (SINGLE_THREAD_P)
 3847         *fb = REVEAL_PTR (victim->fd);
 3848       else
 3849         REMOVE_FB (fb, pp, victim);
 3850       if (__glibc_likely (victim != NULL))
 3851         {
 3852           size_t victim_idx = fastbin_index (chunksize (victim));
 3853           if (__builtin_expect (victim_idx != idx, 0))
 3854         malloc_printerr ("malloc(): memory corruption (fast)");
 3855           check_remalloced_chunk (av, victim, nb);
 3856 #if USE_TCACHE
 3857           /* While we're here, if we see other chunks of the same size,
 3858          stash them in the tcache.  */
 3859           size_t tc_idx = csize2tidx (nb);
 3860           if (tcache && tc_idx < mp_.tcache_bins)
 3861         {
 3862           mchunkptr tc_victim;
 3863 
 3864           /* While bin not empty and tcache not full, copy chunks.  */
 3865           while (tcache->counts[tc_idx] < mp_.tcache_count
 3866              && (tc_victim = *fb) != NULL)
 3867             {
 3868               if (__glibc_unlikely (misaligned_chunk (tc_victim)))
 3869             malloc_printerr ("malloc(): unaligned fastbin chunk detected 3");
 3870               if (SINGLE_THREAD_P)
 3871             *fb = REVEAL_PTR (tc_victim->fd);
 3872               else
 3873             {
 3874               REMOVE_FB (fb, pp, tc_victim);
 3875               if (__glibc_unlikely (tc_victim == NULL))
 3876                 break;
 3877             }
 3878               tcache_put (tc_victim, tc_idx);
 3879             }
 3880         }
 3881 #endif
 3882           void *p = chunk2mem (victim);
 3883           alloc_perturb (p, bytes);
 3884           return p;
 3885         }
 3886     }
 3887     }
 3888 
 3889   /*
 3890      If a small request, check regular bin.  Since these "smallbins"
 3891      hold one size each, no searching within bins is necessary.
 3892      (For a large request, we need to wait until unsorted chunks are
 3893      processed to find best fit. But for small ones, fits are exact
 3894      anyway, so we can check now, which is faster.)
 3895    */
 3896 
 3897   if (in_smallbin_range (nb))
 3898     {
 3899       idx = smallbin_index (nb);
 3900       bin = bin_at (av, idx);
 3901 
 3902       if ((victim = last (bin)) != bin)
 3903         {
 3904           bck = victim->bk;
 3905       if (__glibc_unlikely (bck->fd != victim))
 3906         malloc_printerr ("malloc(): smallbin double linked list corrupted");
 3907           set_inuse_bit_at_offset (victim, nb);
 3908           bin->bk = bck;
 3909           bck->fd = bin;
 3910 
 3911           if (av != &main_arena)
 3912         set_non_main_arena (victim);
 3913           check_malloced_chunk (av, victim, nb);
 3914 #if USE_TCACHE
 3915       /* While we're here, if we see other chunks of the same size,
 3916          stash them in the tcache.  */
 3917       size_t tc_idx = csize2tidx (nb);
 3918       if (tcache && tc_idx < mp_.tcache_bins)
 3919         {
 3920           mchunkptr tc_victim;
 3921 
 3922           /* While bin not empty and tcache not full, copy chunks over.  */
 3923           while (tcache->counts[tc_idx] < mp_.tcache_count
 3924              && (tc_victim = last (bin)) != bin)
 3925         {
 3926           if (tc_victim != 0)
 3927             {
 3928               bck = tc_victim->bk;
 3929               set_inuse_bit_at_offset (tc_victim, nb);
 3930               if (av != &main_arena)
 3931             set_non_main_arena (tc_victim);
 3932               bin->bk = bck;
 3933               bck->fd = bin;
 3934 
 3935               tcache_put (tc_victim, tc_idx);
 3936                 }
 3937         }
 3938         }
 3939 #endif
 3940           void *p = chunk2mem (victim);
 3941           alloc_perturb (p, bytes);
 3942           return p;
 3943         }
 3944     }
 3945 
 3946   /*
 3947      If this is a large request, consolidate fastbins before continuing.
 3948      While it might look excessive to kill all fastbins before
 3949      even seeing if there is space available, this avoids
 3950      fragmentation problems normally associated with fastbins.
 3951      Also, in practice, programs tend to have runs of either small or
 3952      large requests, but less often mixtures, so consolidation is not
 3953      invoked all that often in most programs. And the programs that
 3954      it is called frequently in otherwise tend to fragment.
 3955    */
 3956 
 3957   else
 3958     {
 3959       idx = largebin_index (nb);
 3960       if (atomic_load_relaxed (&av->have_fastchunks))
 3961         malloc_consolidate (av);
 3962     }
 3963 
 3964   /*
 3965      Process recently freed or remaindered chunks, taking one only if
 3966      it is exact fit, or, if this a small request, the chunk is remainder from
 3967      the most recent non-exact fit.  Place other traversed chunks in
 3968      bins.  Note that this step is the only place in any routine where
 3969      chunks are placed in bins.
 3970 
 3971      The outer loop here is needed because we might not realize until
 3972      near the end of malloc that we should have consolidated, so must
 3973      do so and retry. This happens at most once, and only when we would
 3974      otherwise need to expand memory to service a "small" request.
 3975    */
 3976 
 3977 #if USE_TCACHE
 3978   INTERNAL_SIZE_T tcache_nb = 0;
 3979   size_t tc_idx = csize2tidx (nb);
 3980   if (tcache && tc_idx < mp_.tcache_bins)
 3981     tcache_nb = nb;
 3982   int return_cached = 0;
 3983 
 3984   tcache_unsorted_count = 0;
 3985 #endif
 3986 
 3987   for (;; )
 3988     {
 3989       int iters = 0;
 3990       while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av))
 3991         {
 3992           bck = victim->bk;
 3993           size = chunksize (victim);
 3994           mchunkptr next = chunk_at_offset (victim, size);
 3995 
 3996           if (__glibc_unlikely (size <= CHUNK_HDR_SZ)
 3997               || __glibc_unlikely (size > av->system_mem))
 3998             malloc_printerr ("malloc(): invalid size (unsorted)");
 3999           if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ)
 4000               || __glibc_unlikely (chunksize_nomask (next) > av->system_mem))
 4001             malloc_printerr ("malloc(): invalid next size (unsorted)");
 4002           if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size))
 4003             malloc_printerr ("malloc(): mismatching next->prev_size (unsorted)");
 4004           if (__glibc_unlikely (bck->fd != victim)
 4005               || __glibc_unlikely (victim->fd != unsorted_chunks (av)))
 4006             malloc_printerr ("malloc(): unsorted double linked list corrupted");
 4007           if (__glibc_unlikely (prev_inuse (next)))
 4008             malloc_printerr ("malloc(): invalid next->prev_inuse (unsorted)");
 4009 
 4010           /*
 4011              If a small request, try to use last remainder if it is the
 4012              only chunk in unsorted bin.  This helps promote locality for
 4013              runs of consecutive small requests. This is the only
 4014              exception to best-fit, and applies only when there is
 4015              no exact fit for a small chunk.
 4016            */
 4017 
 4018           if (in_smallbin_range (nb) &&
 4019               bck == unsorted_chunks (av) &&
 4020               victim == av->last_remainder &&
 4021               (unsigned long) (size) > (unsigned long) (nb + MINSIZE))
 4022             {
 4023               /* split and reattach remainder */
 4024               remainder_size = size - nb;
 4025               remainder = chunk_at_offset (victim, nb);
 4026               unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder;
 4027               av->last_remainder = remainder;
 4028               remainder->bk = remainder->fd = unsorted_chunks (av);
 4029               if (!in_smallbin_range (remainder_size))
 4030                 {
 4031                   remainder->fd_nextsize = NULL;
 4032                   remainder->bk_nextsize = NULL;
 4033                 }
 4034 
 4035               set_head (victim, nb | PREV_INUSE |
 4036                         (av != &main_arena ? NON_MAIN_ARENA : 0));
 4037               set_head (remainder, remainder_size | PREV_INUSE);
 4038               set_foot (remainder, remainder_size);
 4039 
 4040               check_malloced_chunk (av, victim, nb);
 4041               void *p = chunk2mem (victim);
 4042               alloc_perturb (p, bytes);
 4043               return p;
 4044             }
 4045 
 4046           /* remove from unsorted list */
 4047           if (__glibc_unlikely (bck->fd != victim))
 4048             malloc_printerr ("malloc(): corrupted unsorted chunks 3");
 4049           unsorted_chunks (av)->bk = bck;
 4050           bck->fd = unsorted_chunks (av);
 4051 
 4052           /* Take now instead of binning if exact fit */
 4053 
 4054           if (size == nb)
 4055             {
 4056               set_inuse_bit_at_offset (victim, size);
 4057               if (av != &main_arena)
 4058         set_non_main_arena (victim);
 4059 #if USE_TCACHE
 4060           /* Fill cache first, return to user only if cache fills.
 4061          We may return one of these chunks later.  */
 4062           if (tcache_nb
 4063           && tcache->counts[tc_idx] < mp_.tcache_count)
 4064         {
 4065           tcache_put (victim, tc_idx);
 4066           return_cached = 1;
 4067           continue;
 4068         }
 4069           else
 4070         {
 4071 #endif
 4072               check_malloced_chunk (av, victim, nb);
 4073               void *p = chunk2mem (victim);
 4074               alloc_perturb (p, bytes);
 4075               return p;
 4076 #if USE_TCACHE
 4077         }
 4078 #endif
 4079             }
 4080 
 4081           /* place chunk in bin */
 4082 
 4083           if (in_smallbin_range (size))
 4084             {
 4085               victim_index = smallbin_index (size);
 4086               bck = bin_at (av, victim_index);
 4087               fwd = bck->fd;
 4088             }
 4089           else
 4090             {
 4091               victim_index = largebin_index (size);
 4092               bck = bin_at (av, victim_index);
 4093               fwd = bck->fd;
 4094 
 4095               /* maintain large bins in sorted order */
 4096               if (fwd != bck)
 4097                 {
 4098                   /* Or with inuse bit to speed comparisons */
 4099                   size |= PREV_INUSE;
 4100                   /* if smaller than smallest, bypass loop below */
 4101                   assert (chunk_main_arena (bck->bk));
 4102                   if ((unsigned long) (size)
 4103               < (unsigned long) chunksize_nomask (bck->bk))
 4104                     {
 4105                       fwd = bck;
 4106                       bck = bck->bk;
 4107 
 4108                       victim->fd_nextsize = fwd->fd;
 4109                       victim->bk_nextsize = fwd->fd->bk_nextsize;
 4110                       fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim;
 4111                     }
 4112                   else
 4113                     {
 4114                       assert (chunk_main_arena (fwd));
 4115                       while ((unsigned long) size < chunksize_nomask (fwd))
 4116                         {
 4117                           fwd = fwd->fd_nextsize;
 4118               assert (chunk_main_arena (fwd));
 4119                         }
 4120 
 4121                       if ((unsigned long) size
 4122               == (unsigned long) chunksize_nomask (fwd))
 4123                         /* Always insert in the second position.  */
 4124                         fwd = fwd->fd;
 4125                       else
 4126                         {
 4127                           victim->fd_nextsize = fwd;
 4128                           victim->bk_nextsize = fwd->bk_nextsize;
 4129                           if (__glibc_unlikely (fwd->bk_nextsize->fd_nextsize != fwd))
 4130                             malloc_printerr ("malloc(): largebin double linked list corrupted (nextsize)");
 4131                           fwd->bk_nextsize = victim;
 4132                           victim->bk_nextsize->fd_nextsize = victim;
 4133                         }
 4134                       bck = fwd->bk;
 4135                       if (bck->fd != fwd)
 4136                         malloc_printerr ("malloc(): largebin double linked list corrupted (bk)");
 4137                     }
 4138                 }
 4139               else
 4140                 victim->fd_nextsize = victim->bk_nextsize = victim;
 4141             }
 4142 
 4143           mark_bin (av, victim_index);
 4144           victim->bk = bck;
 4145           victim->fd = fwd;
 4146           fwd->bk = victim;
 4147           bck->fd = victim;
 4148 
 4149 #if USE_TCACHE
 4150       /* If we've processed as many chunks as we're allowed while
 4151      filling the cache, return one of the cached ones.  */
 4152       ++tcache_unsorted_count;
 4153       if (return_cached
 4154       && mp_.tcache_unsorted_limit > 0
 4155       && tcache_unsorted_count > mp_.tcache_unsorted_limit)
 4156     {
 4157       return tcache_get (tc_idx);
 4158     }
 4159 #endif
 4160 
 4161 #define MAX_ITERS       10000
 4162           if (++iters >= MAX_ITERS)
 4163             break;
 4164         }
 4165 
 4166 #if USE_TCACHE
 4167       /* If all the small chunks we found ended up cached, return one now.  */
 4168       if (return_cached)
 4169     {
 4170       return tcache_get (tc_idx);
 4171     }
 4172 #endif
 4173 
 4174       /*
 4175          If a large request, scan through the chunks of current bin in
 4176          sorted order to find smallest that fits.  Use the skip list for this.
 4177        */
 4178 
 4179       if (!in_smallbin_range (nb))
 4180         {
 4181           bin = bin_at (av, idx);
 4182 
 4183           /* skip scan if empty or largest chunk is too small */
 4184           if ((victim = first (bin)) != bin
 4185           && (unsigned long) chunksize_nomask (victim)
 4186             >= (unsigned long) (nb))
 4187             {
 4188               victim = victim->bk_nextsize;
 4189               while (((unsigned long) (size = chunksize (victim)) <
 4190                       (unsigned long) (nb)))
 4191                 victim = victim->bk_nextsize;
 4192 
 4193               /* Avoid removing the first entry for a size so that the skip
 4194                  list does not have to be rerouted.  */
 4195               if (victim != last (bin)
 4196           && chunksize_nomask (victim)
 4197             == chunksize_nomask (victim->fd))
 4198                 victim = victim->fd;
 4199 
 4200               remainder_size = size - nb;
 4201               unlink_chunk (av, victim);
 4202 
 4203               /* Exhaust */
 4204               if (remainder_size < MINSIZE)
 4205                 {
 4206                   set_inuse_bit_at_offset (victim, size);
 4207                   if (av != &main_arena)
 4208             set_non_main_arena (victim);
 4209                 }
 4210               /* Split */
 4211               else
 4212                 {
 4213                   remainder = chunk_at_offset (victim, nb);
 4214                   /* We cannot assume the unsorted list is empty and therefore
 4215                      have to perform a complete insert here.  */
 4216                   bck = unsorted_chunks (av);
 4217                   fwd = bck->fd;
 4218           if (__glibc_unlikely (fwd->bk != bck))
 4219             malloc_printerr ("malloc(): corrupted unsorted chunks");
 4220                   remainder->bk = bck;
 4221                   remainder->fd = fwd;
 4222                   bck->fd = remainder;
 4223                   fwd->bk = remainder;
 4224                   if (!in_smallbin_range (remainder_size))
 4225                     {
 4226                       remainder->fd_nextsize = NULL;
 4227                       remainder->bk_nextsize = NULL;
 4228                     }
 4229                   set_head (victim, nb | PREV_INUSE |
 4230                             (av != &main_arena ? NON_MAIN_ARENA : 0));
 4231                   set_head (remainder, remainder_size | PREV_INUSE);
 4232                   set_foot (remainder, remainder_size);
 4233                 }
 4234               check_malloced_chunk (av, victim, nb);
 4235               void *p = chunk2mem (victim);
 4236               alloc_perturb (p, bytes);
 4237               return p;
 4238             }
 4239         }
 4240 
 4241       /*
 4242          Search for a chunk by scanning bins, starting with next largest
 4243          bin. This search is strictly by best-fit; i.e., the smallest
 4244          (with ties going to approximately the least recently used) chunk
 4245          that fits is selected.
 4246 
 4247          The bitmap avoids needing to check that most blocks are nonempty.
 4248          The particular case of skipping all bins during warm-up phases
 4249          when no chunks have been returned yet is faster than it might look.
 4250        */
 4251 
 4252       ++idx;
 4253       bin = bin_at (av, idx);
 4254       block = idx2block (idx);
 4255       map = av->binmap[block];
 4256       bit = idx2bit (idx);
 4257 
 4258       for (;; )
 4259         {
 4260           /* Skip rest of block if there are no more set bits in this block.  */
 4261           if (bit > map || bit == 0)
 4262             {
 4263               do
 4264                 {
 4265                   if (++block >= BINMAPSIZE) /* out of bins */
 4266                     goto use_top;
 4267                 }
 4268               while ((map = av->binmap[block]) == 0);
 4269 
 4270               bin = bin_at (av, (block << BINMAPSHIFT));
 4271               bit = 1;
 4272             }
 4273 
 4274           /* Advance to bin with set bit. There must be one. */
 4275           while ((bit & map) == 0)
 4276             {
 4277               bin = next_bin (bin);
 4278               bit <<= 1;
 4279               assert (bit != 0);
 4280             }
 4281 
 4282           /* Inspect the bin. It is likely to be non-empty */
 4283           victim = last (bin);
 4284 
 4285           /*  If a false alarm (empty bin), clear the bit. */
 4286           if (victim == bin)
 4287             {
 4288               av->binmap[block] = map &= ~bit; /* Write through */
 4289               bin = next_bin (bin);
 4290               bit <<= 1;
 4291             }
 4292 
 4293           else
 4294             {
 4295               size = chunksize (victim);
 4296 
 4297               /*  We know the first chunk in this bin is big enough to use. */
 4298               assert ((unsigned long) (size) >= (unsigned long) (nb));
 4299 
 4300               remainder_size = size - nb;
 4301 
 4302               /* unlink */
 4303               unlink_chunk (av, victim);
 4304 
 4305               /* Exhaust */
 4306               if (remainder_size < MINSIZE)
 4307                 {
 4308                   set_inuse_bit_at_offset (victim, size);
 4309                   if (av != &main_arena)
 4310             set_non_main_arena (victim);
 4311                 }
 4312 
 4313               /* Split */
 4314               else
 4315                 {
 4316                   remainder = chunk_at_offset (victim, nb);
 4317 
 4318                   /* We cannot assume the unsorted list is empty and therefore
 4319                      have to perform a complete insert here.  */
 4320                   bck = unsorted_chunks (av);
 4321                   fwd = bck->fd;
 4322           if (__glibc_unlikely (fwd->bk != bck))
 4323             malloc_printerr ("malloc(): corrupted unsorted chunks 2");
 4324                   remainder->bk = bck;
 4325                   remainder->fd = fwd;
 4326                   bck->fd = remainder;
 4327                   fwd->bk = remainder;
 4328 
 4329                   /* advertise as last remainder */
 4330                   if (in_smallbin_range (nb))
 4331                     av->last_remainder = remainder;
 4332                   if (!in_smallbin_range (remainder_size))
 4333                     {
 4334                       remainder->fd_nextsize = NULL;
 4335                       remainder->bk_nextsize = NULL;
 4336                     }
 4337                   set_head (victim, nb | PREV_INUSE |
 4338                             (av != &main_arena ? NON_MAIN_ARENA : 0));
 4339                   set_head (remainder, remainder_size | PREV_INUSE);
 4340                   set_foot (remainder, remainder_size);
 4341                 }
 4342               check_malloced_chunk (av, victim, nb);
 4343               void *p = chunk2mem (victim);
 4344               alloc_perturb (p, bytes);
 4345               return p;
 4346             }
 4347         }
 4348 
 4349     use_top:
 4350       /*
 4351          If large enough, split off the chunk bordering the end of memory
 4352          (held in av->top). Note that this is in accord with the best-fit
 4353          search rule.  In effect, av->top is treated as larger (and thus
 4354          less well fitting) than any other available chunk since it can
 4355          be extended to be as large as necessary (up to system
 4356          limitations).
 4357 
 4358          We require that av->top always exists (i.e., has size >=
 4359          MINSIZE) after initialization, so if it would otherwise be
 4360          exhausted by current request, it is replenished. (The main
 4361          reason for ensuring it exists is that we may need MINSIZE space
 4362          to put in fenceposts in sysmalloc.)
 4363        */
 4364 
 4365       victim = av->top;
 4366       size = chunksize (victim);
 4367 
 4368       if (__glibc_unlikely (size > av->system_mem))
 4369         malloc_printerr ("malloc(): corrupted top size");
 4370 
 4371       if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
 4372         {
 4373           remainder_size = size - nb;
 4374           remainder = chunk_at_offset (victim, nb);
 4375           av->top = remainder;
 4376           set_head (victim, nb | PREV_INUSE |
 4377                     (av != &main_arena ? NON_MAIN_ARENA : 0));
 4378           set_head (remainder, remainder_size | PREV_INUSE);
 4379 
 4380           check_malloced_chunk (av, victim, nb);
 4381           void *p = chunk2mem (victim);
 4382           alloc_perturb (p, bytes);
 4383           return p;
 4384         }
 4385 
 4386       /* When we are using atomic ops to free fast chunks we can get
 4387          here for all block sizes.  */
 4388       else if (atomic_load_relaxed (&av->have_fastchunks))
 4389         {
 4390           malloc_consolidate (av);
 4391           /* restore original bin index */
 4392           if (in_smallbin_range (nb))
 4393             idx = smallbin_index (nb);
 4394           else
 4395             idx = largebin_index (nb);
 4396         }
 4397 
 4398       /*
 4399          Otherwise, relay to handle system-dependent cases
 4400        */
 4401       else
 4402         {
 4403           void *p = sysmalloc (nb, av);
 4404           if (p != NULL)
 4405             alloc_perturb (p, bytes);
 4406           return p;
 4407         }
 4408     }
 4409 }
 4410 
 4411 /*
 4412    ------------------------------ free ------------------------------
 4413  */
 4414 
 4415 static void
 4416 _int_free (mstate av, mchunkptr p, int have_lock)
 4417 {
 4418   INTERNAL_SIZE_T size;        /* its size */
 4419   mfastbinptr *fb;             /* associated fastbin */
 4420   mchunkptr nextchunk;         /* next contiguous chunk */
 4421   INTERNAL_SIZE_T nextsize;    /* its size */
 4422   int nextinuse;               /* true if nextchunk is used */
 4423   INTERNAL_SIZE_T prevsize;    /* size of previous contiguous chunk */
 4424   mchunkptr bck;               /* misc temp for linking */
 4425   mchunkptr fwd;               /* misc temp for linking */
 4426 
 4427   size = chunksize (p);
 4428 
 4429   /* Little security check which won't hurt performance: the
 4430      allocator never wrapps around at the end of the address space.
 4431      Therefore we can exclude some size values which might appear
 4432      here by accident or by "design" from some intruder.  */
 4433   if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, 0)
 4434       || __builtin_expect (misaligned_chunk (p), 0))
 4435     malloc_printerr ("free(): invalid pointer");
 4436   /* We know that each chunk is at least MINSIZE bytes in size or a
 4437      multiple of MALLOC_ALIGNMENT.  */
 4438   if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size)))
 4439     malloc_printerr ("free(): invalid size");
 4440 
 4441   check_inuse_chunk(av, p);
 4442 
 4443 #if USE_TCACHE
 4444   {
 4445     size_t tc_idx = csize2tidx (size);
 4446     if (tcache != NULL && tc_idx < mp_.tcache_bins)
 4447       {
 4448     /* Check to see if it's already in the tcache.  */
 4449     tcache_entry *e = (tcache_entry *) chunk2mem (p);
 4450 
 4451     /* This test succeeds on double free.  However, we don't 100%
 4452        trust it (it also matches random payload data at a 1 in
 4453        2^<size_t> chance), so verify it's not an unlikely
 4454        coincidence before aborting.  */
 4455     if (__glibc_unlikely (e->key == tcache_key))
 4456       {
 4457         tcache_entry *tmp;
 4458         size_t cnt = 0;
 4459         LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx);
 4460         for (tmp = tcache->entries[tc_idx];
 4461          tmp;
 4462          tmp = REVEAL_PTR (tmp->next), ++cnt)
 4463           {
 4464         if (cnt >= mp_.tcache_count)
 4465           malloc_printerr ("free(): too many chunks detected in tcache");
 4466         if (__glibc_unlikely (!aligned_OK (tmp)))
 4467           malloc_printerr ("free(): unaligned chunk detected in tcache 2");
 4468         if (tmp == e)
 4469           malloc_printerr ("free(): double free detected in tcache 2");
 4470         /* If we get here, it was a coincidence.  We've wasted a
 4471            few cycles, but don't abort.  */
 4472           }
 4473       }
 4474 
 4475     if (tcache->counts[tc_idx] < mp_.tcache_count)
 4476       {
 4477         tcache_put (p, tc_idx);
 4478         return;
 4479       }
 4480       }
 4481   }
 4482 #endif
 4483 
 4484   /*
 4485     If eligible, place chunk on a fastbin so it can be found
 4486     and used quickly in malloc.
 4487   */
 4488 
 4489   if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
 4490 
 4491 #if TRIM_FASTBINS
 4492       /*
 4493     If TRIM_FASTBINS set, don't place chunks
 4494     bordering top into fastbins
 4495       */
 4496       && (chunk_at_offset(p, size) != av->top)
 4497 #endif
 4498       ) {
 4499 
 4500     if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size))
 4501               <= CHUNK_HDR_SZ, 0)
 4502     || __builtin_expect (chunksize (chunk_at_offset (p, size))
 4503                  >= av->system_mem, 0))
 4504       {
 4505     bool fail = true;
 4506     /* We might not have a lock at this point and concurrent modifications
 4507        of system_mem might result in a false positive.  Redo the test after
 4508        getting the lock.  */
 4509     if (!have_lock)
 4510       {
 4511         __libc_lock_lock (av->mutex);
 4512         fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ
 4513             || chunksize (chunk_at_offset (p, size)) >= av->system_mem);
 4514         __libc_lock_unlock (av->mutex);
 4515       }
 4516 
 4517     if (fail)
 4518       malloc_printerr ("free(): invalid next size (fast)");
 4519       }
 4520 
 4521     free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
 4522 
 4523     atomic_store_relaxed (&av->have_fastchunks, true);
 4524     unsigned int idx = fastbin_index(size);
 4525     fb = &fastbin (av, idx);
 4526 
 4527     /* Atomically link P to its fastbin: P->FD = *FB; *FB = P;  */
 4528     mchunkptr old = *fb, old2;
 4529 
 4530     if (SINGLE_THREAD_P)
 4531       {
 4532     /* Check that the top of the bin is not the record we are going to
 4533        add (i.e., double free).  */
 4534     if (__builtin_expect (old == p, 0))
 4535       malloc_printerr ("double free or corruption (fasttop)");
 4536     p->fd = PROTECT_PTR (&p->fd, old);
 4537     *fb = p;
 4538       }
 4539     else
 4540       do
 4541     {
 4542       /* Check that the top of the bin is not the record we are going to
 4543          add (i.e., double free).  */
 4544       if (__builtin_expect (old == p, 0))
 4545         malloc_printerr ("double free or corruption (fasttop)");
 4546       old2 = old;
 4547       p->fd = PROTECT_PTR (&p->fd, old);
 4548     }
 4549       while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2))
 4550          != old2);
 4551 
 4552     /* Check that size of fastbin chunk at the top is the same as
 4553        size of the chunk that we are adding.  We can dereference OLD
 4554        only if we have the lock, otherwise it might have already been
 4555        allocated again.  */
 4556     if (have_lock && old != NULL
 4557     && __builtin_expect (fastbin_index (chunksize (old)) != idx, 0))
 4558       malloc_printerr ("invalid fastbin entry (free)");
 4559   }
 4560 
 4561   /*
 4562     Consolidate other non-mmapped chunks as they arrive.
 4563   */
 4564 
 4565   else if (!chunk_is_mmapped(p)) {
 4566 
 4567     /* If we're single-threaded, don't lock the arena.  */
 4568     if (SINGLE_THREAD_P)
 4569       have_lock = true;
 4570 
 4571     if (!have_lock)
 4572       __libc_lock_lock (av->mutex);
 4573 
 4574     nextchunk = chunk_at_offset(p, size);
 4575 
 4576     /* Lightweight tests: check whether the block is already the
 4577        top block.  */
 4578     if (__glibc_unlikely (p == av->top))
 4579       malloc_printerr ("double free or corruption (top)");
 4580     /* Or whether the next chunk is beyond the boundaries of the arena.  */
 4581     if (__builtin_expect (contiguous (av)
 4582               && (char *) nextchunk
 4583               >= ((char *) av->top + chunksize(av->top)), 0))
 4584     malloc_printerr ("double free or corruption (out)");
 4585     /* Or whether the block is actually not marked used.  */
 4586     if (__glibc_unlikely (!prev_inuse(nextchunk)))
 4587       malloc_printerr ("double free or corruption (!prev)");
 4588 
 4589     nextsize = chunksize(nextchunk);
 4590     if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0)
 4591     || __builtin_expect (nextsize >= av->system_mem, 0))
 4592       malloc_printerr ("free(): invalid next size (normal)");
 4593 
 4594     free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
 4595 
 4596     /* consolidate backward */
 4597     if (!prev_inuse(p)) {
 4598       prevsize = prev_size (p);
 4599       size += prevsize;
 4600       p = chunk_at_offset(p, -((long) prevsize));
 4601       if (__glibc_unlikely (chunksize(p) != prevsize))
 4602         malloc_printerr ("corrupted size vs. prev_size while consolidating");
 4603       unlink_chunk (av, p);
 4604     }
 4605 
 4606     if (nextchunk != av->top) {
 4607       /* get and clear inuse bit */
 4608       nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
 4609 
 4610       /* consolidate forward */
 4611       if (!nextinuse) {
 4612     unlink_chunk (av, nextchunk);
 4613     size += nextsize;
 4614       } else
 4615     clear_inuse_bit_at_offset(nextchunk, 0);
 4616 
 4617       /*
 4618     Place the chunk in unsorted chunk list. Chunks are
 4619     not placed into regular bins until after they have
 4620     been given one chance to be used in malloc.
 4621       */
 4622 
 4623       bck = unsorted_chunks(av);
 4624       fwd = bck->fd;
 4625       if (__glibc_unlikely (fwd->bk != bck))
 4626     malloc_printerr ("free(): corrupted unsorted chunks");
 4627       p->fd = fwd;
 4628       p->bk = bck;
 4629       if (!in_smallbin_range(size))
 4630     {
 4631       p->fd_nextsize = NULL;
 4632       p->bk_nextsize = NULL;
 4633     }
 4634       bck->fd = p;
 4635       fwd->bk = p;
 4636 
 4637       set_head(p, size | PREV_INUSE);
 4638       set_foot(p, size);
 4639 
 4640       check_free_chunk(av, p);
 4641     }
 4642 
 4643     /*
 4644       If the chunk borders the current high end of memory,
 4645       consolidate into top
 4646     */
 4647 
 4648     else {
 4649       size += nextsize;
 4650       set_head(p, size | PREV_INUSE);
 4651       av->top = p;
 4652       check_chunk(av, p);
 4653     }
 4654 
 4655     /*
 4656       If freeing a large space, consolidate possibly-surrounding
 4657       chunks. Then, if the total unused topmost memory exceeds trim
 4658       threshold, ask malloc_trim to reduce top.
 4659 
 4660       Unless max_fast is 0, we don't know if there are fastbins
 4661       bordering top, so we cannot tell for sure whether threshold
 4662       has been reached unless fastbins are consolidated.  But we
 4663       don't want to consolidate on each free.  As a compromise,
 4664       consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
 4665       is reached.
 4666     */
 4667 
 4668     if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
 4669       if (atomic_load_relaxed (&av->have_fastchunks))
 4670     malloc_consolidate(av);
 4671 
 4672       if (av == &main_arena) {
 4673 #ifndef MORECORE_CANNOT_TRIM
 4674     if ((unsigned long)(chunksize(av->top)) >=
 4675         (unsigned long)(mp_.trim_threshold))
 4676       systrim(mp_.top_pad, av);
 4677 #endif
 4678       } else {
 4679     /* Always try heap_trim(), even if the top chunk is not
 4680        large, because the corresponding heap might go away.  */
 4681     heap_info *heap = heap_for_ptr(top(av));
 4682 
 4683     assert(heap->ar_ptr == av);
 4684     heap_trim(heap, mp_.top_pad);
 4685       }
 4686     }
 4687 
 4688     if (!have_lock)
 4689       __libc_lock_unlock (av->mutex);
 4690   }
 4691   /*
 4692     If the chunk was allocated via mmap, release via munmap().
 4693   */
 4694 
 4695   else {
 4696     munmap_chunk (p);
 4697   }
 4698 }
 4699 
 4700 /*
 4701   ------------------------- malloc_consolidate -------------------------
 4702 
 4703   malloc_consolidate is a specialized version of free() that tears
 4704   down chunks held in fastbins.  Free itself cannot be used for this
 4705   purpose since, among other things, it might place chunks back onto
 4706   fastbins.  So, instead, we need to use a minor variant of the same
 4707   code.
 4708 */
 4709 
 4710 static void malloc_consolidate(mstate av)
 4711 {
 4712   mfastbinptr*    fb;                 /* current fastbin being consolidated */
 4713   mfastbinptr*    maxfb;              /* last fastbin (for loop control) */
 4714   mchunkptr       p;                  /* current chunk being consolidated */
 4715   mchunkptr       nextp;              /* next chunk to consolidate */
 4716   mchunkptr       unsorted_bin;       /* bin header */
 4717   mchunkptr       first_unsorted;     /* chunk to link to */
 4718 
 4719   /* These have same use as in free() */
 4720   mchunkptr       nextchunk;
 4721   INTERNAL_SIZE_T size;
 4722   INTERNAL_SIZE_T nextsize;
 4723   INTERNAL_SIZE_T prevsize;
 4724   int             nextinuse;
 4725 
 4726   atomic_store_relaxed (&av->have_fastchunks, false);
 4727 
 4728   unsorted_bin = unsorted_chunks(av);
 4729 
 4730   /*
 4731     Remove each chunk from fast bin and consolidate it, placing it
 4732     then in unsorted bin. Among other reasons for doing this,
 4733     placing in unsorted bin avoids needing to calculate actual bins
 4734     until malloc is sure that chunks aren't immediately going to be
 4735     reused anyway.
 4736   */
 4737 
 4738   maxfb = &fastbin (av, NFASTBINS - 1);
 4739   fb = &fastbin (av, 0);
 4740   do {
 4741     p = atomic_exchange_acq (fb, NULL);
 4742     if (p != 0) {
 4743       do {
 4744     {
 4745       if (__glibc_unlikely (misaligned_chunk (p)))
 4746         malloc_printerr ("malloc_consolidate(): "
 4747                  "unaligned fastbin chunk detected");
 4748 
 4749       unsigned int idx = fastbin_index (chunksize (p));
 4750       if ((&fastbin (av, idx)) != fb)
 4751         malloc_printerr ("malloc_consolidate(): invalid chunk size");
 4752     }
 4753 
 4754     check_inuse_chunk(av, p);
 4755     nextp = REVEAL_PTR (p->fd);
 4756 
 4757     /* Slightly streamlined version of consolidation code in free() */
 4758     size = chunksize (p);
 4759     nextchunk = chunk_at_offset(p, size);
 4760     nextsize = chunksize(nextchunk);
 4761 
 4762     if (!prev_inuse(p)) {
 4763       prevsize = prev_size (p);
 4764       size += prevsize;
 4765       p = chunk_at_offset(p, -((long) prevsize));
 4766       if (__glibc_unlikely (chunksize(p) != prevsize))
 4767         malloc_printerr ("corrupted size vs. prev_size in fastbins");
 4768       unlink_chunk (av, p);
 4769     }
 4770 
 4771     if (nextchunk != av->top) {
 4772       nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
 4773 
 4774       if (!nextinuse) {
 4775         size += nextsize;
 4776         unlink_chunk (av, nextchunk);
 4777       } else
 4778         clear_inuse_bit_at_offset(nextchunk, 0);
 4779 
 4780       first_unsorted = unsorted_bin->fd;
 4781       unsorted_bin->fd = p;
 4782       first_unsorted->bk = p;
 4783 
 4784       if (!in_smallbin_range (size)) {
 4785         p->fd_nextsize = NULL;
 4786         p->bk_nextsize = NULL;
 4787       }
 4788 
 4789       set_head(p, size | PREV_INUSE);
 4790       p->bk = unsorted_bin;
 4791       p->fd = first_unsorted;
 4792       set_foot(p, size);
 4793     }
 4794 
 4795     else {
 4796       size += nextsize;
 4797       set_head(p, size | PREV_INUSE);
 4798       av->top = p;
 4799     }
 4800 
 4801       } while ( (p = nextp) != 0);
 4802 
 4803     }
 4804   } while (fb++ != maxfb);
 4805 }
 4806 
 4807 /*
 4808   ------------------------------ realloc ------------------------------
 4809 */
 4810 
 4811 static void *
 4812 _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize,
 4813          INTERNAL_SIZE_T nb)
 4814 {
 4815   mchunkptr        newp;            /* chunk to return */
 4816   INTERNAL_SIZE_T  newsize;         /* its size */
 4817   void*          newmem;          /* corresponding user mem */
 4818 
 4819   mchunkptr        next;            /* next contiguous chunk after oldp */
 4820 
 4821   mchunkptr        remainder;       /* extra space at end of newp */
 4822   unsigned long    remainder_size;  /* its size */
 4823 
 4824   /* oldmem size */
 4825   if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, 0)
 4826       || __builtin_expect (oldsize >= av->system_mem, 0))
 4827     malloc_printerr ("realloc(): invalid old size");
 4828 
 4829   check_inuse_chunk (av, oldp);
 4830 
 4831   /* All callers already filter out mmap'ed chunks.  */
 4832   assert (!chunk_is_mmapped (oldp));
 4833 
 4834   next = chunk_at_offset (oldp, oldsize);
 4835   INTERNAL_SIZE_T nextsize = chunksize (next);
 4836   if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, 0)
 4837       || __builtin_expect (nextsize >= av->system_mem, 0))
 4838     malloc_printerr ("realloc(): invalid next size");
 4839 
 4840   if ((unsigned long) (oldsize) >= (unsigned long) (nb))
 4841     {
 4842       /* already big enough; split below */
 4843       newp = oldp;
 4844       newsize = oldsize;
 4845     }
 4846 
 4847   else
 4848     {
 4849       /* Try to expand forward into top */
 4850       if (next == av->top &&
 4851           (unsigned long) (newsize = oldsize + nextsize) >=
 4852           (unsigned long) (nb + MINSIZE))
 4853         {
 4854           set_head_size (oldp, nb | (av != &main_arena ? NON_MAIN_ARENA : 0));
 4855           av->top = chunk_at_offset (oldp, nb);
 4856           set_head (av->top, (newsize - nb) | PREV_INUSE);
 4857           check_inuse_chunk (av, oldp);
 4858           return tag_new_usable (chunk2mem (oldp));
 4859         }
 4860 
 4861       /* Try to expand forward into next chunk;  split off remainder below */
 4862       else if (next != av->top &&
 4863                !inuse (next) &&
 4864                (unsigned long) (newsize = oldsize + nextsize) >=
 4865                (unsigned long) (nb))
 4866         {
 4867           newp = oldp;
 4868           unlink_chunk (av, next);
 4869         }
 4870 
 4871       /* allocate, copy, free */
 4872       else
 4873         {
 4874           newmem = _int_malloc (av, nb - MALLOC_ALIGN_MASK);
 4875           if (newmem == 0)
 4876             return 0; /* propagate failure */
 4877 
 4878           newp = mem2chunk (newmem);
 4879           newsize = chunksize (newp);
 4880 
 4881           /*
 4882              Avoid copy if newp is next chunk after oldp.
 4883            */
 4884           if (newp == next)
 4885             {
 4886               newsize += oldsize;
 4887               newp = oldp;
 4888             }
 4889           else
 4890             {
 4891           void *oldmem = chunk2mem (oldp);
 4892           size_t sz = memsize (oldp);
 4893           (void) tag_region (oldmem, sz);
 4894           newmem = tag_new_usable (newmem);
 4895           memcpy (newmem, oldmem, sz);
 4896           _int_free (av, oldp, 1);
 4897           check_inuse_chunk (av, newp);
 4898           return newmem;
 4899             }
 4900         }
 4901     }
 4902 
 4903   /* If possible, free extra space in old or extended chunk */
 4904 
 4905   assert ((unsigned long) (newsize) >= (unsigned long) (nb));
 4906 
 4907   remainder_size = newsize - nb;
 4908 
 4909   if (remainder_size < MINSIZE)   /* not enough extra to split off */
 4910     {
 4911       set_head_size (newp, newsize | (av != &main_arena ? NON_MAIN_ARENA : 0));
 4912       set_inuse_bit_at_offset (newp, newsize);
 4913     }
 4914   else   /* split remainder */
 4915     {
 4916       remainder = chunk_at_offset (newp, nb);
 4917       /* Clear any user-space tags before writing the header.  */
 4918       remainder = tag_region (remainder, remainder_size);
 4919       set_head_size (newp, nb | (av != &main_arena ? NON_MAIN_ARENA : 0));
 4920       set_head (remainder, remainder_size | PREV_INUSE |
 4921                 (av != &main_arena ? NON_MAIN_ARENA : 0));
 4922       /* Mark remainder as inuse so free() won't complain */
 4923       set_inuse_bit_at_offset (remainder, remainder_size);
 4924       _int_free (av, remainder, 1);
 4925     }
 4926 
 4927   check_inuse_chunk (av, newp);
 4928   return tag_new_usable (chunk2mem (newp));
 4929 }
 4930 
 4931 /*
 4932    ------------------------------ memalign ------------------------------
 4933  */
 4934 
 4935 static void *
 4936 _int_memalign (mstate av, size_t alignment, size_t bytes)
 4937 {
 4938   INTERNAL_SIZE_T nb;             /* padded  request size */
 4939   char *m;                        /* memory returned by malloc call */
 4940   mchunkptr p;                    /* corresponding chunk */
 4941   char *brk;                      /* alignment point within p */
 4942   mchunkptr newp;                 /* chunk to return */
 4943   INTERNAL_SIZE_T newsize;        /* its size */
 4944   INTERNAL_SIZE_T leadsize;       /* leading space before alignment point */
 4945   mchunkptr remainder;            /* spare room at end to split off */
 4946   unsigned long remainder_size;   /* its size */
 4947   INTERNAL_SIZE_T size;
 4948 
 4949 
 4950 
 4951   nb = checked_request2size (bytes);
 4952   if (nb == 0)
 4953     {
 4954       __set_errno (ENOMEM);
 4955       return NULL;
 4956     }
 4957 
 4958   /*
 4959      Strategy: find a spot within that chunk that meets the alignment
 4960      request, and then possibly free the leading and trailing space.
 4961    */
 4962 
 4963   /* Call malloc with worst case padding to hit alignment. */
 4964 
 4965   m = (char *) (_int_malloc (av, nb + alignment + MINSIZE));
 4966 
 4967   if (m == 0)
 4968     return 0;           /* propagate failure */
 4969 
 4970   p = mem2chunk (m);
 4971 
 4972   if ((((unsigned long) (m)) % alignment) != 0)   /* misaligned */
 4973 
 4974     { /*
 4975                 Find an aligned spot inside chunk.  Since we need to give back
 4976                 leading space in a chunk of at least MINSIZE, if the first
 4977                 calculation places us at a spot with less than MINSIZE leader,
 4978                 we can move to the next aligned spot -- we've allocated enough
 4979                 total room so that this is always possible.
 4980                  */
 4981       brk = (char *) mem2chunk (((unsigned long) (m + alignment - 1)) &
 4982                                 - ((signed long) alignment));
 4983       if ((unsigned long) (brk - (char *) (p)) < MINSIZE)
 4984         brk += alignment;
 4985 
 4986       newp = (mchunkptr) brk;
 4987       leadsize = brk - (char *) (p);
 4988       newsize = chunksize (p) - leadsize;
 4989 
 4990       /* For mmapped chunks, just adjust offset */
 4991       if (chunk_is_mmapped (p))
 4992         {
 4993           set_prev_size (newp, prev_size (p) + leadsize);
 4994           set_head (newp, newsize | IS_MMAPPED);
 4995           return chunk2mem (newp);
 4996         }
 4997 
 4998       /* Otherwise, give back leader, use the rest */
 4999       set_head (newp, newsize | PREV_INUSE |
 5000                 (av != &main_arena ? NON_MAIN_ARENA : 0));
 5001       set_inuse_bit_at_offset (newp, newsize);
 5002       set_head_size (p, leadsize | (av != &main_arena ? NON_MAIN_ARENA : 0));
 5003       _int_free (av, p, 1);
 5004       p = newp;
 5005 
 5006       assert (newsize >= nb &&
 5007               (((unsigned long) (chunk2mem (p))) % alignment) == 0);
 5008     }
 5009 
 5010   /* Also give back spare room at the end */
 5011   if (!chunk_is_mmapped (p))
 5012     {
 5013       size = chunksize (p);
 5014       if ((unsigned long) (size) > (unsigned long) (nb + MINSIZE))
 5015         {
 5016           remainder_size = size - nb;
 5017           remainder = chunk_at_offset (p, nb);
 5018           set_head (remainder, remainder_size | PREV_INUSE |
 5019                     (av != &main_arena ? NON_MAIN_ARENA : 0));
 5020           set_head_size (p, nb);
 5021           _int_free (av, remainder, 1);
 5022         }
 5023     }
 5024 
 5025   check_inuse_chunk (av, p);
 5026   return chunk2mem (p);
 5027 }
 5028 
 5029 
 5030 /*
 5031    ------------------------------ malloc_trim ------------------------------
 5032  */
 5033 
 5034 static int
 5035 mtrim (mstate av, size_t pad)
 5036 {
 5037   /* Ensure all blocks are consolidated.  */
 5038   malloc_consolidate (av);
 5039 
 5040   const size_t ps = GLRO (dl_pagesize);
 5041   int psindex = bin_index (ps);
 5042   const size_t psm1 = ps - 1;
 5043 
 5044   int result = 0;
 5045   for (int i = 1; i < NBINS; ++i)
 5046     if (i == 1 || i >= psindex)
 5047       {
 5048         mbinptr bin = bin_at (av, i);
 5049 
 5050         for (mchunkptr p = last (bin); p != bin; p = p->bk)
 5051           {
 5052             INTERNAL_SIZE_T size = chunksize (p);
 5053 
 5054             if (size > psm1 + sizeof (struct malloc_chunk))
 5055               {
 5056                 /* See whether the chunk contains at least one unused page.  */
 5057                 char *paligned_mem = (char *) (((uintptr_t) p
 5058                                                 + sizeof (struct malloc_chunk)
 5059                                                 + psm1) & ~psm1);
 5060 
 5061                 assert ((char *) chunk2mem (p) + 2 * CHUNK_HDR_SZ
 5062             <= paligned_mem);
 5063                 assert ((char *) p + size > paligned_mem);
 5064 
 5065                 /* This is the size we could potentially free.  */
 5066                 size -= paligned_mem - (char *) p;
 5067 
 5068                 if (size > psm1)
 5069                   {
 5070 #if MALLOC_DEBUG
 5071                     /* When debugging we simulate destroying the memory
 5072                        content.  */
 5073                     memset (paligned_mem, 0x89, size & ~psm1);
 5074 #endif
 5075                     __madvise (paligned_mem, size & ~psm1, MADV_DONTNEED);
 5076 
 5077                     result = 1;
 5078                   }
 5079               }
 5080           }
 5081       }
 5082 
 5083 #ifndef MORECORE_CANNOT_TRIM
 5084   return result | (av == &main_arena ? systrim (pad, av) : 0);
 5085 
 5086 #else
 5087   return result;
 5088 #endif
 5089 }
 5090 
 5091 
 5092 int
 5093 __malloc_trim (size_t s)
 5094 {
 5095   int result = 0;
 5096 
 5097   if (!__malloc_initialized)
 5098     ptmalloc_init ();
 5099 
 5100   mstate ar_ptr = &main_arena;
 5101   do
 5102     {
 5103       __libc_lock_lock (ar_ptr->mutex);
 5104       result |= mtrim (ar_ptr, s);
 5105       __libc_lock_unlock (ar_ptr->mutex);
 5106 
 5107       ar_ptr = ar_ptr->next;
 5108     }
 5109   while (ar_ptr != &main_arena);
 5110 
 5111   return result;
 5112 }
 5113 
 5114 
 5115 /*
 5116    ------------------------- malloc_usable_size -------------------------
 5117  */
 5118 
 5119 static size_t
 5120 musable (void *mem)
 5121 {
 5122   mchunkptr p = mem2chunk (mem);
 5123 
 5124   if (chunk_is_mmapped (p))
 5125     return chunksize (p) - CHUNK_HDR_SZ;
 5126   else if (inuse (p))
 5127     return memsize (p);
 5128 
 5129   return 0;
 5130 }
 5131 
 5132 #if IS_IN (libc)
 5133 size_t
 5134 __malloc_usable_size (void *m)
 5135 {
 5136   if (m == NULL)
 5137     return 0;
 5138   return musable (m);
 5139 }
 5140 #endif
 5141 
 5142 /*
 5143    ------------------------------ mallinfo ------------------------------
 5144    Accumulate malloc statistics for arena AV into M.
 5145  */
 5146 static void
 5147 int_mallinfo (mstate av, struct mallinfo2 *m)
 5148 {
 5149   size_t i;
 5150   mbinptr b;
 5151   mchunkptr p;
 5152   INTERNAL_SIZE_T avail;
 5153   INTERNAL_SIZE_T fastavail;
 5154   int nblocks;
 5155   int nfastblocks;
 5156 
 5157   check_malloc_state (av);
 5158 
 5159   /* Account for top */
 5160   avail = chunksize (av->top);
 5161   nblocks = 1;  /* top always exists */
 5162 
 5163   /* traverse fastbins */
 5164   nfastblocks = 0;
 5165   fastavail = 0;
 5166 
 5167   for (i = 0; i < NFASTBINS; ++i)
 5168     {
 5169       for (p = fastbin (av, i);
 5170        p != 0;
 5171        p = REVEAL_PTR (p->fd))
 5172         {
 5173       if (__glibc_unlikely (misaligned_chunk (p)))
 5174         malloc_printerr ("int_mallinfo(): "
 5175                  "unaligned fastbin chunk detected");
 5176           ++nfastblocks;
 5177           fastavail += chunksize (p);
 5178         }
 5179     }
 5180 
 5181   avail += fastavail;
 5182 
 5183   /* traverse regular bins */
 5184   for (i = 1; i < NBINS; ++i)
 5185     {
 5186       b = bin_at (av, i);
 5187       for (p = last (b); p != b; p = p->bk)
 5188         {
 5189           ++nblocks;
 5190           avail += chunksize (p);
 5191         }
 5192     }
 5193 
 5194   m->smblks += nfastblocks;
 5195   m->ordblks += nblocks;
 5196   m->fordblks += avail;
 5197   m->uordblks += av->system_mem - avail;
 5198   m->arena += av->system_mem;
 5199   m->fsmblks += fastavail;
 5200   if (av == &main_arena)
 5201     {
 5202       m->hblks = mp_.n_mmaps;
 5203       m->hblkhd = mp_.mmapped_mem;
 5204       m->usmblks = 0;
 5205       m->keepcost = chunksize (av->top);
 5206     }
 5207 }
 5208 
 5209 
 5210 struct mallinfo2
 5211 __libc_mallinfo2 (void)
 5212 {
 5213   struct mallinfo2 m;
 5214   mstate ar_ptr;
 5215 
 5216   if (!__malloc_initialized)
 5217     ptmalloc_init ();
 5218 
 5219   memset (&m, 0, sizeof (m));
 5220   ar_ptr = &main_arena;
 5221   do
 5222     {
 5223       __libc_lock_lock (ar_ptr->mutex);
 5224       int_mallinfo (ar_ptr, &m);
 5225       __libc_lock_unlock (ar_ptr->mutex);
 5226 
 5227       ar_ptr = ar_ptr->next;
 5228     }
 5229   while (ar_ptr != &main_arena);
 5230 
 5231   return m;
 5232 }
 5233 libc_hidden_def (__libc_mallinfo2)
 5234 
 5235 struct mallinfo
 5236 __libc_mallinfo (void)
 5237 {
 5238   struct mallinfo m;
 5239   struct mallinfo2 m2 = __libc_mallinfo2 ();
 5240 
 5241   m.arena = m2.arena;
 5242   m.ordblks = m2.ordblks;
 5243   m.smblks = m2.smblks;
 5244   m.hblks = m2.hblks;
 5245   m.hblkhd = m2.hblkhd;
 5246   m.usmblks = m2.usmblks;
 5247   m.fsmblks = m2.fsmblks;
 5248   m.uordblks = m2.uordblks;
 5249   m.fordblks = m2.fordblks;
 5250   m.keepcost = m2.keepcost;
 5251 
 5252   return m;
 5253 }
 5254 
 5255 
 5256 /*
 5257    ------------------------------ malloc_stats ------------------------------
 5258  */
 5259 
 5260 void
 5261 __malloc_stats (void)
 5262 {
 5263   int i;
 5264   mstate ar_ptr;
 5265   unsigned int in_use_b = mp_.mmapped_mem, system_b = in_use_b;
 5266 
 5267   if (!__malloc_initialized)
 5268     ptmalloc_init ();
 5269   _IO_flockfile (stderr);
 5270   int old_flags2 = stderr->_flags2;
 5271   stderr->_flags2 |= _IO_FLAGS2_NOTCANCEL;
 5272   for (i = 0, ar_ptr = &main_arena;; i++)
 5273     {
 5274       struct mallinfo2 mi;
 5275 
 5276       memset (&mi, 0, sizeof (mi));
 5277       __libc_lock_lock (ar_ptr->mutex);
 5278       int_mallinfo (ar_ptr, &mi);
 5279       fprintf (stderr, "Arena %d:\n", i);
 5280       fprintf (stderr, "system bytes     = %10u\n", (unsigned int) mi.arena);
 5281       fprintf (stderr, "in use bytes     = %10u\n", (unsigned int) mi.uordblks);
 5282 #if MALLOC_DEBUG > 1
 5283       if (i > 0)
 5284         dump_heap (heap_for_ptr (top (ar_ptr)));
 5285 #endif
 5286       system_b += mi.arena;
 5287       in_use_b += mi.uordblks;
 5288       __libc_lock_unlock (ar_ptr->mutex);
 5289       ar_ptr = ar_ptr->next;
 5290       if (ar_ptr == &main_arena)
 5291         break;
 5292     }
 5293   fprintf (stderr, "Total (incl. mmap):\n");
 5294   fprintf (stderr, "system bytes     = %10u\n", system_b);
 5295   fprintf (stderr, "in use bytes     = %10u\n", in_use_b);
 5296   fprintf (stderr, "max mmap regions = %10u\n", (unsigned int) mp_.max_n_mmaps);
 5297   fprintf (stderr, "max mmap bytes   = %10lu\n",
 5298            (unsigned long) mp_.max_mmapped_mem);
 5299   stderr->_flags2 = old_flags2;
 5300   _IO_funlockfile (stderr);
 5301 }
 5302 
 5303 
 5304 /*
 5305    ------------------------------ mallopt ------------------------------
 5306  */
 5307 static __always_inline int
 5308 do_set_trim_threshold (size_t value)
 5309 {
 5310   LIBC_PROBE (memory_mallopt_trim_threshold, 3, value, mp_.trim_threshold,
 5311           mp_.no_dyn_threshold);
 5312   mp_.trim_threshold = value;
 5313   mp_.no_dyn_threshold = 1;
 5314   return 1;
 5315 }
 5316 
 5317 static __always_inline int
 5318 do_set_top_pad (size_t value)
 5319 {
 5320   LIBC_PROBE (memory_mallopt_top_pad, 3, value, mp_.top_pad,
 5321           mp_.no_dyn_threshold);
 5322   mp_.top_pad = value;
 5323   mp_.no_dyn_threshold = 1;
 5324   return 1;
 5325 }
 5326 
 5327 static __always_inline int
 5328 do_set_mmap_threshold (size_t value)
 5329 {
 5330   LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold,
 5331           mp_.no_dyn_threshold);
 5332   mp_.mmap_threshold = value;
 5333   mp_.no_dyn_threshold = 1;
 5334   return 1;
 5335 }
 5336 
 5337 static __always_inline int
 5338 do_set_mmaps_max (int32_t value)
 5339 {
 5340   LIBC_PROBE (memory_mallopt_mmap_max, 3, value, mp_.n_mmaps_max,
 5341           mp_.no_dyn_threshold);
 5342   mp_.n_mmaps_max = value;
 5343   mp_.no_dyn_threshold = 1;
 5344   return 1;
 5345 }
 5346 
 5347 static __always_inline int
 5348 do_set_mallopt_check (int32_t value)
 5349 {
 5350   return 1;
 5351 }
 5352 
 5353 static __always_inline int
 5354 do_set_perturb_byte (int32_t value)
 5355 {
 5356   LIBC_PROBE (memory_mallopt_perturb, 2, value, perturb_byte);
 5357   perturb_byte = value;
 5358   return 1;
 5359 }
 5360 
 5361 static __always_inline int
 5362 do_set_arena_test (size_t value)
 5363 {
 5364   LIBC_PROBE (memory_mallopt_arena_test, 2, value, mp_.arena_test);
 5365   mp_.arena_test = value;
 5366   return 1;
 5367 }
 5368 
 5369 static __always_inline int
 5370 do_set_arena_max (size_t value)
 5371 {
 5372   LIBC_PROBE (memory_mallopt_arena_max, 2, value, mp_.arena_max);
 5373   mp_.arena_max = value;
 5374   return 1;
 5375 }
 5376 
 5377 #if USE_TCACHE
 5378 static __always_inline int
 5379 do_set_tcache_max (size_t value)
 5380 {
 5381   if (value <= MAX_TCACHE_SIZE)
 5382     {
 5383       LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes);
 5384       mp_.tcache_max_bytes = value;
 5385       mp_.tcache_bins = csize2tidx (request2size(value)) + 1;
 5386       return 1;
 5387     }
 5388   return 0;
 5389 }
 5390 
 5391 static __always_inline int
 5392 do_set_tcache_count (size_t value)
 5393 {
 5394   if (value <= MAX_TCACHE_COUNT)
 5395     {
 5396       LIBC_PROBE (memory_tunable_tcache_count, 2, value, mp_.tcache_count);
 5397       mp_.tcache_count = value;
 5398       return 1;
 5399     }
 5400   return 0;
 5401 }
 5402 
 5403 static __always_inline int
 5404 do_set_tcache_unsorted_limit (size_t value)
 5405 {
 5406   LIBC_PROBE (memory_tunable_tcache_unsorted_limit, 2, value, mp_.tcache_unsorted_limit);
 5407   mp_.tcache_unsorted_limit = value;
 5408   return 1;
 5409 }
 5410 #endif
 5411 
 5412 static __always_inline int
 5413 do_set_mxfast (size_t value)
 5414 {
 5415   if (value <= MAX_FAST_SIZE)
 5416     {
 5417       LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ());
 5418       set_max_fast (value);
 5419       return 1;
 5420     }
 5421   return 0;
 5422 }
 5423 
 5424 #if HAVE_TUNABLES
 5425 static __always_inline int
 5426 do_set_hugetlb (size_t value)
 5427 {
 5428   if (value == 1)
 5429     {
 5430       enum malloc_thp_mode_t thp_mode = __malloc_thp_mode ();
 5431       /*
 5432      Only enable THP madvise usage if system does support it and
 5433      has 'madvise' mode.  Otherwise the madvise() call is wasteful.
 5434        */
 5435       if (thp_mode == malloc_thp_mode_madvise)
 5436     mp_.thp_pagesize = __malloc_default_thp_pagesize ();
 5437     }
 5438   else if (value >= 2)
 5439     __malloc_hugepage_config (value == 2 ? 0 : value, &mp_.hp_pagesize,
 5440                   &mp_.hp_flags);
 5441   return 0;
 5442 }
 5443 #endif
 5444 
 5445 int
 5446 __libc_mallopt (int param_number, int value)
 5447 {
 5448   mstate av = &main_arena;
 5449   int res = 1;
 5450 
 5451   if (!__malloc_initialized)
 5452     ptmalloc_init ();
 5453   __libc_lock_lock (av->mutex);
 5454 
 5455   LIBC_PROBE (memory_mallopt, 2, param_number, value);
 5456 
 5457   /* We must consolidate main arena before changing max_fast
 5458      (see definition of set_max_fast).  */
 5459   malloc_consolidate (av);
 5460 
 5461   /* Many of these helper functions take a size_t.  We do not worry
 5462      about overflow here, because negative int values will wrap to
 5463      very large size_t values and the helpers have sufficient range
 5464      checking for such conversions.  Many of these helpers are also
 5465      used by the tunables macros in arena.c.  */
 5466 
 5467   switch (param_number)
 5468     {
 5469     case M_MXFAST:
 5470       res = do_set_mxfast (value);
 5471       break;
 5472 
 5473     case M_TRIM_THRESHOLD:
 5474       res = do_set_trim_threshold (value);
 5475       break;
 5476 
 5477     case M_TOP_PAD:
 5478       res = do_set_top_pad (value);
 5479       break;
 5480 
 5481     case M_MMAP_THRESHOLD:
 5482       res = do_set_mmap_threshold (value);
 5483       break;
 5484 
 5485     case M_MMAP_MAX:
 5486       res = do_set_mmaps_max (value);
 5487       break;
 5488 
 5489     case M_CHECK_ACTION:
 5490       res = do_set_mallopt_check (value);
 5491       break;
 5492 
 5493     case M_PERTURB:
 5494       res = do_set_perturb_byte (value);
 5495       break;
 5496 
 5497     case M_ARENA_TEST:
 5498       if (value > 0)
 5499     res = do_set_arena_test (value);
 5500       break;
 5501 
 5502     case M_ARENA_MAX:
 5503       if (value > 0)
 5504     res = do_set_arena_max (value);
 5505       break;
 5506     }
 5507   __libc_lock_unlock (av->mutex);
 5508   return res;
 5509 }
 5510 libc_hidden_def (__libc_mallopt)
 5511 
 5512 
 5513 /*
 5514    -------------------- Alternative MORECORE functions --------------------
 5515  */
 5516 
 5517 
 5518 /*
 5519    General Requirements for MORECORE.
 5520 
 5521    The MORECORE function must have the following properties:
 5522 
 5523    If MORECORE_CONTIGUOUS is false:
 5524 
 5525  * MORECORE must allocate in multiples of pagesize. It will
 5526       only be called with arguments that are multiples of pagesize.
 5527 
 5528  * MORECORE(0) must return an address that is at least
 5529       MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
 5530 
 5531    else (i.e. If MORECORE_CONTIGUOUS is true):
 5532 
 5533  * Consecutive calls to MORECORE with positive arguments
 5534       return increasing addresses, indicating that space has been
 5535       contiguously extended.
 5536 
 5537  * MORECORE need not allocate in multiples of pagesize.
 5538       Calls to MORECORE need not have args of multiples of pagesize.
 5539 
 5540  * MORECORE need not page-align.
 5541 
 5542    In either case:
 5543 
 5544  * MORECORE may allocate more memory than requested. (Or even less,
 5545       but this will generally result in a malloc failure.)
 5546 
 5547  * MORECORE must not allocate memory when given argument zero, but
 5548       instead return one past the end address of memory from previous
 5549       nonzero call. This malloc does NOT call MORECORE(0)
 5550       until at least one call with positive arguments is made, so
 5551       the initial value returned is not important.
 5552 
 5553  * Even though consecutive calls to MORECORE need not return contiguous
 5554       addresses, it must be OK for malloc'ed chunks to span multiple
 5555       regions in those cases where they do happen to be contiguous.
 5556 
 5557  * MORECORE need not handle negative arguments -- it may instead
 5558       just return MORECORE_FAILURE when given negative arguments.
 5559       Negative arguments are always multiples of pagesize. MORECORE
 5560       must not misinterpret negative args as large positive unsigned
 5561       args. You can suppress all such calls from even occurring by defining
 5562       MORECORE_CANNOT_TRIM,
 5563 
 5564    There is some variation across systems about the type of the
 5565    argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
 5566    actually be size_t, because sbrk supports negative args, so it is
 5567    normally the signed type of the same width as size_t (sometimes
 5568    declared as "intptr_t", and sometimes "ptrdiff_t").  It doesn't much
 5569    matter though. Internally, we use "long" as arguments, which should
 5570    work across all reasonable possibilities.
 5571 
 5572    Additionally, if MORECORE ever returns failure for a positive
 5573    request, then mmap is used as a noncontiguous system allocator. This
 5574    is a useful backup strategy for systems with holes in address spaces
 5575    -- in this case sbrk cannot contiguously expand the heap, but mmap
 5576    may be able to map noncontiguous space.
 5577 
 5578    If you'd like mmap to ALWAYS be used, you can define MORECORE to be
 5579    a function that always returns MORECORE_FAILURE.
 5580 
 5581    If you are using this malloc with something other than sbrk (or its
 5582    emulation) to supply memory regions, you probably want to set
 5583    MORECORE_CONTIGUOUS as false.  As an example, here is a custom
 5584    allocator kindly contributed for pre-OSX macOS.  It uses virtually
 5585    but not necessarily physically contiguous non-paged memory (locked
 5586    in, present and won't get swapped out).  You can use it by
 5587    uncommenting this section, adding some #includes, and setting up the
 5588    appropriate defines above:
 5589 
 5590  *#define MORECORE osMoreCore
 5591  *#define MORECORE_CONTIGUOUS 0
 5592 
 5593    There is also a shutdown routine that should somehow be called for
 5594    cleanup upon program exit.
 5595 
 5596  *#define MAX_POOL_ENTRIES 100
 5597  *#define MINIMUM_MORECORE_SIZE  (64 * 1024)
 5598    static int next_os_pool;
 5599    void *our_os_pools[MAX_POOL_ENTRIES];
 5600 
 5601    void *osMoreCore(int size)
 5602    {
 5603     void *ptr = 0;
 5604     static void *sbrk_top = 0;
 5605 
 5606     if (size > 0)
 5607     {
 5608       if (size < MINIMUM_MORECORE_SIZE)
 5609          size = MINIMUM_MORECORE_SIZE;
 5610       if (CurrentExecutionLevel() == kTaskLevel)
 5611          ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
 5612       if (ptr == 0)
 5613       {
 5614         return (void *) MORECORE_FAILURE;
 5615       }
 5616       // save ptrs so they can be freed during cleanup
 5617       our_os_pools[next_os_pool] = ptr;
 5618       next_os_pool++;
 5619       ptr = (void *) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
 5620       sbrk_top = (char *) ptr + size;
 5621       return ptr;
 5622     }
 5623     else if (size < 0)
 5624     {
 5625       // we don't currently support shrink behavior
 5626       return (void *) MORECORE_FAILURE;
 5627     }
 5628     else
 5629     {
 5630       return sbrk_top;
 5631     }
 5632    }
 5633 
 5634    // cleanup any allocated memory pools
 5635    // called as last thing before shutting down driver
 5636 
 5637    void osCleanupMem(void)
 5638    {
 5639     void **ptr;
 5640 
 5641     for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
 5642       if (*ptr)
 5643       {
 5644          PoolDeallocate(*ptr);
 5645  * ptr = 0;
 5646       }
 5647    }
 5648 
 5649  */
 5650 
 5651 
 5652 /* Helper code.  */
 5653 
 5654 extern char **__libc_argv attribute_hidden;
 5655 
 5656 static void
 5657 malloc_printerr (const char *str)
 5658 {
 5659 #if IS_IN (libc)
 5660   __libc_message (do_abort, "%s\n", str);
 5661 #else
 5662   __libc_fatal (str);
 5663 #endif
 5664   __builtin_unreachable ();
 5665 }
 5666 
 5667 #if IS_IN (libc)
 5668 /* We need a wrapper function for one of the additions of POSIX.  */
 5669 int
 5670 __posix_memalign (void **memptr, size_t alignment, size_t size)
 5671 {
 5672   void *mem;
 5673 
 5674   if (!__malloc_initialized)
 5675     ptmalloc_init ();
 5676 
 5677   /* Test whether the SIZE argument is valid.  It must be a power of
 5678      two multiple of sizeof (void *).  */
 5679   if (alignment % sizeof (void *) != 0
 5680       || !powerof2 (alignment / sizeof (void *))
 5681       || alignment == 0)
 5682     return EINVAL;
 5683 
 5684 
 5685   void *address = RETURN_ADDRESS (0);
 5686   mem = _mid_memalign (alignment, size, address);
 5687 
 5688   if (mem != NULL)
 5689     {
 5690       *memptr = mem;
 5691       return 0;
 5692     }
 5693 
 5694   return ENOMEM;
 5695 }
 5696 weak_alias (__posix_memalign, posix_memalign)
 5697 #endif
 5698 
 5699 
 5700 int
 5701 __malloc_info (int options, FILE *fp)
 5702 {
 5703   /* For now, at least.  */
 5704   if (options != 0)
 5705     return EINVAL;
 5706 
 5707   int n = 0;
 5708   size_t total_nblocks = 0;
 5709   size_t total_nfastblocks = 0;
 5710   size_t total_avail = 0;
 5711   size_t total_fastavail = 0;
 5712   size_t total_system = 0;
 5713   size_t total_max_system = 0;
 5714   size_t total_aspace = 0;
 5715   size_t total_aspace_mprotect = 0;
 5716 
 5717 
 5718 
 5719   if (!__malloc_initialized)
 5720     ptmalloc_init ();
 5721 
 5722   fputs ("<malloc version=\"1\">\n", fp);
 5723 
 5724   /* Iterate over all arenas currently in use.  */
 5725   mstate ar_ptr = &main_arena;
 5726   do
 5727     {
 5728       fprintf (fp, "<heap nr=\"%d\">\n<sizes>\n", n++);
 5729 
 5730       size_t nblocks = 0;
 5731       size_t nfastblocks = 0;
 5732       size_t avail = 0;
 5733       size_t fastavail = 0;
 5734       struct
 5735       {
 5736     size_t from;
 5737     size_t to;
 5738     size_t total;
 5739     size_t count;
 5740       } sizes[NFASTBINS + NBINS - 1];
 5741 #define nsizes (sizeof (sizes) / sizeof (sizes[0]))
 5742 
 5743       __libc_lock_lock (ar_ptr->mutex);
 5744 
 5745       /* Account for top chunk.  The top-most available chunk is
 5746      treated specially and is never in any bin. See "initial_top"
 5747      comments.  */
 5748       avail = chunksize (ar_ptr->top);
 5749       nblocks = 1;  /* Top always exists.  */
 5750 
 5751       for (size_t i = 0; i < NFASTBINS; ++i)
 5752     {
 5753       mchunkptr p = fastbin (ar_ptr, i);
 5754       if (p != NULL)
 5755         {
 5756           size_t nthissize = 0;
 5757           size_t thissize = chunksize (p);
 5758 
 5759           while (p != NULL)
 5760         {
 5761           if (__glibc_unlikely (misaligned_chunk (p)))
 5762             malloc_printerr ("__malloc_info(): "
 5763                      "unaligned fastbin chunk detected");
 5764           ++nthissize;
 5765           p = REVEAL_PTR (p->fd);
 5766         }
 5767 
 5768           fastavail += nthissize * thissize;
 5769           nfastblocks += nthissize;
 5770           sizes[i].from = thissize - (MALLOC_ALIGNMENT - 1);
 5771           sizes[i].to = thissize;
 5772           sizes[i].count = nthissize;
 5773         }
 5774       else
 5775         sizes[i].from = sizes[i].to = sizes[i].count = 0;
 5776 
 5777       sizes[i].total = sizes[i].count * sizes[i].to;
 5778     }
 5779 
 5780 
 5781       mbinptr bin;
 5782       struct malloc_chunk *r;
 5783 
 5784       for (size_t i = 1; i < NBINS; ++i)
 5785     {
 5786       bin = bin_at (ar_ptr, i);
 5787       r = bin->fd;
 5788       sizes[NFASTBINS - 1 + i].from = ~((size_t) 0);
 5789       sizes[NFASTBINS - 1 + i].to = sizes[NFASTBINS - 1 + i].total
 5790                       = sizes[NFASTBINS - 1 + i].count = 0;
 5791 
 5792       if (r != NULL)
 5793         while (r != bin)
 5794           {
 5795         size_t r_size = chunksize_nomask (r);
 5796         ++sizes[NFASTBINS - 1 + i].count;
 5797         sizes[NFASTBINS - 1 + i].total += r_size;
 5798         sizes[NFASTBINS - 1 + i].from
 5799           = MIN (sizes[NFASTBINS - 1 + i].from, r_size);
 5800         sizes[NFASTBINS - 1 + i].to = MAX (sizes[NFASTBINS - 1 + i].to,
 5801                            r_size);
 5802 
 5803         r = r->fd;
 5804           }
 5805 
 5806       if (sizes[NFASTBINS - 1 + i].count == 0)
 5807         sizes[NFASTBINS - 1 + i].from = 0;
 5808       nblocks += sizes[NFASTBINS - 1 + i].count;
 5809       avail += sizes[NFASTBINS - 1 + i].total;
 5810     }
 5811 
 5812       size_t heap_size = 0;
 5813       size_t heap_mprotect_size = 0;
 5814       size_t heap_count = 0;
 5815       if (ar_ptr != &main_arena)
 5816     {
 5817       /* Iterate over the arena heaps from back to front.  */
 5818       heap_info *heap = heap_for_ptr (top (ar_ptr));
 5819       do
 5820         {
 5821           heap_size += heap->size;
 5822           heap_mprotect_size += heap->mprotect_size;
 5823           heap = heap->prev;
 5824           ++heap_count;
 5825         }
 5826       while (heap != NULL);
 5827     }
 5828 
 5829       __libc_lock_unlock (ar_ptr->mutex);
 5830 
 5831       total_nfastblocks += nfastblocks;
 5832       total_fastavail += fastavail;
 5833 
 5834       total_nblocks += nblocks;
 5835       total_avail += avail;
 5836 
 5837       for (size_t i = 0; i < nsizes; ++i)
 5838     if (sizes[i].count != 0 && i != NFASTBINS)
 5839       fprintf (fp, "\
 5840   <size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
 5841            sizes[i].from, sizes[i].to, sizes[i].total, sizes[i].count);
 5842 
 5843       if (sizes[NFASTBINS].count != 0)
 5844     fprintf (fp, "\
 5845   <unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
 5846          sizes[NFASTBINS].from, sizes[NFASTBINS].to,
 5847          sizes[NFASTBINS].total, sizes[NFASTBINS].count);
 5848 
 5849       total_system += ar_ptr->system_mem;
 5850       total_max_system += ar_ptr->max_system_mem;
 5851 
 5852       fprintf (fp,
 5853            "</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
 5854            "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
 5855            "<system type=\"current\" size=\"%zu\"/>\n"
 5856            "<system type=\"max\" size=\"%zu\"/>\n",
 5857            nfastblocks, fastavail, nblocks, avail,
 5858            ar_ptr->system_mem, ar_ptr->max_system_mem);
 5859 
 5860       if (ar_ptr != &main_arena)
 5861     {
 5862       fprintf (fp,
 5863            "<aspace type=\"total\" size=\"%zu\"/>\n"
 5864            "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
 5865            "<aspace type=\"subheaps\" size=\"%zu\"/>\n",
 5866            heap_size, heap_mprotect_size, heap_count);
 5867       total_aspace += heap_size;
 5868       total_aspace_mprotect += heap_mprotect_size;
 5869     }
 5870       else
 5871     {
 5872       fprintf (fp,
 5873            "<aspace type=\"total\" size=\"%zu\"/>\n"
 5874            "<aspace type=\"mprotect\" size=\"%zu\"/>\n",
 5875            ar_ptr->system_mem, ar_ptr->system_mem);
 5876       total_aspace += ar_ptr->system_mem;
 5877       total_aspace_mprotect += ar_ptr->system_mem;
 5878     }
 5879 
 5880       fputs ("</heap>\n", fp);
 5881       ar_ptr = ar_ptr->next;
 5882     }
 5883   while (ar_ptr != &main_arena);
 5884 
 5885   fprintf (fp,
 5886        "<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
 5887        "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
 5888        "<total type=\"mmap\" count=\"%d\" size=\"%zu\"/>\n"
 5889        "<system type=\"current\" size=\"%zu\"/>\n"
 5890        "<system type=\"max\" size=\"%zu\"/>\n"
 5891        "<aspace type=\"total\" size=\"%zu\"/>\n"
 5892        "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
 5893        "</malloc>\n",
 5894        total_nfastblocks, total_fastavail, total_nblocks, total_avail,
 5895        mp_.n_mmaps, mp_.mmapped_mem,
 5896        total_system, total_max_system,
 5897        total_aspace, total_aspace_mprotect);
 5898 
 5899   return 0;
 5900 }
 5901 #if IS_IN (libc)
 5902 weak_alias (__malloc_info, malloc_info)
 5903 
 5904 strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
 5905 strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
 5906 strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
 5907 strong_alias (__libc_memalign, __memalign)
 5908 weak_alias (__libc_memalign, memalign)
 5909 strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
 5910 strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
 5911 strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
 5912 strong_alias (__libc_mallinfo, __mallinfo)
 5913 weak_alias (__libc_mallinfo, mallinfo)
 5914 strong_alias (__libc_mallinfo2, __mallinfo2)
 5915 weak_alias (__libc_mallinfo2, mallinfo2)
 5916 strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
 5917 
 5918 weak_alias (__malloc_stats, malloc_stats)
 5919 weak_alias (__malloc_usable_size, malloc_usable_size)
 5920 weak_alias (__malloc_trim, malloc_trim)
 5921 #endif
 5922 
 5923 #if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_26)
 5924 compat_symbol (libc, __libc_free, cfree, GLIBC_2_0);
 5925 #endif
 5926 
 5927 /* ------------------------------------------------------------
 5928    History:
 5929 
 5930    [see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]
 5931 
 5932  */
 5933 /*
 5934  * Local variables:
 5935  * c-basic-offset: 2
 5936  * End:
 5937  */