"Fossies" - the Fresh Open Source Software Archive

Member "glibc-2.34/malloc/malloc.c" (2 Aug 2021, 190637 Bytes) of package /linux/misc/glibc-2.34.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "malloc.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.33_vs_2.34.

    1 /* Malloc implementation for multiple threads without lock contention.
    2    Copyright (C) 1996-2021 Free Software Foundation, Inc.
    3    This file is part of the GNU C Library.
    4    Contributed by Wolfram Gloger <wg@malloc.de>
    5    and Doug Lea <dl@cs.oswego.edu>, 2001.
    6 
    7    The GNU C Library is free software; you can redistribute it and/or
    8    modify it under the terms of the GNU Lesser General Public License as
    9    published by the Free Software Foundation; either version 2.1 of the
   10    License, or (at your option) any later version.
   11 
   12    The GNU C Library is distributed in the hope that it will be useful,
   13    but WITHOUT ANY WARRANTY; without even the implied warranty of
   14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   15    Lesser General Public License for more details.
   16 
   17    You should have received a copy of the GNU Lesser General Public
   18    License along with the GNU C Library; see the file COPYING.LIB.  If
   19    not, see <https://www.gnu.org/licenses/>.  */
   20 
   21 /*
   22   This is a version (aka ptmalloc2) of malloc/free/realloc written by
   23   Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
   24 
   25   There have been substantial changes made after the integration into
   26   glibc in all parts of the code.  Do not look for much commonality
   27   with the ptmalloc2 version.
   28 
   29 * Version ptmalloc2-20011215
   30   based on:
   31   VERSION 2.7.0 Sun Mar 11 14:14:06 2001  Doug Lea  (dl at gee)
   32 
   33 * Quickstart
   34 
   35   In order to compile this implementation, a Makefile is provided with
   36   the ptmalloc2 distribution, which has pre-defined targets for some
   37   popular systems (e.g. "make posix" for Posix threads).  All that is
   38   typically required with regard to compiler flags is the selection of
   39   the thread package via defining one out of USE_PTHREADS, USE_THR or
   40   USE_SPROC.  Check the thread-m.h file for what effects this has.
   41   Many/most systems will additionally require USE_TSD_DATA_HACK to be
   42   defined, so this is the default for "make posix".
   43 
   44 * Why use this malloc?
   45 
   46   This is not the fastest, most space-conserving, most portable, or
   47   most tunable malloc ever written. However it is among the fastest
   48   while also being among the most space-conserving, portable and tunable.
   49   Consistent balance across these factors results in a good general-purpose
   50   allocator for malloc-intensive programs.
   51 
   52   The main properties of the algorithms are:
   53   * For large (>= 512 bytes) requests, it is a pure best-fit allocator,
   54     with ties normally decided via FIFO (i.e. least recently used).
   55   * For small (<= 64 bytes by default) requests, it is a caching
   56     allocator, that maintains pools of quickly recycled chunks.
   57   * In between, and for combinations of large and small requests, it does
   58     the best it can trying to meet both goals at once.
   59   * For very large requests (>= 128KB by default), it relies on system
   60     memory mapping facilities, if supported.
   61 
   62   For a longer but slightly out of date high-level description, see
   63      http://gee.cs.oswego.edu/dl/html/malloc.html
   64 
   65   You may already by default be using a C library containing a malloc
   66   that is  based on some version of this malloc (for example in
   67   linux). You might still want to use the one in this file in order to
   68   customize settings or to avoid overheads associated with library
   69   versions.
   70 
   71 * Contents, described in more detail in "description of public routines" below.
   72 
   73   Standard (ANSI/SVID/...)  functions:
   74     malloc(size_t n);
   75     calloc(size_t n_elements, size_t element_size);
   76     free(void* p);
   77     realloc(void* p, size_t n);
   78     memalign(size_t alignment, size_t n);
   79     valloc(size_t n);
   80     mallinfo()
   81     mallopt(int parameter_number, int parameter_value)
   82 
   83   Additional functions:
   84     independent_calloc(size_t n_elements, size_t size, void* chunks[]);
   85     independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
   86     pvalloc(size_t n);
   87     malloc_trim(size_t pad);
   88     malloc_usable_size(void* p);
   89     malloc_stats();
   90 
   91 * Vital statistics:
   92 
   93   Supported pointer representation:       4 or 8 bytes
   94   Supported size_t  representation:       4 or 8 bytes
   95        Note that size_t is allowed to be 4 bytes even if pointers are 8.
   96        You can adjust this by defining INTERNAL_SIZE_T
   97 
   98   Alignment:                              2 * sizeof(size_t) (default)
   99        (i.e., 8 byte alignment with 4byte size_t). This suffices for
  100        nearly all current machines and C compilers. However, you can
  101        define MALLOC_ALIGNMENT to be wider than this if necessary.
  102 
  103   Minimum overhead per allocated chunk:   4 or 8 bytes
  104        Each malloced chunk has a hidden word of overhead holding size
  105        and status information.
  106 
  107   Minimum allocated size: 4-byte ptrs:  16 bytes    (including 4 overhead)
  108               8-byte ptrs:  24/32 bytes (including, 4/8 overhead)
  109 
  110        When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
  111        ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
  112        needed; 4 (8) for a trailing size field and 8 (16) bytes for
  113        free list pointers. Thus, the minimum allocatable size is
  114        16/24/32 bytes.
  115 
  116        Even a request for zero bytes (i.e., malloc(0)) returns a
  117        pointer to something of the minimum allocatable size.
  118 
  119        The maximum overhead wastage (i.e., number of extra bytes
  120        allocated than were requested in malloc) is less than or equal
  121        to the minimum size, except for requests >= mmap_threshold that
  122        are serviced via mmap(), where the worst case wastage is 2 *
  123        sizeof(size_t) bytes plus the remainder from a system page (the
  124        minimal mmap unit); typically 4096 or 8192 bytes.
  125 
  126   Maximum allocated size:  4-byte size_t: 2^32 minus about two pages
  127                8-byte size_t: 2^64 minus about two pages
  128 
  129        It is assumed that (possibly signed) size_t values suffice to
  130        represent chunk sizes. `Possibly signed' is due to the fact
  131        that `size_t' may be defined on a system as either a signed or
  132        an unsigned type. The ISO C standard says that it must be
  133        unsigned, but a few systems are known not to adhere to this.
  134        Additionally, even when size_t is unsigned, sbrk (which is by
  135        default used to obtain memory from system) accepts signed
  136        arguments, and may not be able to handle size_t-wide arguments
  137        with negative sign bit.  Generally, values that would
  138        appear as negative after accounting for overhead and alignment
  139        are supported only via mmap(), which does not have this
  140        limitation.
  141 
  142        Requests for sizes outside the allowed range will perform an optional
  143        failure action and then return null. (Requests may also
  144        also fail because a system is out of memory.)
  145 
  146   Thread-safety: thread-safe
  147 
  148   Compliance: I believe it is compliant with the 1997 Single Unix Specification
  149        Also SVID/XPG, ANSI C, and probably others as well.
  150 
  151 * Synopsis of compile-time options:
  152 
  153     People have reported using previous versions of this malloc on all
  154     versions of Unix, sometimes by tweaking some of the defines
  155     below. It has been tested most extensively on Solaris and Linux.
  156     People also report using it in stand-alone embedded systems.
  157 
  158     The implementation is in straight, hand-tuned ANSI C.  It is not
  159     at all modular. (Sorry!)  It uses a lot of macros.  To be at all
  160     usable, this code should be compiled using an optimizing compiler
  161     (for example gcc -O3) that can simplify expressions and control
  162     paths. (FAQ: some macros import variables as arguments rather than
  163     declare locals because people reported that some debuggers
  164     otherwise get confused.)
  165 
  166     OPTION                     DEFAULT VALUE
  167 
  168     Compilation Environment options:
  169 
  170     HAVE_MREMAP                0
  171 
  172     Changing default word sizes:
  173 
  174     INTERNAL_SIZE_T            size_t
  175 
  176     Configuration and functionality options:
  177 
  178     USE_PUBLIC_MALLOC_WRAPPERS NOT defined
  179     USE_MALLOC_LOCK            NOT defined
  180     MALLOC_DEBUG               NOT defined
  181     REALLOC_ZERO_BYTES_FREES   1
  182     TRIM_FASTBINS              0
  183 
  184     Options for customizing MORECORE:
  185 
  186     MORECORE                   sbrk
  187     MORECORE_FAILURE           -1
  188     MORECORE_CONTIGUOUS        1
  189     MORECORE_CANNOT_TRIM       NOT defined
  190     MORECORE_CLEARS            1
  191     MMAP_AS_MORECORE_SIZE      (1024 * 1024)
  192 
  193     Tuning options that are also dynamically changeable via mallopt:
  194 
  195     DEFAULT_MXFAST             64 (for 32bit), 128 (for 64bit)
  196     DEFAULT_TRIM_THRESHOLD     128 * 1024
  197     DEFAULT_TOP_PAD            0
  198     DEFAULT_MMAP_THRESHOLD     128 * 1024
  199     DEFAULT_MMAP_MAX           65536
  200 
  201     There are several other #defined constants and macros that you
  202     probably don't want to touch unless you are extending or adapting malloc.  */
  203 
  204 /*
  205   void* is the pointer type that malloc should say it returns
  206 */
  207 
  208 #ifndef void
  209 #define void      void
  210 #endif /*void*/
  211 
  212 #include <stddef.h>   /* for size_t */
  213 #include <stdlib.h>   /* for getenv(), abort() */
  214 #include <unistd.h>   /* for __libc_enable_secure */
  215 
  216 #include <atomic.h>
  217 #include <_itoa.h>
  218 #include <bits/wordsize.h>
  219 #include <sys/sysinfo.h>
  220 
  221 #include <ldsodefs.h>
  222 
  223 #include <unistd.h>
  224 #include <stdio.h>    /* needed for malloc_stats */
  225 #include <errno.h>
  226 #include <assert.h>
  227 
  228 #include <shlib-compat.h>
  229 
  230 /* For uintptr_t.  */
  231 #include <stdint.h>
  232 
  233 /* For va_arg, va_start, va_end.  */
  234 #include <stdarg.h>
  235 
  236 /* For MIN, MAX, powerof2.  */
  237 #include <sys/param.h>
  238 
  239 /* For ALIGN_UP et. al.  */
  240 #include <libc-pointer-arith.h>
  241 
  242 /* For DIAG_PUSH/POP_NEEDS_COMMENT et al.  */
  243 #include <libc-diag.h>
  244 
  245 /* For memory tagging.  */
  246 #include <libc-mtag.h>
  247 
  248 #include <malloc/malloc-internal.h>
  249 
  250 /* For SINGLE_THREAD_P.  */
  251 #include <sysdep-cancel.h>
  252 
  253 #include <libc-internal.h>
  254 
  255 /* For tcache double-free check.  */
  256 #include <random-bits.h>
  257 #include <sys/random.h>
  258 
  259 /*
  260   Debugging:
  261 
  262   Because freed chunks may be overwritten with bookkeeping fields, this
  263   malloc will often die when freed memory is overwritten by user
  264   programs.  This can be very effective (albeit in an annoying way)
  265   in helping track down dangling pointers.
  266 
  267   If you compile with -DMALLOC_DEBUG, a number of assertion checks are
  268   enabled that will catch more memory errors. You probably won't be
  269   able to make much sense of the actual assertion errors, but they
  270   should help you locate incorrectly overwritten memory.  The checking
  271   is fairly extensive, and will slow down execution
  272   noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
  273   will attempt to check every non-mmapped allocated and free chunk in
  274   the course of computing the summmaries. (By nature, mmapped regions
  275   cannot be checked very much automatically.)
  276 
  277   Setting MALLOC_DEBUG may also be helpful if you are trying to modify
  278   this code. The assertions in the check routines spell out in more
  279   detail the assumptions and invariants underlying the algorithms.
  280 
  281   Setting MALLOC_DEBUG does NOT provide an automated mechanism for
  282   checking that all accesses to malloced memory stay within their
  283   bounds. However, there are several add-ons and adaptations of this
  284   or other mallocs available that do this.
  285 */
  286 
  287 #ifndef MALLOC_DEBUG
  288 #define MALLOC_DEBUG 0
  289 #endif
  290 
  291 #if IS_IN (libc)
  292 #ifndef NDEBUG
  293 # define __assert_fail(assertion, file, line, function)         \
  294      __malloc_assert(assertion, file, line, function)
  295 
  296 extern const char *__progname;
  297 
  298 static void
  299 __malloc_assert (const char *assertion, const char *file, unsigned int line,
  300          const char *function)
  301 {
  302   (void) __fxprintf (NULL, "%s%s%s:%u: %s%sAssertion `%s' failed.\n",
  303              __progname, __progname[0] ? ": " : "",
  304              file, line,
  305              function ? function : "", function ? ": " : "",
  306              assertion);
  307   fflush (stderr);
  308   abort ();
  309 }
  310 #endif
  311 #endif
  312 
  313 #if USE_TCACHE
  314 /* We want 64 entries.  This is an arbitrary limit, which tunables can reduce.  */
  315 # define TCACHE_MAX_BINS        64
  316 # define MAX_TCACHE_SIZE    tidx2usize (TCACHE_MAX_BINS-1)
  317 
  318 /* Only used to pre-fill the tunables.  */
  319 # define tidx2usize(idx)    (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
  320 
  321 /* When "x" is from chunksize().  */
  322 # define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT)
  323 /* When "x" is a user-provided size.  */
  324 # define usize2tidx(x) csize2tidx (request2size (x))
  325 
  326 /* With rounding and alignment, the bins are...
  327    idx 0   bytes 0..24 (64-bit) or 0..12 (32-bit)
  328    idx 1   bytes 25..40 or 13..20
  329    idx 2   bytes 41..56 or 21..28
  330    etc.  */
  331 
  332 /* This is another arbitrary limit, which tunables can change.  Each
  333    tcache bin will hold at most this number of chunks.  */
  334 # define TCACHE_FILL_COUNT 7
  335 
  336 /* Maximum chunks in tcache bins for tunables.  This value must fit the range
  337    of tcache->counts[] entries, else they may overflow.  */
  338 # define MAX_TCACHE_COUNT UINT16_MAX
  339 #endif
  340 
  341 /* Safe-Linking:
  342    Use randomness from ASLR (mmap_base) to protect single-linked lists
  343    of Fast-Bins and TCache.  That is, mask the "next" pointers of the
  344    lists' chunks, and also perform allocation alignment checks on them.
  345    This mechanism reduces the risk of pointer hijacking, as was done with
  346    Safe-Unlinking in the double-linked lists of Small-Bins.
  347    It assumes a minimum page size of 4096 bytes (12 bits).  Systems with
  348    larger pages provide less entropy, although the pointer mangling
  349    still works.  */
  350 #define PROTECT_PTR(pos, ptr) \
  351   ((__typeof (ptr)) ((((size_t) pos) >> 12) ^ ((size_t) ptr)))
  352 #define REVEAL_PTR(ptr)  PROTECT_PTR (&ptr, ptr)
  353 
  354 /*
  355   The REALLOC_ZERO_BYTES_FREES macro controls the behavior of realloc (p, 0)
  356   when p is nonnull.  If the macro is nonzero, the realloc call returns NULL;
  357   otherwise, the call returns what malloc (0) would.  In either case,
  358   p is freed.  Glibc uses a nonzero REALLOC_ZERO_BYTES_FREES, which
  359   implements common historical practice.
  360 
  361   ISO C17 says the realloc call has implementation-defined behavior,
  362   and it might not even free p.
  363 */
  364 
  365 #ifndef REALLOC_ZERO_BYTES_FREES
  366 #define REALLOC_ZERO_BYTES_FREES 1
  367 #endif
  368 
  369 /*
  370   TRIM_FASTBINS controls whether free() of a very small chunk can
  371   immediately lead to trimming. Setting to true (1) can reduce memory
  372   footprint, but will almost always slow down programs that use a lot
  373   of small chunks.
  374 
  375   Define this only if you are willing to give up some speed to more
  376   aggressively reduce system-level memory footprint when releasing
  377   memory in programs that use many small chunks.  You can get
  378   essentially the same effect by setting MXFAST to 0, but this can
  379   lead to even greater slowdowns in programs using many small chunks.
  380   TRIM_FASTBINS is an in-between compile-time option, that disables
  381   only those chunks bordering topmost memory from being placed in
  382   fastbins.
  383 */
  384 
  385 #ifndef TRIM_FASTBINS
  386 #define TRIM_FASTBINS  0
  387 #endif
  388 
  389 /* Definition for getting more memory from the OS.  */
  390 #include "morecore.c"
  391 
  392 #define MORECORE         (*__glibc_morecore)
  393 #define MORECORE_FAILURE 0
  394 
  395 /* Memory tagging.  */
  396 
  397 /* Some systems support the concept of tagging (sometimes known as
  398    coloring) memory locations on a fine grained basis.  Each memory
  399    location is given a color (normally allocated randomly) and
  400    pointers are also colored.  When the pointer is dereferenced, the
  401    pointer's color is checked against the memory's color and if they
  402    differ the access is faulted (sometimes lazily).
  403 
  404    We use this in glibc by maintaining a single color for the malloc
  405    data structures that are interleaved with the user data and then
  406    assigning separate colors for each block allocation handed out.  In
  407    this way simple buffer overruns will be rapidly detected.  When
  408    memory is freed, the memory is recolored back to the glibc default
  409    so that simple use-after-free errors can also be detected.
  410 
  411    If memory is reallocated the buffer is recolored even if the
  412    address remains the same.  This has a performance impact, but
  413    guarantees that the old pointer cannot mistakenly be reused (code
  414    that compares old against new will see a mismatch and will then
  415    need to behave as though realloc moved the data to a new location).
  416 
  417    Internal API for memory tagging support.
  418 
  419    The aim is to keep the code for memory tagging support as close to
  420    the normal APIs in glibc as possible, so that if tagging is not
  421    enabled in the library, or is disabled at runtime then standard
  422    operations can continue to be used.  Support macros are used to do
  423    this:
  424 
  425    void *tag_new_zero_region (void *ptr, size_t size)
  426 
  427    Allocates a new tag, colors the memory with that tag, zeros the
  428    memory and returns a pointer that is correctly colored for that
  429    location.  The non-tagging version will simply call memset with 0.
  430 
  431    void *tag_region (void *ptr, size_t size)
  432 
  433    Color the region of memory pointed to by PTR and size SIZE with
  434    the color of PTR.  Returns the original pointer.
  435 
  436    void *tag_new_usable (void *ptr)
  437 
  438    Allocate a new random color and use it to color the user region of
  439    a chunk; this may include data from the subsequent chunk's header
  440    if tagging is sufficiently fine grained.  Returns PTR suitably
  441    recolored for accessing the memory there.
  442 
  443    void *tag_at (void *ptr)
  444 
  445    Read the current color of the memory at the address pointed to by
  446    PTR (ignoring it's current color) and return PTR recolored to that
  447    color.  PTR must be valid address in all other respects.  When
  448    tagging is not enabled, it simply returns the original pointer.
  449 */
  450 
  451 #ifdef USE_MTAG
  452 static bool mtag_enabled = false;
  453 static int mtag_mmap_flags = 0;
  454 #else
  455 # define mtag_enabled false
  456 # define mtag_mmap_flags 0
  457 #endif
  458 
  459 static __always_inline void *
  460 tag_region (void *ptr, size_t size)
  461 {
  462   if (__glibc_unlikely (mtag_enabled))
  463     return __libc_mtag_tag_region (ptr, size);
  464   return ptr;
  465 }
  466 
  467 static __always_inline void *
  468 tag_new_zero_region (void *ptr, size_t size)
  469 {
  470   if (__glibc_unlikely (mtag_enabled))
  471     return __libc_mtag_tag_zero_region (__libc_mtag_new_tag (ptr), size);
  472   return memset (ptr, 0, size);
  473 }
  474 
  475 /* Defined later.  */
  476 static void *
  477 tag_new_usable (void *ptr);
  478 
  479 static __always_inline void *
  480 tag_at (void *ptr)
  481 {
  482   if (__glibc_unlikely (mtag_enabled))
  483     return __libc_mtag_address_get_tag (ptr);
  484   return ptr;
  485 }
  486 
  487 #include <string.h>
  488 
  489 /*
  490   MORECORE-related declarations. By default, rely on sbrk
  491 */
  492 
  493 
  494 /*
  495   MORECORE is the name of the routine to call to obtain more memory
  496   from the system.  See below for general guidance on writing
  497   alternative MORECORE functions, as well as a version for WIN32 and a
  498   sample version for pre-OSX macos.
  499 */
  500 
  501 #ifndef MORECORE
  502 #define MORECORE sbrk
  503 #endif
  504 
  505 /*
  506   MORECORE_FAILURE is the value returned upon failure of MORECORE
  507   as well as mmap. Since it cannot be an otherwise valid memory address,
  508   and must reflect values of standard sys calls, you probably ought not
  509   try to redefine it.
  510 */
  511 
  512 #ifndef MORECORE_FAILURE
  513 #define MORECORE_FAILURE (-1)
  514 #endif
  515 
  516 /*
  517   If MORECORE_CONTIGUOUS is true, take advantage of fact that
  518   consecutive calls to MORECORE with positive arguments always return
  519   contiguous increasing addresses.  This is true of unix sbrk.  Even
  520   if not defined, when regions happen to be contiguous, malloc will
  521   permit allocations spanning regions obtained from different
  522   calls. But defining this when applicable enables some stronger
  523   consistency checks and space efficiencies.
  524 */
  525 
  526 #ifndef MORECORE_CONTIGUOUS
  527 #define MORECORE_CONTIGUOUS 1
  528 #endif
  529 
  530 /*
  531   Define MORECORE_CANNOT_TRIM if your version of MORECORE
  532   cannot release space back to the system when given negative
  533   arguments. This is generally necessary only if you are using
  534   a hand-crafted MORECORE function that cannot handle negative arguments.
  535 */
  536 
  537 /* #define MORECORE_CANNOT_TRIM */
  538 
  539 /*  MORECORE_CLEARS           (default 1)
  540      The degree to which the routine mapped to MORECORE zeroes out
  541      memory: never (0), only for newly allocated space (1) or always
  542      (2).  The distinction between (1) and (2) is necessary because on
  543      some systems, if the application first decrements and then
  544      increments the break value, the contents of the reallocated space
  545      are unspecified.
  546  */
  547 
  548 #ifndef MORECORE_CLEARS
  549 # define MORECORE_CLEARS 1
  550 #endif
  551 
  552 
  553 /*
  554    MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
  555    sbrk fails, and mmap is used as a backup.  The value must be a
  556    multiple of page size.  This backup strategy generally applies only
  557    when systems have "holes" in address space, so sbrk cannot perform
  558    contiguous expansion, but there is still space available on system.
  559    On systems for which this is known to be useful (i.e. most linux
  560    kernels), this occurs only when programs allocate huge amounts of
  561    memory.  Between this, and the fact that mmap regions tend to be
  562    limited, the size should be large, to avoid too many mmap calls and
  563    thus avoid running out of kernel resources.  */
  564 
  565 #ifndef MMAP_AS_MORECORE_SIZE
  566 #define MMAP_AS_MORECORE_SIZE (1024 * 1024)
  567 #endif
  568 
  569 /*
  570   Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
  571   large blocks.
  572 */
  573 
  574 #ifndef HAVE_MREMAP
  575 #define HAVE_MREMAP 0
  576 #endif
  577 
  578 /*
  579   This version of malloc supports the standard SVID/XPG mallinfo
  580   routine that returns a struct containing usage properties and
  581   statistics. It should work on any SVID/XPG compliant system that has
  582   a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
  583   install such a thing yourself, cut out the preliminary declarations
  584   as described above and below and save them in a malloc.h file. But
  585   there's no compelling reason to bother to do this.)
  586 
  587   The main declaration needed is the mallinfo struct that is returned
  588   (by-copy) by mallinfo().  The SVID/XPG malloinfo struct contains a
  589   bunch of fields that are not even meaningful in this version of
  590   malloc.  These fields are are instead filled by mallinfo() with
  591   other numbers that might be of interest.
  592 */
  593 
  594 
  595 /* ---------- description of public routines ------------ */
  596 
  597 #if IS_IN (libc)
  598 /*
  599   malloc(size_t n)
  600   Returns a pointer to a newly allocated chunk of at least n bytes, or null
  601   if no space is available. Additionally, on failure, errno is
  602   set to ENOMEM on ANSI C systems.
  603 
  604   If n is zero, malloc returns a minimum-sized chunk. (The minimum
  605   size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
  606   systems.)  On most systems, size_t is an unsigned type, so calls
  607   with negative arguments are interpreted as requests for huge amounts
  608   of space, which will often fail. The maximum supported value of n
  609   differs across systems, but is in all cases less than the maximum
  610   representable value of a size_t.
  611 */
  612 void*  __libc_malloc(size_t);
  613 libc_hidden_proto (__libc_malloc)
  614 
  615 /*
  616   free(void* p)
  617   Releases the chunk of memory pointed to by p, that had been previously
  618   allocated using malloc or a related routine such as realloc.
  619   It has no effect if p is null. It can have arbitrary (i.e., bad!)
  620   effects if p has already been freed.
  621 
  622   Unless disabled (using mallopt), freeing very large spaces will
  623   when possible, automatically trigger operations that give
  624   back unused memory to the system, thus reducing program footprint.
  625 */
  626 void     __libc_free(void*);
  627 libc_hidden_proto (__libc_free)
  628 
  629 /*
  630   calloc(size_t n_elements, size_t element_size);
  631   Returns a pointer to n_elements * element_size bytes, with all locations
  632   set to zero.
  633 */
  634 void*  __libc_calloc(size_t, size_t);
  635 
  636 /*
  637   realloc(void* p, size_t n)
  638   Returns a pointer to a chunk of size n that contains the same data
  639   as does chunk p up to the minimum of (n, p's size) bytes, or null
  640   if no space is available.
  641 
  642   The returned pointer may or may not be the same as p. The algorithm
  643   prefers extending p when possible, otherwise it employs the
  644   equivalent of a malloc-copy-free sequence.
  645 
  646   If p is null, realloc is equivalent to malloc.
  647 
  648   If space is not available, realloc returns null, errno is set (if on
  649   ANSI) and p is NOT freed.
  650 
  651   if n is for fewer bytes than already held by p, the newly unused
  652   space is lopped off and freed if possible.  Unless the #define
  653   REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
  654   zero (re)allocates a minimum-sized chunk.
  655 
  656   Large chunks that were internally obtained via mmap will always be
  657   grown using malloc-copy-free sequences unless the system supports
  658   MREMAP (currently only linux).
  659 
  660   The old unix realloc convention of allowing the last-free'd chunk
  661   to be used as an argument to realloc is not supported.
  662 */
  663 void*  __libc_realloc(void*, size_t);
  664 libc_hidden_proto (__libc_realloc)
  665 
  666 /*
  667   memalign(size_t alignment, size_t n);
  668   Returns a pointer to a newly allocated chunk of n bytes, aligned
  669   in accord with the alignment argument.
  670 
  671   The alignment argument should be a power of two. If the argument is
  672   not a power of two, the nearest greater power is used.
  673   8-byte alignment is guaranteed by normal malloc calls, so don't
  674   bother calling memalign with an argument of 8 or less.
  675 
  676   Overreliance on memalign is a sure way to fragment space.
  677 */
  678 void*  __libc_memalign(size_t, size_t);
  679 libc_hidden_proto (__libc_memalign)
  680 
  681 /*
  682   valloc(size_t n);
  683   Equivalent to memalign(pagesize, n), where pagesize is the page
  684   size of the system. If the pagesize is unknown, 4096 is used.
  685 */
  686 void*  __libc_valloc(size_t);
  687 
  688 
  689 
  690 /*
  691   mallinfo()
  692   Returns (by copy) a struct containing various summary statistics:
  693 
  694   arena:     current total non-mmapped bytes allocated from system
  695   ordblks:   the number of free chunks
  696   smblks:    the number of fastbin blocks (i.e., small chunks that
  697            have been freed but not use resused or consolidated)
  698   hblks:     current number of mmapped regions
  699   hblkhd:    total bytes held in mmapped regions
  700   usmblks:   always 0
  701   fsmblks:   total bytes held in fastbin blocks
  702   uordblks:  current total allocated space (normal or mmapped)
  703   fordblks:  total free space
  704   keepcost:  the maximum number of bytes that could ideally be released
  705            back to system via malloc_trim. ("ideally" means that
  706            it ignores page restrictions etc.)
  707 
  708   Because these fields are ints, but internal bookkeeping may
  709   be kept as longs, the reported values may wrap around zero and
  710   thus be inaccurate.
  711 */
  712 struct mallinfo2 __libc_mallinfo2(void);
  713 libc_hidden_proto (__libc_mallinfo2)
  714 
  715 struct mallinfo __libc_mallinfo(void);
  716 
  717 
  718 /*
  719   pvalloc(size_t n);
  720   Equivalent to valloc(minimum-page-that-holds(n)), that is,
  721   round up n to nearest pagesize.
  722  */
  723 void*  __libc_pvalloc(size_t);
  724 
  725 /*
  726   malloc_trim(size_t pad);
  727 
  728   If possible, gives memory back to the system (via negative
  729   arguments to sbrk) if there is unused memory at the `high' end of
  730   the malloc pool. You can call this after freeing large blocks of
  731   memory to potentially reduce the system-level memory requirements
  732   of a program. However, it cannot guarantee to reduce memory. Under
  733   some allocation patterns, some large free blocks of memory will be
  734   locked between two used chunks, so they cannot be given back to
  735   the system.
  736 
  737   The `pad' argument to malloc_trim represents the amount of free
  738   trailing space to leave untrimmed. If this argument is zero,
  739   only the minimum amount of memory to maintain internal data
  740   structures will be left (one page or less). Non-zero arguments
  741   can be supplied to maintain enough trailing space to service
  742   future expected allocations without having to re-obtain memory
  743   from the system.
  744 
  745   Malloc_trim returns 1 if it actually released any memory, else 0.
  746   On systems that do not support "negative sbrks", it will always
  747   return 0.
  748 */
  749 int      __malloc_trim(size_t);
  750 
  751 /*
  752   malloc_usable_size(void* p);
  753 
  754   Returns the number of bytes you can actually use in
  755   an allocated chunk, which may be more than you requested (although
  756   often not) due to alignment and minimum size constraints.
  757   You can use this many bytes without worrying about
  758   overwriting other allocated objects. This is not a particularly great
  759   programming practice. malloc_usable_size can be more useful in
  760   debugging and assertions, for example:
  761 
  762   p = malloc(n);
  763   assert(malloc_usable_size(p) >= 256);
  764 
  765 */
  766 size_t   __malloc_usable_size(void*);
  767 
  768 /*
  769   malloc_stats();
  770   Prints on stderr the amount of space obtained from the system (both
  771   via sbrk and mmap), the maximum amount (which may be more than
  772   current if malloc_trim and/or munmap got called), and the current
  773   number of bytes allocated via malloc (or realloc, etc) but not yet
  774   freed. Note that this is the number of bytes allocated, not the
  775   number requested. It will be larger than the number requested
  776   because of alignment and bookkeeping overhead. Because it includes
  777   alignment wastage as being in use, this figure may be greater than
  778   zero even when no user-level chunks are allocated.
  779 
  780   The reported current and maximum system memory can be inaccurate if
  781   a program makes other calls to system memory allocation functions
  782   (normally sbrk) outside of malloc.
  783 
  784   malloc_stats prints only the most commonly interesting statistics.
  785   More information can be obtained by calling mallinfo.
  786 
  787 */
  788 void     __malloc_stats(void);
  789 
  790 /*
  791   posix_memalign(void **memptr, size_t alignment, size_t size);
  792 
  793   POSIX wrapper like memalign(), checking for validity of size.
  794 */
  795 int      __posix_memalign(void **, size_t, size_t);
  796 #endif /* IS_IN (libc) */
  797 
  798 /*
  799   mallopt(int parameter_number, int parameter_value)
  800   Sets tunable parameters The format is to provide a
  801   (parameter-number, parameter-value) pair.  mallopt then sets the
  802   corresponding parameter to the argument value if it can (i.e., so
  803   long as the value is meaningful), and returns 1 if successful else
  804   0.  SVID/XPG/ANSI defines four standard param numbers for mallopt,
  805   normally defined in malloc.h.  Only one of these (M_MXFAST) is used
  806   in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
  807   so setting them has no effect. But this malloc also supports four
  808   other options in mallopt. See below for details.  Briefly, supported
  809   parameters are as follows (listed defaults are for "typical"
  810   configurations).
  811 
  812   Symbol            param #   default    allowed param values
  813   M_MXFAST          1         64         0-80  (0 disables fastbins)
  814   M_TRIM_THRESHOLD -1         128*1024   any   (-1U disables trimming)
  815   M_TOP_PAD        -2         0          any
  816   M_MMAP_THRESHOLD -3         128*1024   any   (or 0 if no MMAP support)
  817   M_MMAP_MAX       -4         65536      any   (0 disables use of mmap)
  818 */
  819 int      __libc_mallopt(int, int);
  820 #if IS_IN (libc)
  821 libc_hidden_proto (__libc_mallopt)
  822 #endif
  823 
  824 /* mallopt tuning options */
  825 
  826 /*
  827   M_MXFAST is the maximum request size used for "fastbins", special bins
  828   that hold returned chunks without consolidating their spaces. This
  829   enables future requests for chunks of the same size to be handled
  830   very quickly, but can increase fragmentation, and thus increase the
  831   overall memory footprint of a program.
  832 
  833   This malloc manages fastbins very conservatively yet still
  834   efficiently, so fragmentation is rarely a problem for values less
  835   than or equal to the default.  The maximum supported value of MXFAST
  836   is 80. You wouldn't want it any higher than this anyway.  Fastbins
  837   are designed especially for use with many small structs, objects or
  838   strings -- the default handles structs/objects/arrays with sizes up
  839   to 8 4byte fields, or small strings representing words, tokens,
  840   etc. Using fastbins for larger objects normally worsens
  841   fragmentation without improving speed.
  842 
  843   M_MXFAST is set in REQUEST size units. It is internally used in
  844   chunksize units, which adds padding and alignment.  You can reduce
  845   M_MXFAST to 0 to disable all use of fastbins.  This causes the malloc
  846   algorithm to be a closer approximation of fifo-best-fit in all cases,
  847   not just for larger requests, but will generally cause it to be
  848   slower.
  849 */
  850 
  851 
  852 /* M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h */
  853 #ifndef M_MXFAST
  854 #define M_MXFAST            1
  855 #endif
  856 
  857 #ifndef DEFAULT_MXFAST
  858 #define DEFAULT_MXFAST     (64 * SIZE_SZ / 4)
  859 #endif
  860 
  861 
  862 /*
  863   M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
  864   to keep before releasing via malloc_trim in free().
  865 
  866   Automatic trimming is mainly useful in long-lived programs.
  867   Because trimming via sbrk can be slow on some systems, and can
  868   sometimes be wasteful (in cases where programs immediately
  869   afterward allocate more large chunks) the value should be high
  870   enough so that your overall system performance would improve by
  871   releasing this much memory.
  872 
  873   The trim threshold and the mmap control parameters (see below)
  874   can be traded off with one another. Trimming and mmapping are
  875   two different ways of releasing unused memory back to the
  876   system. Between these two, it is often possible to keep
  877   system-level demands of a long-lived program down to a bare
  878   minimum. For example, in one test suite of sessions measuring
  879   the XF86 X server on Linux, using a trim threshold of 128K and a
  880   mmap threshold of 192K led to near-minimal long term resource
  881   consumption.
  882 
  883   If you are using this malloc in a long-lived program, it should
  884   pay to experiment with these values.  As a rough guide, you
  885   might set to a value close to the average size of a process
  886   (program) running on your system.  Releasing this much memory
  887   would allow such a process to run in memory.  Generally, it's
  888   worth it to tune for trimming rather tham memory mapping when a
  889   program undergoes phases where several large chunks are
  890   allocated and released in ways that can reuse each other's
  891   storage, perhaps mixed with phases where there are no such
  892   chunks at all.  And in well-behaved long-lived programs,
  893   controlling release of large blocks via trimming versus mapping
  894   is usually faster.
  895 
  896   However, in most programs, these parameters serve mainly as
  897   protection against the system-level effects of carrying around
  898   massive amounts of unneeded memory. Since frequent calls to
  899   sbrk, mmap, and munmap otherwise degrade performance, the default
  900   parameters are set to relatively high values that serve only as
  901   safeguards.
  902 
  903   The trim value It must be greater than page size to have any useful
  904   effect.  To disable trimming completely, you can set to
  905   (unsigned long)(-1)
  906 
  907   Trim settings interact with fastbin (MXFAST) settings: Unless
  908   TRIM_FASTBINS is defined, automatic trimming never takes place upon
  909   freeing a chunk with size less than or equal to MXFAST. Trimming is
  910   instead delayed until subsequent freeing of larger chunks. However,
  911   you can still force an attempted trim by calling malloc_trim.
  912 
  913   Also, trimming is not generally possible in cases where
  914   the main arena is obtained via mmap.
  915 
  916   Note that the trick some people use of mallocing a huge space and
  917   then freeing it at program startup, in an attempt to reserve system
  918   memory, doesn't have the intended effect under automatic trimming,
  919   since that memory will immediately be returned to the system.
  920 */
  921 
  922 #define M_TRIM_THRESHOLD       -1
  923 
  924 #ifndef DEFAULT_TRIM_THRESHOLD
  925 #define DEFAULT_TRIM_THRESHOLD (128 * 1024)
  926 #endif
  927 
  928 /*
  929   M_TOP_PAD is the amount of extra `padding' space to allocate or
  930   retain whenever sbrk is called. It is used in two ways internally:
  931 
  932   * When sbrk is called to extend the top of the arena to satisfy
  933   a new malloc request, this much padding is added to the sbrk
  934   request.
  935 
  936   * When malloc_trim is called automatically from free(),
  937   it is used as the `pad' argument.
  938 
  939   In both cases, the actual amount of padding is rounded
  940   so that the end of the arena is always a system page boundary.
  941 
  942   The main reason for using padding is to avoid calling sbrk so
  943   often. Having even a small pad greatly reduces the likelihood
  944   that nearly every malloc request during program start-up (or
  945   after trimming) will invoke sbrk, which needlessly wastes
  946   time.
  947 
  948   Automatic rounding-up to page-size units is normally sufficient
  949   to avoid measurable overhead, so the default is 0.  However, in
  950   systems where sbrk is relatively slow, it can pay to increase
  951   this value, at the expense of carrying around more memory than
  952   the program needs.
  953 */
  954 
  955 #define M_TOP_PAD              -2
  956 
  957 #ifndef DEFAULT_TOP_PAD
  958 #define DEFAULT_TOP_PAD        (0)
  959 #endif
  960 
  961 /*
  962   MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
  963   adjusted MMAP_THRESHOLD.
  964 */
  965 
  966 #ifndef DEFAULT_MMAP_THRESHOLD_MIN
  967 #define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
  968 #endif
  969 
  970 #ifndef DEFAULT_MMAP_THRESHOLD_MAX
  971   /* For 32-bit platforms we cannot increase the maximum mmap
  972      threshold much because it is also the minimum value for the
  973      maximum heap size and its alignment.  Going above 512k (i.e., 1M
  974      for new heaps) wastes too much address space.  */
  975 # if __WORDSIZE == 32
  976 #  define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
  977 # else
  978 #  define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
  979 # endif
  980 #endif
  981 
  982 /*
  983   M_MMAP_THRESHOLD is the request size threshold for using mmap()
  984   to service a request. Requests of at least this size that cannot
  985   be allocated using already-existing space will be serviced via mmap.
  986   (If enough normal freed space already exists it is used instead.)
  987 
  988   Using mmap segregates relatively large chunks of memory so that
  989   they can be individually obtained and released from the host
  990   system. A request serviced through mmap is never reused by any
  991   other request (at least not directly; the system may just so
  992   happen to remap successive requests to the same locations).
  993 
  994   Segregating space in this way has the benefits that:
  995 
  996    1. Mmapped space can ALWAYS be individually released back
  997       to the system, which helps keep the system level memory
  998       demands of a long-lived program low.
  999    2. Mapped memory can never become `locked' between
 1000       other chunks, as can happen with normally allocated chunks, which
 1001       means that even trimming via malloc_trim would not release them.
 1002    3. On some systems with "holes" in address spaces, mmap can obtain
 1003       memory that sbrk cannot.
 1004 
 1005   However, it has the disadvantages that:
 1006 
 1007    1. The space cannot be reclaimed, consolidated, and then
 1008       used to service later requests, as happens with normal chunks.
 1009    2. It can lead to more wastage because of mmap page alignment
 1010       requirements
 1011    3. It causes malloc performance to be more dependent on host
 1012       system memory management support routines which may vary in
 1013       implementation quality and may impose arbitrary
 1014       limitations. Generally, servicing a request via normal
 1015       malloc steps is faster than going through a system's mmap.
 1016 
 1017   The advantages of mmap nearly always outweigh disadvantages for
 1018   "large" chunks, but the value of "large" varies across systems.  The
 1019   default is an empirically derived value that works well in most
 1020   systems.
 1021 
 1022 
 1023   Update in 2006:
 1024   The above was written in 2001. Since then the world has changed a lot.
 1025   Memory got bigger. Applications got bigger. The virtual address space
 1026   layout in 32 bit linux changed.
 1027 
 1028   In the new situation, brk() and mmap space is shared and there are no
 1029   artificial limits on brk size imposed by the kernel. What is more,
 1030   applications have started using transient allocations larger than the
 1031   128Kb as was imagined in 2001.
 1032 
 1033   The price for mmap is also high now; each time glibc mmaps from the
 1034   kernel, the kernel is forced to zero out the memory it gives to the
 1035   application. Zeroing memory is expensive and eats a lot of cache and
 1036   memory bandwidth. This has nothing to do with the efficiency of the
 1037   virtual memory system, by doing mmap the kernel just has no choice but
 1038   to zero.
 1039 
 1040   In 2001, the kernel had a maximum size for brk() which was about 800
 1041   megabytes on 32 bit x86, at that point brk() would hit the first
 1042   mmaped shared libaries and couldn't expand anymore. With current 2.6
 1043   kernels, the VA space layout is different and brk() and mmap
 1044   both can span the entire heap at will.
 1045 
 1046   Rather than using a static threshold for the brk/mmap tradeoff,
 1047   we are now using a simple dynamic one. The goal is still to avoid
 1048   fragmentation. The old goals we kept are
 1049   1) try to get the long lived large allocations to use mmap()
 1050   2) really large allocations should always use mmap()
 1051   and we're adding now:
 1052   3) transient allocations should use brk() to avoid forcing the kernel
 1053      having to zero memory over and over again
 1054 
 1055   The implementation works with a sliding threshold, which is by default
 1056   limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
 1057   out at 128Kb as per the 2001 default.
 1058 
 1059   This allows us to satisfy requirement 1) under the assumption that long
 1060   lived allocations are made early in the process' lifespan, before it has
 1061   started doing dynamic allocations of the same size (which will
 1062   increase the threshold).
 1063 
 1064   The upperbound on the threshold satisfies requirement 2)
 1065 
 1066   The threshold goes up in value when the application frees memory that was
 1067   allocated with the mmap allocator. The idea is that once the application
 1068   starts freeing memory of a certain size, it's highly probable that this is
 1069   a size the application uses for transient allocations. This estimator
 1070   is there to satisfy the new third requirement.
 1071 
 1072 */
 1073 
 1074 #define M_MMAP_THRESHOLD      -3
 1075 
 1076 #ifndef DEFAULT_MMAP_THRESHOLD
 1077 #define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
 1078 #endif
 1079 
 1080 /*
 1081   M_MMAP_MAX is the maximum number of requests to simultaneously
 1082   service using mmap. This parameter exists because
 1083   some systems have a limited number of internal tables for
 1084   use by mmap, and using more than a few of them may degrade
 1085   performance.
 1086 
 1087   The default is set to a value that serves only as a safeguard.
 1088   Setting to 0 disables use of mmap for servicing large requests.
 1089 */
 1090 
 1091 #define M_MMAP_MAX             -4
 1092 
 1093 #ifndef DEFAULT_MMAP_MAX
 1094 #define DEFAULT_MMAP_MAX       (65536)
 1095 #endif
 1096 
 1097 #include <malloc.h>
 1098 
 1099 #ifndef RETURN_ADDRESS
 1100 #define RETURN_ADDRESS(X_) (NULL)
 1101 #endif
 1102 
 1103 /* Forward declarations.  */
 1104 struct malloc_chunk;
 1105 typedef struct malloc_chunk* mchunkptr;
 1106 
 1107 /* Internal routines.  */
 1108 
 1109 static void*  _int_malloc(mstate, size_t);
 1110 static void     _int_free(mstate, mchunkptr, int);
 1111 static void*  _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T,
 1112                INTERNAL_SIZE_T);
 1113 static void*  _int_memalign(mstate, size_t, size_t);
 1114 #if IS_IN (libc)
 1115 static void*  _mid_memalign(size_t, size_t, void *);
 1116 #endif
 1117 
 1118 static void malloc_printerr(const char *str) __attribute__ ((noreturn));
 1119 
 1120 static void munmap_chunk(mchunkptr p);
 1121 #if HAVE_MREMAP
 1122 static mchunkptr mremap_chunk(mchunkptr p, size_t new_size);
 1123 #endif
 1124 
 1125 /* ------------------ MMAP support ------------------  */
 1126 
 1127 
 1128 #include <fcntl.h>
 1129 #include <sys/mman.h>
 1130 
 1131 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
 1132 # define MAP_ANONYMOUS MAP_ANON
 1133 #endif
 1134 
 1135 #ifndef MAP_NORESERVE
 1136 # define MAP_NORESERVE 0
 1137 #endif
 1138 
 1139 #define MMAP(addr, size, prot, flags) \
 1140  __mmap((addr), (size), (prot), (flags)|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0)
 1141 
 1142 
 1143 /*
 1144   -----------------------  Chunk representations -----------------------
 1145 */
 1146 
 1147 
 1148 /*
 1149   This struct declaration is misleading (but accurate and necessary).
 1150   It declares a "view" into memory allowing access to necessary
 1151   fields at known offsets from a given base. See explanation below.
 1152 */
 1153 
 1154 struct malloc_chunk {
 1155 
 1156   INTERNAL_SIZE_T      mchunk_prev_size;  /* Size of previous chunk (if free).  */
 1157   INTERNAL_SIZE_T      mchunk_size;       /* Size in bytes, including overhead. */
 1158 
 1159   struct malloc_chunk* fd;         /* double links -- used only if free. */
 1160   struct malloc_chunk* bk;
 1161 
 1162   /* Only used for large blocks: pointer to next larger size.  */
 1163   struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
 1164   struct malloc_chunk* bk_nextsize;
 1165 };
 1166 
 1167 
 1168 /*
 1169    malloc_chunk details:
 1170 
 1171     (The following includes lightly edited explanations by Colin Plumb.)
 1172 
 1173     Chunks of memory are maintained using a `boundary tag' method as
 1174     described in e.g., Knuth or Standish.  (See the paper by Paul
 1175     Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
 1176     survey of such techniques.)  Sizes of free chunks are stored both
 1177     in the front of each chunk and at the end.  This makes
 1178     consolidating fragmented chunks into bigger chunks very fast.  The
 1179     size fields also hold bits representing whether chunks are free or
 1180     in use.
 1181 
 1182     An allocated chunk looks like this:
 1183 
 1184 
 1185     chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1186         |             Size of previous chunk, if unallocated (P clear)  |
 1187         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1188         |             Size of chunk, in bytes                     |A|M|P|
 1189       mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1190         |             User data starts here...                          .
 1191         .                                                               .
 1192         .             (malloc_usable_size() bytes)                      .
 1193         .                                                               |
 1194 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1195         |             (size of chunk, but used for application data)    |
 1196         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1197         |             Size of next chunk, in bytes                |A|0|1|
 1198         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1199 
 1200     Where "chunk" is the front of the chunk for the purpose of most of
 1201     the malloc code, but "mem" is the pointer that is returned to the
 1202     user.  "Nextchunk" is the beginning of the next contiguous chunk.
 1203 
 1204     Chunks always begin on even word boundaries, so the mem portion
 1205     (which is returned to the user) is also on an even word boundary, and
 1206     thus at least double-word aligned.
 1207 
 1208     Free chunks are stored in circular doubly-linked lists, and look like this:
 1209 
 1210     chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1211         |             Size of previous chunk, if unallocated (P clear)  |
 1212         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1213     `head:' |             Size of chunk, in bytes                     |A|0|P|
 1214       mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1215         |             Forward pointer to next chunk in list             |
 1216         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1217         |             Back pointer to previous chunk in list            |
 1218         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1219         |             Unused space (may be 0 bytes long)                .
 1220         .                                                               .
 1221         .                                                               |
 1222 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1223     `foot:' |             Size of chunk, in bytes                           |
 1224         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1225         |             Size of next chunk, in bytes                |A|0|0|
 1226         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 1227 
 1228     The P (PREV_INUSE) bit, stored in the unused low-order bit of the
 1229     chunk size (which is always a multiple of two words), is an in-use
 1230     bit for the *previous* chunk.  If that bit is *clear*, then the
 1231     word before the current chunk size contains the previous chunk
 1232     size, and can be used to find the front of the previous chunk.
 1233     The very first chunk allocated always has this bit set,
 1234     preventing access to non-existent (or non-owned) memory. If
 1235     prev_inuse is set for any given chunk, then you CANNOT determine
 1236     the size of the previous chunk, and might even get a memory
 1237     addressing fault when trying to do so.
 1238 
 1239     The A (NON_MAIN_ARENA) bit is cleared for chunks on the initial,
 1240     main arena, described by the main_arena variable.  When additional
 1241     threads are spawned, each thread receives its own arena (up to a
 1242     configurable limit, after which arenas are reused for multiple
 1243     threads), and the chunks in these arenas have the A bit set.  To
 1244     find the arena for a chunk on such a non-main arena, heap_for_ptr
 1245     performs a bit mask operation and indirection through the ar_ptr
 1246     member of the per-heap header heap_info (see arena.c).
 1247 
 1248     Note that the `foot' of the current chunk is actually represented
 1249     as the prev_size of the NEXT chunk. This makes it easier to
 1250     deal with alignments etc but can be very confusing when trying
 1251     to extend or adapt this code.
 1252 
 1253     The three exceptions to all this are:
 1254 
 1255      1. The special chunk `top' doesn't bother using the
 1256     trailing size field since there is no next contiguous chunk
 1257     that would have to index off it. After initialization, `top'
 1258     is forced to always exist.  If it would become less than
 1259     MINSIZE bytes long, it is replenished.
 1260 
 1261      2. Chunks allocated via mmap, which have the second-lowest-order
 1262     bit M (IS_MMAPPED) set in their size fields.  Because they are
 1263     allocated one-by-one, each must contain its own trailing size
 1264     field.  If the M bit is set, the other bits are ignored
 1265     (because mmapped chunks are neither in an arena, nor adjacent
 1266     to a freed chunk).  The M bit is also used for chunks which
 1267     originally came from a dumped heap via malloc_set_state in
 1268     hooks.c.
 1269 
 1270      3. Chunks in fastbins are treated as allocated chunks from the
 1271     point of view of the chunk allocator.  They are consolidated
 1272     with their neighbors only in bulk, in malloc_consolidate.
 1273 */
 1274 
 1275 /*
 1276   ---------- Size and alignment checks and conversions ----------
 1277 */
 1278 
 1279 /* Conversion from malloc headers to user pointers, and back.  When
 1280    using memory tagging the user data and the malloc data structure
 1281    headers have distinct tags.  Converting fully from one to the other
 1282    involves extracting the tag at the other address and creating a
 1283    suitable pointer using it.  That can be quite expensive.  There are
 1284    cases when the pointers are not dereferenced (for example only used
 1285    for alignment check) so the tags are not relevant, and there are
 1286    cases when user data is not tagged distinctly from malloc headers
 1287    (user data is untagged because tagging is done late in malloc and
 1288    early in free).  User memory tagging across internal interfaces:
 1289 
 1290       sysmalloc: Returns untagged memory.
 1291       _int_malloc: Returns untagged memory.
 1292       _int_free: Takes untagged memory.
 1293       _int_memalign: Returns untagged memory.
 1294       _int_memalign: Returns untagged memory.
 1295       _mid_memalign: Returns tagged memory.
 1296       _int_realloc: Takes and returns tagged memory.
 1297 */
 1298 
 1299 /* The chunk header is two SIZE_SZ elements, but this is used widely, so
 1300    we define it here for clarity later.  */
 1301 #define CHUNK_HDR_SZ (2 * SIZE_SZ)
 1302 
 1303 /* Convert a chunk address to a user mem pointer without correcting
 1304    the tag.  */
 1305 #define chunk2mem(p) ((void*)((char*)(p) + CHUNK_HDR_SZ))
 1306 
 1307 /* Convert a chunk address to a user mem pointer and extract the right tag.  */
 1308 #define chunk2mem_tag(p) ((void*)tag_at ((char*)(p) + CHUNK_HDR_SZ))
 1309 
 1310 /* Convert a user mem pointer to a chunk address and extract the right tag.  */
 1311 #define mem2chunk(mem) ((mchunkptr)tag_at (((char*)(mem) - CHUNK_HDR_SZ)))
 1312 
 1313 /* The smallest possible chunk */
 1314 #define MIN_CHUNK_SIZE        (offsetof(struct malloc_chunk, fd_nextsize))
 1315 
 1316 /* The smallest size we can malloc is an aligned minimal chunk */
 1317 
 1318 #define MINSIZE  \
 1319   (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
 1320 
 1321 /* Check if m has acceptable alignment */
 1322 
 1323 #define aligned_OK(m)  (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
 1324 
 1325 #define misaligned_chunk(p) \
 1326   ((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \
 1327    & MALLOC_ALIGN_MASK)
 1328 
 1329 /* pad request bytes into a usable size -- internal version */
 1330 /* Note: This must be a macro that evaluates to a compile time constant
 1331    if passed a literal constant.  */
 1332 #define request2size(req)                                         \
 1333   (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE)  ?             \
 1334    MINSIZE :                                                      \
 1335    ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
 1336 
 1337 /* Check if REQ overflows when padded and aligned and if the resulting value
 1338    is less than PTRDIFF_T.  Returns TRUE and the requested size or MINSIZE in
 1339    case the value is less than MINSIZE on SZ or false if any of the previous
 1340    check fail.  */
 1341 static inline bool
 1342 checked_request2size (size_t req, size_t *sz) __nonnull (1)
 1343 {
 1344   if (__glibc_unlikely (req > PTRDIFF_MAX))
 1345     return false;
 1346 
 1347   /* When using tagged memory, we cannot share the end of the user
 1348      block with the header for the next chunk, so ensure that we
 1349      allocate blocks that are rounded up to the granule size.  Take
 1350      care not to overflow from close to MAX_SIZE_T to a small
 1351      number.  Ideally, this would be part of request2size(), but that
 1352      must be a macro that produces a compile time constant if passed
 1353      a constant literal.  */
 1354   if (__glibc_unlikely (mtag_enabled))
 1355     {
 1356       /* Ensure this is not evaluated if !mtag_enabled, see gcc PR 99551.  */
 1357       asm ("");
 1358 
 1359       req = (req + (__MTAG_GRANULE_SIZE - 1)) &
 1360         ~(size_t)(__MTAG_GRANULE_SIZE - 1);
 1361     }
 1362 
 1363   *sz = request2size (req);
 1364   return true;
 1365 }
 1366 
 1367 /*
 1368    --------------- Physical chunk operations ---------------
 1369  */
 1370 
 1371 
 1372 /* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
 1373 #define PREV_INUSE 0x1
 1374 
 1375 /* extract inuse bit of previous chunk */
 1376 #define prev_inuse(p)       ((p)->mchunk_size & PREV_INUSE)
 1377 
 1378 
 1379 /* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
 1380 #define IS_MMAPPED 0x2
 1381 
 1382 /* check for mmap()'ed chunk */
 1383 #define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
 1384 
 1385 
 1386 /* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
 1387    from a non-main arena.  This is only set immediately before handing
 1388    the chunk to the user, if necessary.  */
 1389 #define NON_MAIN_ARENA 0x4
 1390 
 1391 /* Check for chunk from main arena.  */
 1392 #define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
 1393 
 1394 /* Mark a chunk as not being on the main arena.  */
 1395 #define set_non_main_arena(p) ((p)->mchunk_size |= NON_MAIN_ARENA)
 1396 
 1397 
 1398 /*
 1399    Bits to mask off when extracting size
 1400 
 1401    Note: IS_MMAPPED is intentionally not masked off from size field in
 1402    macros for which mmapped chunks should never be seen. This should
 1403    cause helpful core dumps to occur if it is tried by accident by
 1404    people extending or adapting this malloc.
 1405  */
 1406 #define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
 1407 
 1408 /* Get size, ignoring use bits */
 1409 #define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
 1410 
 1411 /* Like chunksize, but do not mask SIZE_BITS.  */
 1412 #define chunksize_nomask(p)         ((p)->mchunk_size)
 1413 
 1414 /* Ptr to next physical malloc_chunk. */
 1415 #define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
 1416 
 1417 /* Size of the chunk below P.  Only valid if !prev_inuse (P).  */
 1418 #define prev_size(p) ((p)->mchunk_prev_size)
 1419 
 1420 /* Set the size of the chunk below P.  Only valid if !prev_inuse (P).  */
 1421 #define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz))
 1422 
 1423 /* Ptr to previous physical malloc_chunk.  Only valid if !prev_inuse (P).  */
 1424 #define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p)))
 1425 
 1426 /* Treat space at ptr + offset as a chunk */
 1427 #define chunk_at_offset(p, s)  ((mchunkptr) (((char *) (p)) + (s)))
 1428 
 1429 /* extract p's inuse bit */
 1430 #define inuse(p)                                  \
 1431   ((((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size) & PREV_INUSE)
 1432 
 1433 /* set/clear chunk as being inuse without otherwise disturbing */
 1434 #define set_inuse(p)                                  \
 1435   ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size |= PREV_INUSE
 1436 
 1437 #define clear_inuse(p)                                \
 1438   ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size &= ~(PREV_INUSE)
 1439 
 1440 
 1441 /* check/set/clear inuse bits in known places */
 1442 #define inuse_bit_at_offset(p, s)                         \
 1443   (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE)
 1444 
 1445 #define set_inuse_bit_at_offset(p, s)                         \
 1446   (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size |= PREV_INUSE)
 1447 
 1448 #define clear_inuse_bit_at_offset(p, s)                       \
 1449   (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE))
 1450 
 1451 
 1452 /* Set size at head, without disturbing its use bit */
 1453 #define set_head_size(p, s)  ((p)->mchunk_size = (((p)->mchunk_size & SIZE_BITS) | (s)))
 1454 
 1455 /* Set size/use field */
 1456 #define set_head(p, s)       ((p)->mchunk_size = (s))
 1457 
 1458 /* Set size at footer (only when chunk is not in use) */
 1459 #define set_foot(p, s)       (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s))
 1460 
 1461 #pragma GCC poison mchunk_size
 1462 #pragma GCC poison mchunk_prev_size
 1463 
 1464 /* This is the size of the real usable data in the chunk.  Not valid for
 1465    dumped heap chunks.  */
 1466 #define memsize(p)                                                    \
 1467   (__MTAG_GRANULE_SIZE > SIZE_SZ && __glibc_unlikely (mtag_enabled) ? \
 1468     chunksize (p) - CHUNK_HDR_SZ :                                    \
 1469     chunksize (p) - CHUNK_HDR_SZ + (chunk_is_mmapped (p) ? 0 : SIZE_SZ))
 1470 
 1471 /* If memory tagging is enabled the layout changes to accommodate the granule
 1472    size, this is wasteful for small allocations so not done by default.
 1473    Both the chunk header and user data has to be granule aligned.  */
 1474 _Static_assert (__MTAG_GRANULE_SIZE <= CHUNK_HDR_SZ,
 1475         "memory tagging is not supported with large granule.");
 1476 
 1477 static __always_inline void *
 1478 tag_new_usable (void *ptr)
 1479 {
 1480   if (__glibc_unlikely (mtag_enabled) && ptr)
 1481     {
 1482       mchunkptr cp = mem2chunk(ptr);
 1483       ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), memsize (cp));
 1484     }
 1485   return ptr;
 1486 }
 1487 
 1488 /*
 1489    -------------------- Internal data structures --------------------
 1490 
 1491    All internal state is held in an instance of malloc_state defined
 1492    below. There are no other static variables, except in two optional
 1493    cases:
 1494  * If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
 1495  * If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
 1496      for mmap.
 1497 
 1498    Beware of lots of tricks that minimize the total bookkeeping space
 1499    requirements. The result is a little over 1K bytes (for 4byte
 1500    pointers and size_t.)
 1501  */
 1502 
 1503 /*
 1504    Bins
 1505 
 1506     An array of bin headers for free chunks. Each bin is doubly
 1507     linked.  The bins are approximately proportionally (log) spaced.
 1508     There are a lot of these bins (128). This may look excessive, but
 1509     works very well in practice.  Most bins hold sizes that are
 1510     unusual as malloc request sizes, but are more usual for fragments
 1511     and consolidated sets of chunks, which is what these bins hold, so
 1512     they can be found quickly.  All procedures maintain the invariant
 1513     that no consolidated chunk physically borders another one, so each
 1514     chunk in a list is known to be preceeded and followed by either
 1515     inuse chunks or the ends of memory.
 1516 
 1517     Chunks in bins are kept in size order, with ties going to the
 1518     approximately least recently used chunk. Ordering isn't needed
 1519     for the small bins, which all contain the same-sized chunks, but
 1520     facilitates best-fit allocation for larger chunks. These lists
 1521     are just sequential. Keeping them in order almost never requires
 1522     enough traversal to warrant using fancier ordered data
 1523     structures.
 1524 
 1525     Chunks of the same size are linked with the most
 1526     recently freed at the front, and allocations are taken from the
 1527     back.  This results in LRU (FIFO) allocation order, which tends
 1528     to give each chunk an equal opportunity to be consolidated with
 1529     adjacent freed chunks, resulting in larger free chunks and less
 1530     fragmentation.
 1531 
 1532     To simplify use in double-linked lists, each bin header acts
 1533     as a malloc_chunk. This avoids special-casing for headers.
 1534     But to conserve space and improve locality, we allocate
 1535     only the fd/bk pointers of bins, and then use repositioning tricks
 1536     to treat these as the fields of a malloc_chunk*.
 1537  */
 1538 
 1539 typedef struct malloc_chunk *mbinptr;
 1540 
 1541 /* addressing -- note that bin_at(0) does not exist */
 1542 #define bin_at(m, i) \
 1543   (mbinptr) (((char *) &((m)->bins[((i) - 1) * 2]))               \
 1544              - offsetof (struct malloc_chunk, fd))
 1545 
 1546 /* analog of ++bin */
 1547 #define next_bin(b)  ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1)))
 1548 
 1549 /* Reminders about list directionality within bins */
 1550 #define first(b)     ((b)->fd)
 1551 #define last(b)      ((b)->bk)
 1552 
 1553 /*
 1554    Indexing
 1555 
 1556     Bins for sizes < 512 bytes contain chunks of all the same size, spaced
 1557     8 bytes apart. Larger bins are approximately logarithmically spaced:
 1558 
 1559     64 bins of size       8
 1560     32 bins of size      64
 1561     16 bins of size     512
 1562      8 bins of size    4096
 1563      4 bins of size   32768
 1564      2 bins of size  262144
 1565      1 bin  of size what's left
 1566 
 1567     There is actually a little bit of slop in the numbers in bin_index
 1568     for the sake of speed. This makes no difference elsewhere.
 1569 
 1570     The bins top out around 1MB because we expect to service large
 1571     requests via mmap.
 1572 
 1573     Bin 0 does not exist.  Bin 1 is the unordered list; if that would be
 1574     a valid chunk size the small bins are bumped up one.
 1575  */
 1576 
 1577 #define NBINS             128
 1578 #define NSMALLBINS         64
 1579 #define SMALLBIN_WIDTH    MALLOC_ALIGNMENT
 1580 #define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ)
 1581 #define MIN_LARGE_SIZE    ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
 1582 
 1583 #define in_smallbin_range(sz)  \
 1584   ((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE)
 1585 
 1586 #define smallbin_index(sz) \
 1587   ((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\
 1588    + SMALLBIN_CORRECTION)
 1589 
 1590 #define largebin_index_32(sz)                                                \
 1591   (((((unsigned long) (sz)) >> 6) <= 38) ?  56 + (((unsigned long) (sz)) >> 6) :\
 1592    ((((unsigned long) (sz)) >> 9) <= 20) ?  91 + (((unsigned long) (sz)) >> 9) :\
 1593    ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
 1594    ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
 1595    ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
 1596    126)
 1597 
 1598 #define largebin_index_32_big(sz)                                            \
 1599   (((((unsigned long) (sz)) >> 6) <= 45) ?  49 + (((unsigned long) (sz)) >> 6) :\
 1600    ((((unsigned long) (sz)) >> 9) <= 20) ?  91 + (((unsigned long) (sz)) >> 9) :\
 1601    ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
 1602    ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
 1603    ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
 1604    126)
 1605 
 1606 // XXX It remains to be seen whether it is good to keep the widths of
 1607 // XXX the buckets the same or whether it should be scaled by a factor
 1608 // XXX of two as well.
 1609 #define largebin_index_64(sz)                                                \
 1610   (((((unsigned long) (sz)) >> 6) <= 48) ?  48 + (((unsigned long) (sz)) >> 6) :\
 1611    ((((unsigned long) (sz)) >> 9) <= 20) ?  91 + (((unsigned long) (sz)) >> 9) :\
 1612    ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
 1613    ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
 1614    ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
 1615    126)
 1616 
 1617 #define largebin_index(sz) \
 1618   (SIZE_SZ == 8 ? largebin_index_64 (sz)                                     \
 1619    : MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz)                     \
 1620    : largebin_index_32 (sz))
 1621 
 1622 #define bin_index(sz) \
 1623   ((in_smallbin_range (sz)) ? smallbin_index (sz) : largebin_index (sz))
 1624 
 1625 /* Take a chunk off a bin list.  */
 1626 static void
 1627 unlink_chunk (mstate av, mchunkptr p)
 1628 {
 1629   if (chunksize (p) != prev_size (next_chunk (p)))
 1630     malloc_printerr ("corrupted size vs. prev_size");
 1631 
 1632   mchunkptr fd = p->fd;
 1633   mchunkptr bk = p->bk;
 1634 
 1635   if (__builtin_expect (fd->bk != p || bk->fd != p, 0))
 1636     malloc_printerr ("corrupted double-linked list");
 1637 
 1638   fd->bk = bk;
 1639   bk->fd = fd;
 1640   if (!in_smallbin_range (chunksize_nomask (p)) && p->fd_nextsize != NULL)
 1641     {
 1642       if (p->fd_nextsize->bk_nextsize != p
 1643       || p->bk_nextsize->fd_nextsize != p)
 1644     malloc_printerr ("corrupted double-linked list (not small)");
 1645 
 1646       if (fd->fd_nextsize == NULL)
 1647     {
 1648       if (p->fd_nextsize == p)
 1649         fd->fd_nextsize = fd->bk_nextsize = fd;
 1650       else
 1651         {
 1652           fd->fd_nextsize = p->fd_nextsize;
 1653           fd->bk_nextsize = p->bk_nextsize;
 1654           p->fd_nextsize->bk_nextsize = fd;
 1655           p->bk_nextsize->fd_nextsize = fd;
 1656         }
 1657     }
 1658       else
 1659     {
 1660       p->fd_nextsize->bk_nextsize = p->bk_nextsize;
 1661       p->bk_nextsize->fd_nextsize = p->fd_nextsize;
 1662     }
 1663     }
 1664 }
 1665 
 1666 /*
 1667    Unsorted chunks
 1668 
 1669     All remainders from chunk splits, as well as all returned chunks,
 1670     are first placed in the "unsorted" bin. They are then placed
 1671     in regular bins after malloc gives them ONE chance to be used before
 1672     binning. So, basically, the unsorted_chunks list acts as a queue,
 1673     with chunks being placed on it in free (and malloc_consolidate),
 1674     and taken off (to be either used or placed in bins) in malloc.
 1675 
 1676     The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
 1677     does not have to be taken into account in size comparisons.
 1678  */
 1679 
 1680 /* The otherwise unindexable 1-bin is used to hold unsorted chunks. */
 1681 #define unsorted_chunks(M)          (bin_at (M, 1))
 1682 
 1683 /*
 1684    Top
 1685 
 1686     The top-most available chunk (i.e., the one bordering the end of
 1687     available memory) is treated specially. It is never included in
 1688     any bin, is used only if no other chunk is available, and is
 1689     released back to the system if it is very large (see
 1690     M_TRIM_THRESHOLD).  Because top initially
 1691     points to its own bin with initial zero size, thus forcing
 1692     extension on the first malloc request, we avoid having any special
 1693     code in malloc to check whether it even exists yet. But we still
 1694     need to do so when getting memory from system, so we make
 1695     initial_top treat the bin as a legal but unusable chunk during the
 1696     interval between initialization and the first call to
 1697     sysmalloc. (This is somewhat delicate, since it relies on
 1698     the 2 preceding words to be zero during this interval as well.)
 1699  */
 1700 
 1701 /* Conveniently, the unsorted bin can be used as dummy top on first call */
 1702 #define initial_top(M)              (unsorted_chunks (M))
 1703 
 1704 /*
 1705    Binmap
 1706 
 1707     To help compensate for the large number of bins, a one-level index
 1708     structure is used for bin-by-bin searching.  `binmap' is a
 1709     bitvector recording whether bins are definitely empty so they can
 1710     be skipped over during during traversals.  The bits are NOT always
 1711     cleared as soon as bins are empty, but instead only
 1712     when they are noticed to be empty during traversal in malloc.
 1713  */
 1714 
 1715 /* Conservatively use 32 bits per map word, even if on 64bit system */
 1716 #define BINMAPSHIFT      5
 1717 #define BITSPERMAP       (1U << BINMAPSHIFT)
 1718 #define BINMAPSIZE       (NBINS / BITSPERMAP)
 1719 
 1720 #define idx2block(i)     ((i) >> BINMAPSHIFT)
 1721 #define idx2bit(i)       ((1U << ((i) & ((1U << BINMAPSHIFT) - 1))))
 1722 
 1723 #define mark_bin(m, i)    ((m)->binmap[idx2block (i)] |= idx2bit (i))
 1724 #define unmark_bin(m, i)  ((m)->binmap[idx2block (i)] &= ~(idx2bit (i)))
 1725 #define get_binmap(m, i)  ((m)->binmap[idx2block (i)] & idx2bit (i))
 1726 
 1727 /*
 1728    Fastbins
 1729 
 1730     An array of lists holding recently freed small chunks.  Fastbins
 1731     are not doubly linked.  It is faster to single-link them, and
 1732     since chunks are never removed from the middles of these lists,
 1733     double linking is not necessary. Also, unlike regular bins, they
 1734     are not even processed in FIFO order (they use faster LIFO) since
 1735     ordering doesn't much matter in the transient contexts in which
 1736     fastbins are normally used.
 1737 
 1738     Chunks in fastbins keep their inuse bit set, so they cannot
 1739     be consolidated with other free chunks. malloc_consolidate
 1740     releases all chunks in fastbins and consolidates them with
 1741     other free chunks.
 1742  */
 1743 
 1744 typedef struct malloc_chunk *mfastbinptr;
 1745 #define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
 1746 
 1747 /* offset 2 to use otherwise unindexable first 2 bins */
 1748 #define fastbin_index(sz) \
 1749   ((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
 1750 
 1751 
 1752 /* The maximum fastbin request size we support */
 1753 #define MAX_FAST_SIZE     (80 * SIZE_SZ / 4)
 1754 
 1755 #define NFASTBINS  (fastbin_index (request2size (MAX_FAST_SIZE)) + 1)
 1756 
 1757 /*
 1758    FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
 1759    that triggers automatic consolidation of possibly-surrounding
 1760    fastbin chunks. This is a heuristic, so the exact value should not
 1761    matter too much. It is defined at half the default trim threshold as a
 1762    compromise heuristic to only attempt consolidation if it is likely
 1763    to lead to trimming. However, it is not dynamically tunable, since
 1764    consolidation reduces fragmentation surrounding large chunks even
 1765    if trimming is not used.
 1766  */
 1767 
 1768 #define FASTBIN_CONSOLIDATION_THRESHOLD  (65536UL)
 1769 
 1770 /*
 1771    NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
 1772    regions.  Otherwise, contiguity is exploited in merging together,
 1773    when possible, results from consecutive MORECORE calls.
 1774 
 1775    The initial value comes from MORECORE_CONTIGUOUS, but is
 1776    changed dynamically if mmap is ever used as an sbrk substitute.
 1777  */
 1778 
 1779 #define NONCONTIGUOUS_BIT     (2U)
 1780 
 1781 #define contiguous(M)          (((M)->flags & NONCONTIGUOUS_BIT) == 0)
 1782 #define noncontiguous(M)       (((M)->flags & NONCONTIGUOUS_BIT) != 0)
 1783 #define set_noncontiguous(M)   ((M)->flags |= NONCONTIGUOUS_BIT)
 1784 #define set_contiguous(M)      ((M)->flags &= ~NONCONTIGUOUS_BIT)
 1785 
 1786 /* Maximum size of memory handled in fastbins.  */
 1787 static INTERNAL_SIZE_T global_max_fast;
 1788 
 1789 /*
 1790    Set value of max_fast.
 1791    Use impossibly small value if 0.
 1792    Precondition: there are no existing fastbin chunks in the main arena.
 1793    Since do_check_malloc_state () checks this, we call malloc_consolidate ()
 1794    before changing max_fast.  Note other arenas will leak their fast bin
 1795    entries if max_fast is reduced.
 1796  */
 1797 
 1798 #define set_max_fast(s) \
 1799   global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ)  \
 1800                      ? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
 1801 
 1802 static inline INTERNAL_SIZE_T
 1803 get_max_fast (void)
 1804 {
 1805   /* Tell the GCC optimizers that global_max_fast is never larger
 1806      than MAX_FAST_SIZE.  This avoids out-of-bounds array accesses in
 1807      _int_malloc after constant propagation of the size parameter.
 1808      (The code never executes because malloc preserves the
 1809      global_max_fast invariant, but the optimizers may not recognize
 1810      this.)  */
 1811   if (global_max_fast > MAX_FAST_SIZE)
 1812     __builtin_unreachable ();
 1813   return global_max_fast;
 1814 }
 1815 
 1816 /*
 1817    ----------- Internal state representation and initialization -----------
 1818  */
 1819 
 1820 /*
 1821    have_fastchunks indicates that there are probably some fastbin chunks.
 1822    It is set true on entering a chunk into any fastbin, and cleared early in
 1823    malloc_consolidate.  The value is approximate since it may be set when there
 1824    are no fastbin chunks, or it may be clear even if there are fastbin chunks
 1825    available.  Given it's sole purpose is to reduce number of redundant calls to
 1826    malloc_consolidate, it does not affect correctness.  As a result we can safely
 1827    use relaxed atomic accesses.
 1828  */
 1829 
 1830 
 1831 struct malloc_state
 1832 {
 1833   /* Serialize access.  */
 1834   __libc_lock_define (, mutex);
 1835 
 1836   /* Flags (formerly in max_fast).  */
 1837   int flags;
 1838 
 1839   /* Set if the fastbin chunks contain recently inserted free blocks.  */
 1840   /* Note this is a bool but not all targets support atomics on booleans.  */
 1841   int have_fastchunks;
 1842 
 1843   /* Fastbins */
 1844   mfastbinptr fastbinsY[NFASTBINS];
 1845 
 1846   /* Base of the topmost chunk -- not otherwise kept in a bin */
 1847   mchunkptr top;
 1848 
 1849   /* The remainder from the most recent split of a small request */
 1850   mchunkptr last_remainder;
 1851 
 1852   /* Normal bins packed as described above */
 1853   mchunkptr bins[NBINS * 2 - 2];
 1854 
 1855   /* Bitmap of bins */
 1856   unsigned int binmap[BINMAPSIZE];
 1857 
 1858   /* Linked list */
 1859   struct malloc_state *next;
 1860 
 1861   /* Linked list for free arenas.  Access to this field is serialized
 1862      by free_list_lock in arena.c.  */
 1863   struct malloc_state *next_free;
 1864 
 1865   /* Number of threads attached to this arena.  0 if the arena is on
 1866      the free list.  Access to this field is serialized by
 1867      free_list_lock in arena.c.  */
 1868   INTERNAL_SIZE_T attached_threads;
 1869 
 1870   /* Memory allocated from the system in this arena.  */
 1871   INTERNAL_SIZE_T system_mem;
 1872   INTERNAL_SIZE_T max_system_mem;
 1873 };
 1874 
 1875 struct malloc_par
 1876 {
 1877   /* Tunable parameters */
 1878   unsigned long trim_threshold;
 1879   INTERNAL_SIZE_T top_pad;
 1880   INTERNAL_SIZE_T mmap_threshold;
 1881   INTERNAL_SIZE_T arena_test;
 1882   INTERNAL_SIZE_T arena_max;
 1883 
 1884   /* Memory map support */
 1885   int n_mmaps;
 1886   int n_mmaps_max;
 1887   int max_n_mmaps;
 1888   /* the mmap_threshold is dynamic, until the user sets
 1889      it manually, at which point we need to disable any
 1890      dynamic behavior. */
 1891   int no_dyn_threshold;
 1892 
 1893   /* Statistics */
 1894   INTERNAL_SIZE_T mmapped_mem;
 1895   INTERNAL_SIZE_T max_mmapped_mem;
 1896 
 1897   /* First address handed out by MORECORE/sbrk.  */
 1898   char *sbrk_base;
 1899 
 1900 #if USE_TCACHE
 1901   /* Maximum number of buckets to use.  */
 1902   size_t tcache_bins;
 1903   size_t tcache_max_bytes;
 1904   /* Maximum number of chunks in each bucket.  */
 1905   size_t tcache_count;
 1906   /* Maximum number of chunks to remove from the unsorted list, which
 1907      aren't used to prefill the cache.  */
 1908   size_t tcache_unsorted_limit;
 1909 #endif
 1910 };
 1911 
 1912 /* There are several instances of this struct ("arenas") in this
 1913    malloc.  If you are adapting this malloc in a way that does NOT use
 1914    a static or mmapped malloc_state, you MUST explicitly zero-fill it
 1915    before using. This malloc relies on the property that malloc_state
 1916    is initialized to all zeroes (as is true of C statics).  */
 1917 
 1918 static struct malloc_state main_arena =
 1919 {
 1920   .mutex = _LIBC_LOCK_INITIALIZER,
 1921   .next = &main_arena,
 1922   .attached_threads = 1
 1923 };
 1924 
 1925 /* There is only one instance of the malloc parameters.  */
 1926 
 1927 static struct malloc_par mp_ =
 1928 {
 1929   .top_pad = DEFAULT_TOP_PAD,
 1930   .n_mmaps_max = DEFAULT_MMAP_MAX,
 1931   .mmap_threshold = DEFAULT_MMAP_THRESHOLD,
 1932   .trim_threshold = DEFAULT_TRIM_THRESHOLD,
 1933 #define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8))
 1934   .arena_test = NARENAS_FROM_NCORES (1)
 1935 #if USE_TCACHE
 1936   ,
 1937   .tcache_count = TCACHE_FILL_COUNT,
 1938   .tcache_bins = TCACHE_MAX_BINS,
 1939   .tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-1),
 1940   .tcache_unsorted_limit = 0 /* No limit.  */
 1941 #endif
 1942 };
 1943 
 1944 /*
 1945    Initialize a malloc_state struct.
 1946 
 1947    This is called from ptmalloc_init () or from _int_new_arena ()
 1948    when creating a new arena.
 1949  */
 1950 
 1951 static void
 1952 malloc_init_state (mstate av)
 1953 {
 1954   int i;
 1955   mbinptr bin;
 1956 
 1957   /* Establish circular links for normal bins */
 1958   for (i = 1; i < NBINS; ++i)
 1959     {
 1960       bin = bin_at (av, i);
 1961       bin->fd = bin->bk = bin;
 1962     }
 1963 
 1964 #if MORECORE_CONTIGUOUS
 1965   if (av != &main_arena)
 1966 #endif
 1967   set_noncontiguous (av);
 1968   if (av == &main_arena)
 1969     set_max_fast (DEFAULT_MXFAST);
 1970   atomic_store_relaxed (&av->have_fastchunks, false);
 1971 
 1972   av->top = initial_top (av);
 1973 }
 1974 
 1975 /*
 1976    Other internal utilities operating on mstates
 1977  */
 1978 
 1979 static void *sysmalloc (INTERNAL_SIZE_T, mstate);
 1980 static int      systrim (size_t, mstate);
 1981 static void     malloc_consolidate (mstate);
 1982 
 1983 
 1984 /* -------------- Early definitions for debugging hooks ---------------- */
 1985 
 1986 /* This function is called from the arena shutdown hook, to free the
 1987    thread cache (if it exists).  */
 1988 static void tcache_thread_shutdown (void);
 1989 
 1990 /* ------------------ Testing support ----------------------------------*/
 1991 
 1992 static int perturb_byte;
 1993 
 1994 static void
 1995 alloc_perturb (char *p, size_t n)
 1996 {
 1997   if (__glibc_unlikely (perturb_byte))
 1998     memset (p, perturb_byte ^ 0xff, n);
 1999 }
 2000 
 2001 static void
 2002 free_perturb (char *p, size_t n)
 2003 {
 2004   if (__glibc_unlikely (perturb_byte))
 2005     memset (p, perturb_byte, n);
 2006 }
 2007 
 2008 
 2009 
 2010 #include <stap-probe.h>
 2011 
 2012 /* ------------------- Support for multiple arenas -------------------- */
 2013 #include "arena.c"
 2014 
 2015 /*
 2016    Debugging support
 2017 
 2018    These routines make a number of assertions about the states
 2019    of data structures that should be true at all times. If any
 2020    are not true, it's very likely that a user program has somehow
 2021    trashed memory. (It's also possible that there is a coding error
 2022    in malloc. In which case, please report it!)
 2023  */
 2024 
 2025 #if !MALLOC_DEBUG
 2026 
 2027 # define check_chunk(A, P)
 2028 # define check_free_chunk(A, P)
 2029 # define check_inuse_chunk(A, P)
 2030 # define check_remalloced_chunk(A, P, N)
 2031 # define check_malloced_chunk(A, P, N)
 2032 # define check_malloc_state(A)
 2033 
 2034 #else
 2035 
 2036 # define check_chunk(A, P)              do_check_chunk (A, P)
 2037 # define check_free_chunk(A, P)         do_check_free_chunk (A, P)
 2038 # define check_inuse_chunk(A, P)        do_check_inuse_chunk (A, P)
 2039 # define check_remalloced_chunk(A, P, N) do_check_remalloced_chunk (A, P, N)
 2040 # define check_malloced_chunk(A, P, N)   do_check_malloced_chunk (A, P, N)
 2041 # define check_malloc_state(A)         do_check_malloc_state (A)
 2042 
 2043 /*
 2044    Properties of all chunks
 2045  */
 2046 
 2047 static void
 2048 do_check_chunk (mstate av, mchunkptr p)
 2049 {
 2050   unsigned long sz = chunksize (p);
 2051   /* min and max possible addresses assuming contiguous allocation */
 2052   char *max_address = (char *) (av->top) + chunksize (av->top);
 2053   char *min_address = max_address - av->system_mem;
 2054 
 2055   if (!chunk_is_mmapped (p))
 2056     {
 2057       /* Has legal address ... */
 2058       if (p != av->top)
 2059         {
 2060           if (contiguous (av))
 2061             {
 2062               assert (((char *) p) >= min_address);
 2063               assert (((char *) p + sz) <= ((char *) (av->top)));
 2064             }
 2065         }
 2066       else
 2067         {
 2068           /* top size is always at least MINSIZE */
 2069           assert ((unsigned long) (sz) >= MINSIZE);
 2070           /* top predecessor always marked inuse */
 2071           assert (prev_inuse (p));
 2072         }
 2073     }
 2074   else
 2075     {
 2076       /* address is outside main heap  */
 2077       if (contiguous (av) && av->top != initial_top (av))
 2078         {
 2079           assert (((char *) p) < min_address || ((char *) p) >= max_address);
 2080         }
 2081       /* chunk is page-aligned */
 2082       assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - 1)) == 0);
 2083       /* mem is aligned */
 2084       assert (aligned_OK (chunk2mem (p)));
 2085     }
 2086 }
 2087 
 2088 /*
 2089    Properties of free chunks
 2090  */
 2091 
 2092 static void
 2093 do_check_free_chunk (mstate av, mchunkptr p)
 2094 {
 2095   INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE | NON_MAIN_ARENA);
 2096   mchunkptr next = chunk_at_offset (p, sz);
 2097 
 2098   do_check_chunk (av, p);
 2099 
 2100   /* Chunk must claim to be free ... */
 2101   assert (!inuse (p));
 2102   assert (!chunk_is_mmapped (p));
 2103 
 2104   /* Unless a special marker, must have OK fields */
 2105   if ((unsigned long) (sz) >= MINSIZE)
 2106     {
 2107       assert ((sz & MALLOC_ALIGN_MASK) == 0);
 2108       assert (aligned_OK (chunk2mem (p)));
 2109       /* ... matching footer field */
 2110       assert (prev_size (next_chunk (p)) == sz);
 2111       /* ... and is fully consolidated */
 2112       assert (prev_inuse (p));
 2113       assert (next == av->top || inuse (next));
 2114 
 2115       /* ... and has minimally sane links */
 2116       assert (p->fd->bk == p);
 2117       assert (p->bk->fd == p);
 2118     }
 2119   else /* markers are always of size SIZE_SZ */
 2120     assert (sz == SIZE_SZ);
 2121 }
 2122 
 2123 /*
 2124    Properties of inuse chunks
 2125  */
 2126 
 2127 static void
 2128 do_check_inuse_chunk (mstate av, mchunkptr p)
 2129 {
 2130   mchunkptr next;
 2131 
 2132   do_check_chunk (av, p);
 2133 
 2134   if (chunk_is_mmapped (p))
 2135     return; /* mmapped chunks have no next/prev */
 2136 
 2137   /* Check whether it claims to be in use ... */
 2138   assert (inuse (p));
 2139 
 2140   next = next_chunk (p);
 2141 
 2142   /* ... and is surrounded by OK chunks.
 2143      Since more things can be checked with free chunks than inuse ones,
 2144      if an inuse chunk borders them and debug is on, it's worth doing them.
 2145    */
 2146   if (!prev_inuse (p))
 2147     {
 2148       /* Note that we cannot even look at prev unless it is not inuse */
 2149       mchunkptr prv = prev_chunk (p);
 2150       assert (next_chunk (prv) == p);
 2151       do_check_free_chunk (av, prv);
 2152     }
 2153 
 2154   if (next == av->top)
 2155     {
 2156       assert (prev_inuse (next));
 2157       assert (chunksize (next) >= MINSIZE);
 2158     }
 2159   else if (!inuse (next))
 2160     do_check_free_chunk (av, next);
 2161 }
 2162 
 2163 /*
 2164    Properties of chunks recycled from fastbins
 2165  */
 2166 
 2167 static void
 2168 do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
 2169 {
 2170   INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE | NON_MAIN_ARENA);
 2171 
 2172   if (!chunk_is_mmapped (p))
 2173     {
 2174       assert (av == arena_for_chunk (p));
 2175       if (chunk_main_arena (p))
 2176         assert (av == &main_arena);
 2177       else
 2178         assert (av != &main_arena);
 2179     }
 2180 
 2181   do_check_inuse_chunk (av, p);
 2182 
 2183   /* Legal size ... */
 2184   assert ((sz & MALLOC_ALIGN_MASK) == 0);
 2185   assert ((unsigned long) (sz) >= MINSIZE);
 2186   /* ... and alignment */
 2187   assert (aligned_OK (chunk2mem (p)));
 2188   /* chunk is less than MINSIZE more than request */
 2189   assert ((long) (sz) - (long) (s) >= 0);
 2190   assert ((long) (sz) - (long) (s + MINSIZE) < 0);
 2191 }
 2192 
 2193 /*
 2194    Properties of nonrecycled chunks at the point they are malloced
 2195  */
 2196 
 2197 static void
 2198 do_check_malloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
 2199 {
 2200   /* same as recycled case ... */
 2201   do_check_remalloced_chunk (av, p, s);
 2202 
 2203   /*
 2204      ... plus,  must obey implementation invariant that prev_inuse is
 2205      always true of any allocated chunk; i.e., that each allocated
 2206      chunk borders either a previously allocated and still in-use
 2207      chunk, or the base of its memory arena. This is ensured
 2208      by making all allocations from the `lowest' part of any found
 2209      chunk.  This does not necessarily hold however for chunks
 2210      recycled via fastbins.
 2211    */
 2212 
 2213   assert (prev_inuse (p));
 2214 }
 2215 
 2216 
 2217 /*
 2218    Properties of malloc_state.
 2219 
 2220    This may be useful for debugging malloc, as well as detecting user
 2221    programmer errors that somehow write into malloc_state.
 2222 
 2223    If you are extending or experimenting with this malloc, you can
 2224    probably figure out how to hack this routine to print out or
 2225    display chunk addresses, sizes, bins, and other instrumentation.
 2226  */
 2227 
 2228 static void
 2229 do_check_malloc_state (mstate av)
 2230 {
 2231   int i;
 2232   mchunkptr p;
 2233   mchunkptr q;
 2234   mbinptr b;
 2235   unsigned int idx;
 2236   INTERNAL_SIZE_T size;
 2237   unsigned long total = 0;
 2238   int max_fast_bin;
 2239 
 2240   /* internal size_t must be no wider than pointer type */
 2241   assert (sizeof (INTERNAL_SIZE_T) <= sizeof (char *));
 2242 
 2243   /* alignment is a power of 2 */
 2244   assert ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT - 1)) == 0);
 2245 
 2246   /* Check the arena is initialized. */
 2247   assert (av->top != 0);
 2248 
 2249   /* No memory has been allocated yet, so doing more tests is not possible.  */
 2250   if (av->top == initial_top (av))
 2251     return;
 2252 
 2253   /* pagesize is a power of 2 */
 2254   assert (powerof2(GLRO (dl_pagesize)));
 2255 
 2256   /* A contiguous main_arena is consistent with sbrk_base.  */
 2257   if (av == &main_arena && contiguous (av))
 2258     assert ((char *) mp_.sbrk_base + av->system_mem ==
 2259             (char *) av->top + chunksize (av->top));
 2260 
 2261   /* properties of fastbins */
 2262 
 2263   /* max_fast is in allowed range */
 2264   assert ((get_max_fast () & ~1) <= request2size (MAX_FAST_SIZE));
 2265 
 2266   max_fast_bin = fastbin_index (get_max_fast ());
 2267 
 2268   for (i = 0; i < NFASTBINS; ++i)
 2269     {
 2270       p = fastbin (av, i);
 2271 
 2272       /* The following test can only be performed for the main arena.
 2273          While mallopt calls malloc_consolidate to get rid of all fast
 2274          bins (especially those larger than the new maximum) this does
 2275          only happen for the main arena.  Trying to do this for any
 2276          other arena would mean those arenas have to be locked and
 2277          malloc_consolidate be called for them.  This is excessive.  And
 2278          even if this is acceptable to somebody it still cannot solve
 2279          the problem completely since if the arena is locked a
 2280          concurrent malloc call might create a new arena which then
 2281          could use the newly invalid fast bins.  */
 2282 
 2283       /* all bins past max_fast are empty */
 2284       if (av == &main_arena && i > max_fast_bin)
 2285         assert (p == 0);
 2286 
 2287       while (p != 0)
 2288         {
 2289       if (__glibc_unlikely (misaligned_chunk (p)))
 2290         malloc_printerr ("do_check_malloc_state(): "
 2291                  "unaligned fastbin chunk detected");
 2292           /* each chunk claims to be inuse */
 2293           do_check_inuse_chunk (av, p);
 2294           total += chunksize (p);
 2295           /* chunk belongs in this bin */
 2296           assert (fastbin_index (chunksize (p)) == i);
 2297       p = REVEAL_PTR (p->fd);
 2298         }
 2299     }
 2300 
 2301   /* check normal bins */
 2302   for (i = 1; i < NBINS; ++i)
 2303     {
 2304       b = bin_at (av, i);
 2305 
 2306       /* binmap is accurate (except for bin 1 == unsorted_chunks) */
 2307       if (i >= 2)
 2308         {
 2309           unsigned int binbit = get_binmap (av, i);
 2310           int empty = last (b) == b;
 2311           if (!binbit)
 2312             assert (empty);
 2313           else if (!empty)
 2314             assert (binbit);
 2315         }
 2316 
 2317       for (p = last (b); p != b; p = p->bk)
 2318         {
 2319           /* each chunk claims to be free */
 2320           do_check_free_chunk (av, p);
 2321           size = chunksize (p);
 2322           total += size;
 2323           if (i >= 2)
 2324             {
 2325               /* chunk belongs in bin */
 2326               idx = bin_index (size);
 2327               assert (idx == i);
 2328               /* lists are sorted */
 2329               assert (p->bk == b ||
 2330                       (unsigned long) chunksize (p->bk) >= (unsigned long) chunksize (p));
 2331 
 2332               if (!in_smallbin_range (size))
 2333                 {
 2334                   if (p->fd_nextsize != NULL)
 2335                     {
 2336                       if (p->fd_nextsize == p)
 2337                         assert (p->bk_nextsize == p);
 2338                       else
 2339                         {
 2340                           if (p->fd_nextsize == first (b))
 2341                             assert (chunksize (p) < chunksize (p->fd_nextsize));
 2342                           else
 2343                             assert (chunksize (p) > chunksize (p->fd_nextsize));
 2344 
 2345                           if (p == first (b))
 2346                             assert (chunksize (p) > chunksize (p->bk_nextsize));
 2347                           else
 2348                             assert (chunksize (p) < chunksize (p->bk_nextsize));
 2349                         }
 2350                     }
 2351                   else
 2352                     assert (p->bk_nextsize == NULL);
 2353                 }
 2354             }
 2355           else if (!in_smallbin_range (size))
 2356             assert (p->fd_nextsize == NULL && p->bk_nextsize == NULL);
 2357           /* chunk is followed by a legal chain of inuse chunks */
 2358           for (q = next_chunk (p);
 2359                (q != av->top && inuse (q) &&
 2360                 (unsigned long) (chunksize (q)) >= MINSIZE);
 2361                q = next_chunk (q))
 2362             do_check_inuse_chunk (av, q);
 2363         }
 2364     }
 2365 
 2366   /* top chunk is OK */
 2367   check_chunk (av, av->top);
 2368 }
 2369 #endif
 2370 
 2371 
 2372 /* ----------------- Support for debugging hooks -------------------- */
 2373 #if IS_IN (libc)
 2374 #include "hooks.c"
 2375 #endif
 2376 
 2377 
 2378 /* ----------- Routines dealing with system allocation -------------- */
 2379 
 2380 /*
 2381    sysmalloc handles malloc cases requiring more memory from the system.
 2382    On entry, it is assumed that av->top does not have enough
 2383    space to service request for nb bytes, thus requiring that av->top
 2384    be extended or replaced.
 2385  */
 2386 
 2387 static void *
 2388 sysmalloc (INTERNAL_SIZE_T nb, mstate av)
 2389 {
 2390   mchunkptr old_top;              /* incoming value of av->top */
 2391   INTERNAL_SIZE_T old_size;       /* its size */
 2392   char *old_end;                  /* its end address */
 2393 
 2394   long size;                      /* arg to first MORECORE or mmap call */
 2395   char *brk;                      /* return value from MORECORE */
 2396 
 2397   long correction;                /* arg to 2nd MORECORE call */
 2398   char *snd_brk;                  /* 2nd return val */
 2399 
 2400   INTERNAL_SIZE_T front_misalign; /* unusable bytes at front of new space */
 2401   INTERNAL_SIZE_T end_misalign;   /* partial page left at end of new space */
 2402   char *aligned_brk;              /* aligned offset into brk */
 2403 
 2404   mchunkptr p;                    /* the allocated/returned chunk */
 2405   mchunkptr remainder;            /* remainder from allocation */
 2406   unsigned long remainder_size;   /* its size */
 2407 
 2408 
 2409   size_t pagesize = GLRO (dl_pagesize);
 2410   bool tried_mmap = false;
 2411 
 2412 
 2413   /*
 2414      If have mmap, and the request size meets the mmap threshold, and
 2415      the system supports mmap, and there are few enough currently
 2416      allocated mmapped regions, try to directly map this request
 2417      rather than expanding top.
 2418    */
 2419 
 2420   if (av == NULL
 2421       || ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold)
 2422       && (mp_.n_mmaps < mp_.n_mmaps_max)))
 2423     {
 2424       char *mm;           /* return value from mmap call*/
 2425 
 2426     try_mmap:
 2427       /*
 2428          Round up size to nearest page.  For mmapped chunks, the overhead
 2429          is one SIZE_SZ unit larger than for normal chunks, because there
 2430          is no following chunk whose prev_size field could be used.
 2431 
 2432          See the front_misalign handling below, for glibc there is no
 2433          need for further alignments unless we have have high alignment.
 2434        */
 2435       if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
 2436         size = ALIGN_UP (nb + SIZE_SZ, pagesize);
 2437       else
 2438         size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize);
 2439       tried_mmap = true;
 2440 
 2441       /* Don't try if size wraps around 0 */
 2442       if ((unsigned long) (size) > (unsigned long) (nb))
 2443         {
 2444           mm = (char *) (MMAP (0, size,
 2445                    mtag_mmap_flags | PROT_READ | PROT_WRITE, 0));
 2446 
 2447           if (mm != MAP_FAILED)
 2448             {
 2449               /*
 2450                  The offset to the start of the mmapped region is stored
 2451                  in the prev_size field of the chunk. This allows us to adjust
 2452                  returned start address to meet alignment requirements here
 2453                  and in memalign(), and still be able to compute proper
 2454                  address argument for later munmap in free() and realloc().
 2455                */
 2456 
 2457               if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
 2458                 {
 2459                   /* For glibc, chunk2mem increases the address by
 2460                      CHUNK_HDR_SZ and MALLOC_ALIGN_MASK is
 2461                      CHUNK_HDR_SZ-1.  Each mmap'ed area is page
 2462                      aligned and therefore definitely
 2463                      MALLOC_ALIGN_MASK-aligned.  */
 2464                   assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == 0);
 2465                   front_misalign = 0;
 2466                 }
 2467               else
 2468                 front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK;
 2469               if (front_misalign > 0)
 2470                 {
 2471                   correction = MALLOC_ALIGNMENT - front_misalign;
 2472                   p = (mchunkptr) (mm + correction);
 2473           set_prev_size (p, correction);
 2474                   set_head (p, (size - correction) | IS_MMAPPED);
 2475                 }
 2476               else
 2477                 {
 2478                   p = (mchunkptr) mm;
 2479           set_prev_size (p, 0);
 2480                   set_head (p, size | IS_MMAPPED);
 2481                 }
 2482 
 2483               /* update statistics */
 2484 
 2485               int new = atomic_exchange_and_add (&mp_.n_mmaps, 1) + 1;
 2486               atomic_max (&mp_.max_n_mmaps, new);
 2487 
 2488               unsigned long sum;
 2489               sum = atomic_exchange_and_add (&mp_.mmapped_mem, size) + size;
 2490               atomic_max (&mp_.max_mmapped_mem, sum);
 2491 
 2492               check_chunk (av, p);
 2493 
 2494               return chunk2mem (p);
 2495             }
 2496         }
 2497     }
 2498 
 2499   /* There are no usable arenas and mmap also failed.  */
 2500   if (av == NULL)
 2501     return 0;
 2502 
 2503   /* Record incoming configuration of top */
 2504 
 2505   old_top = av->top;
 2506   old_size = chunksize (old_top);
 2507   old_end = (char *) (chunk_at_offset (old_top, old_size));
 2508 
 2509   brk = snd_brk = (char *) (MORECORE_FAILURE);
 2510 
 2511   /*
 2512      If not the first time through, we require old_size to be
 2513      at least MINSIZE and to have prev_inuse set.
 2514    */
 2515 
 2516   assert ((old_top == initial_top (av) && old_size == 0) ||
 2517           ((unsigned long) (old_size) >= MINSIZE &&
 2518            prev_inuse (old_top) &&
 2519            ((unsigned long) old_end & (pagesize - 1)) == 0));
 2520 
 2521   /* Precondition: not enough current space to satisfy nb request */
 2522   assert ((unsigned long) (old_size) < (unsigned long) (nb + MINSIZE));
 2523 
 2524 
 2525   if (av != &main_arena)
 2526     {
 2527       heap_info *old_heap, *heap;
 2528       size_t old_heap_size;
 2529 
 2530       /* First try to extend the current heap. */
 2531       old_heap = heap_for_ptr (old_top);
 2532       old_heap_size = old_heap->size;
 2533       if ((long) (MINSIZE + nb - old_size) > 0
 2534           && grow_heap (old_heap, MINSIZE + nb - old_size) == 0)
 2535         {
 2536           av->system_mem += old_heap->size - old_heap_size;
 2537           set_head (old_top, (((char *) old_heap + old_heap->size) - (char *) old_top)
 2538                     | PREV_INUSE);
 2539         }
 2540       else if ((heap = new_heap (nb + (MINSIZE + sizeof (*heap)), mp_.top_pad)))
 2541         {
 2542           /* Use a newly allocated heap.  */
 2543           heap->ar_ptr = av;
 2544           heap->prev = old_heap;
 2545           av->system_mem += heap->size;
 2546           /* Set up the new top.  */
 2547           top (av) = chunk_at_offset (heap, sizeof (*heap));
 2548           set_head (top (av), (heap->size - sizeof (*heap)) | PREV_INUSE);
 2549 
 2550           /* Setup fencepost and free the old top chunk with a multiple of
 2551              MALLOC_ALIGNMENT in size. */
 2552           /* The fencepost takes at least MINSIZE bytes, because it might
 2553              become the top chunk again later.  Note that a footer is set
 2554              up, too, although the chunk is marked in use. */
 2555           old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK;
 2556           set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ),
 2557             0 | PREV_INUSE);
 2558           if (old_size >= MINSIZE)
 2559             {
 2560               set_head (chunk_at_offset (old_top, old_size),
 2561             CHUNK_HDR_SZ | PREV_INUSE);
 2562               set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ);
 2563               set_head (old_top, old_size | PREV_INUSE | NON_MAIN_ARENA);
 2564               _int_free (av, old_top, 1);
 2565             }
 2566           else
 2567             {
 2568               set_head (old_top, (old_size + CHUNK_HDR_SZ) | PREV_INUSE);
 2569               set_foot (old_top, (old_size + CHUNK_HDR_SZ));
 2570             }
 2571         }
 2572       else if (!tried_mmap)
 2573         /* We can at least try to use to mmap memory.  */
 2574         goto try_mmap;
 2575     }
 2576   else     /* av == main_arena */
 2577 
 2578 
 2579     { /* Request enough space for nb + pad + overhead */
 2580       size = nb + mp_.top_pad + MINSIZE;
 2581 
 2582       /*
 2583          If contiguous, we can subtract out existing space that we hope to
 2584          combine with new space. We add it back later only if
 2585          we don't actually get contiguous space.
 2586        */
 2587 
 2588       if (contiguous (av))
 2589         size -= old_size;
 2590 
 2591       /*
 2592          Round to a multiple of page size.
 2593          If MORECORE is not contiguous, this ensures that we only call it
 2594          with whole-page arguments.  And if MORECORE is contiguous and
 2595          this is not first time through, this preserves page-alignment of
 2596          previous calls. Otherwise, we correct to page-align below.
 2597        */
 2598 
 2599       size = ALIGN_UP (size, pagesize);
 2600 
 2601       /*
 2602          Don't try to call MORECORE if argument is so big as to appear
 2603          negative. Note that since mmap takes size_t arg, it may succeed
 2604          below even if we cannot call MORECORE.
 2605        */
 2606 
 2607       if (size > 0)
 2608         {
 2609           brk = (char *) (MORECORE (size));
 2610           LIBC_PROBE (memory_sbrk_more, 2, brk, size);
 2611         }
 2612 
 2613       if (brk == (char *) (MORECORE_FAILURE))
 2614         {
 2615           /*
 2616              If have mmap, try using it as a backup when MORECORE fails or
 2617              cannot be used. This is worth doing on systems that have "holes" in
 2618              address space, so sbrk cannot extend to give contiguous space, but
 2619              space is available elsewhere.  Note that we ignore mmap max count
 2620              and threshold limits, since the space will not be used as a
 2621              segregated mmap region.
 2622            */
 2623 
 2624           /* Cannot merge with old top, so add its size back in */
 2625           if (contiguous (av))
 2626             size = ALIGN_UP (size + old_size, pagesize);
 2627 
 2628           /* If we are relying on mmap as backup, then use larger units */
 2629           if ((unsigned long) (size) < (unsigned long) (MMAP_AS_MORECORE_SIZE))
 2630             size = MMAP_AS_MORECORE_SIZE;
 2631 
 2632           /* Don't try if size wraps around 0 */
 2633           if ((unsigned long) (size) > (unsigned long) (nb))
 2634             {
 2635               char *mbrk = (char *) (MMAP (0, size,
 2636                        mtag_mmap_flags | PROT_READ | PROT_WRITE,
 2637                        0));
 2638 
 2639               if (mbrk != MAP_FAILED)
 2640                 {
 2641                   /* We do not need, and cannot use, another sbrk call to find end */
 2642                   brk = mbrk;
 2643                   snd_brk = brk + size;
 2644 
 2645                   /*
 2646                      Record that we no longer have a contiguous sbrk region.
 2647                      After the first time mmap is used as backup, we do not
 2648                      ever rely on contiguous space since this could incorrectly
 2649                      bridge regions.
 2650                    */
 2651                   set_noncontiguous (av);
 2652                 }
 2653             }
 2654         }
 2655 
 2656       if (brk != (char *) (MORECORE_FAILURE))
 2657         {
 2658           if (mp_.sbrk_base == 0)
 2659             mp_.sbrk_base = brk;
 2660           av->system_mem += size;
 2661 
 2662           /*
 2663              If MORECORE extends previous space, we can likewise extend top size.
 2664            */
 2665 
 2666           if (brk == old_end && snd_brk == (char *) (MORECORE_FAILURE))
 2667             set_head (old_top, (size + old_size) | PREV_INUSE);
 2668 
 2669           else if (contiguous (av) && old_size && brk < old_end)
 2670         /* Oops!  Someone else killed our space..  Can't touch anything.  */
 2671         malloc_printerr ("break adjusted to free malloc space");
 2672 
 2673           /*
 2674              Otherwise, make adjustments:
 2675 
 2676            * If the first time through or noncontiguous, we need to call sbrk
 2677               just to find out where the end of memory lies.
 2678 
 2679            * We need to ensure that all returned chunks from malloc will meet
 2680               MALLOC_ALIGNMENT
 2681 
 2682            * If there was an intervening foreign sbrk, we need to adjust sbrk
 2683               request size to account for fact that we will not be able to
 2684               combine new space with existing space in old_top.
 2685 
 2686            * Almost all systems internally allocate whole pages at a time, in
 2687               which case we might as well use the whole last page of request.
 2688               So we allocate enough more memory to hit a page boundary now,
 2689               which in turn causes future contiguous calls to page-align.
 2690            */
 2691 
 2692           else
 2693             {
 2694               front_misalign = 0;
 2695               end_misalign = 0;
 2696               correction = 0;
 2697               aligned_brk = brk;
 2698 
 2699               /* handle contiguous cases */
 2700               if (contiguous (av))
 2701                 {
 2702                   /* Count foreign sbrk as system_mem.  */
 2703                   if (old_size)
 2704                     av->system_mem += brk - old_end;
 2705 
 2706                   /* Guarantee alignment of first new chunk made from this space */
 2707 
 2708                   front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
 2709                   if (front_misalign > 0)
 2710                     {
 2711                       /*
 2712                          Skip over some bytes to arrive at an aligned position.
 2713                          We don't need to specially mark these wasted front bytes.
 2714                          They will never be accessed anyway because
 2715                          prev_inuse of av->top (and any chunk created from its start)
 2716                          is always true after initialization.
 2717                        */
 2718 
 2719                       correction = MALLOC_ALIGNMENT - front_misalign;
 2720                       aligned_brk += correction;
 2721                     }
 2722 
 2723                   /*
 2724                      If this isn't adjacent to existing space, then we will not
 2725                      be able to merge with old_top space, so must add to 2nd request.
 2726                    */
 2727 
 2728                   correction += old_size;
 2729 
 2730                   /* Extend the end address to hit a page boundary */
 2731                   end_misalign = (INTERNAL_SIZE_T) (brk + size + correction);
 2732                   correction += (ALIGN_UP (end_misalign, pagesize)) - end_misalign;
 2733 
 2734                   assert (correction >= 0);
 2735                   snd_brk = (char *) (MORECORE (correction));
 2736 
 2737                   /*
 2738                      If can't allocate correction, try to at least find out current
 2739                      brk.  It might be enough to proceed without failing.
 2740 
 2741                      Note that if second sbrk did NOT fail, we assume that space
 2742                      is contiguous with first sbrk. This is a safe assumption unless
 2743                      program is multithreaded but doesn't use locks and a foreign sbrk
 2744                      occurred between our first and second calls.
 2745                    */
 2746 
 2747                   if (snd_brk == (char *) (MORECORE_FAILURE))
 2748                     {
 2749                       correction = 0;
 2750                       snd_brk = (char *) (MORECORE (0));
 2751                     }
 2752                 }
 2753 
 2754               /* handle non-contiguous cases */
 2755               else
 2756                 {
 2757                   if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
 2758                     /* MORECORE/mmap must correctly align */
 2759                     assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == 0);
 2760                   else
 2761                     {
 2762                       front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
 2763                       if (front_misalign > 0)
 2764                         {
 2765                           /*
 2766                              Skip over some bytes to arrive at an aligned position.
 2767                              We don't need to specially mark these wasted front bytes.
 2768                              They will never be accessed anyway because
 2769                              prev_inuse of av->top (and any chunk created from its start)
 2770                              is always true after initialization.
 2771                            */
 2772 
 2773                           aligned_brk += MALLOC_ALIGNMENT - front_misalign;
 2774                         }
 2775                     }
 2776 
 2777                   /* Find out current end of memory */
 2778                   if (snd_brk == (char *) (MORECORE_FAILURE))
 2779                     {
 2780                       snd_brk = (char *) (MORECORE (0));
 2781                     }
 2782                 }
 2783 
 2784               /* Adjust top based on results of second sbrk */
 2785               if (snd_brk != (char *) (MORECORE_FAILURE))
 2786                 {
 2787                   av->top = (mchunkptr) aligned_brk;
 2788                   set_head (av->top, (snd_brk - aligned_brk + correction) | PREV_INUSE);
 2789                   av->system_mem += correction;
 2790 
 2791                   /*
 2792                      If not the first time through, we either have a
 2793                      gap due to foreign sbrk or a non-contiguous region.  Insert a
 2794                      double fencepost at old_top to prevent consolidation with space
 2795                      we don't own. These fenceposts are artificial chunks that are
 2796                      marked as inuse and are in any case too small to use.  We need
 2797                      two to make sizes and alignments work out.
 2798                    */
 2799 
 2800                   if (old_size != 0)
 2801                     {
 2802                       /*
 2803                          Shrink old_top to insert fenceposts, keeping size a
 2804                          multiple of MALLOC_ALIGNMENT. We know there is at least
 2805                          enough space in old_top to do this.
 2806                        */
 2807                       old_size = (old_size - 2 * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK;
 2808                       set_head (old_top, old_size | PREV_INUSE);
 2809 
 2810                       /*
 2811                          Note that the following assignments completely overwrite
 2812                          old_top when old_size was previously MINSIZE.  This is
 2813                          intentional. We need the fencepost, even if old_top otherwise gets
 2814                          lost.
 2815                        */
 2816               set_head (chunk_at_offset (old_top, old_size),
 2817                 CHUNK_HDR_SZ | PREV_INUSE);
 2818               set_head (chunk_at_offset (old_top,
 2819                          old_size + CHUNK_HDR_SZ),
 2820                 CHUNK_HDR_SZ | PREV_INUSE);
 2821 
 2822                       /* If possible, release the rest. */
 2823                       if (old_size >= MINSIZE)
 2824                         {
 2825                           _int_free (av, old_top, 1);
 2826                         }
 2827                     }
 2828                 }
 2829             }
 2830         }
 2831     } /* if (av !=  &main_arena) */
 2832 
 2833   if ((unsigned long) av->system_mem > (unsigned long) (av->max_system_mem))
 2834     av->max_system_mem = av->system_mem;
 2835   check_malloc_state (av);
 2836 
 2837   /* finally, do the allocation */
 2838   p = av->top;
 2839   size = chunksize (p);
 2840 
 2841   /* check that one of the above allocation paths succeeded */
 2842   if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
 2843     {
 2844       remainder_size = size - nb;
 2845       remainder = chunk_at_offset (p, nb);
 2846       av->top = remainder;
 2847       set_head (p, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0));
 2848       set_head (remainder, remainder_size | PREV_INUSE);
 2849       check_malloced_chunk (av, p, nb);
 2850       return chunk2mem (p);
 2851     }
 2852 
 2853   /* catch all failure paths */
 2854   __set_errno (ENOMEM);
 2855   return 0;
 2856 }
 2857 
 2858 
 2859 /*
 2860    systrim is an inverse of sorts to sysmalloc.  It gives memory back
 2861    to the system (via negative arguments to sbrk) if there is unused
 2862    memory at the `high' end of the malloc pool. It is called
 2863    automatically by free() when top space exceeds the trim
 2864    threshold. It is also called by the public malloc_trim routine.  It
 2865    returns 1 if it actually released any memory, else 0.
 2866  */
 2867 
 2868 static int
 2869 systrim (size_t pad, mstate av)
 2870 {
 2871   long top_size;         /* Amount of top-most memory */
 2872   long extra;            /* Amount to release */
 2873   long released;         /* Amount actually released */
 2874   char *current_brk;     /* address returned by pre-check sbrk call */
 2875   char *new_brk;         /* address returned by post-check sbrk call */
 2876   size_t pagesize;
 2877   long top_area;
 2878 
 2879   pagesize = GLRO (dl_pagesize);
 2880   top_size = chunksize (av->top);
 2881 
 2882   top_area = top_size - MINSIZE - 1;
 2883   if (top_area <= pad)
 2884     return 0;
 2885 
 2886   /* Release in pagesize units and round down to the nearest page.  */
 2887   extra = ALIGN_DOWN(top_area - pad, pagesize);
 2888 
 2889   if (extra == 0)
 2890     return 0;
 2891 
 2892   /*
 2893      Only proceed if end of memory is where we last set it.
 2894      This avoids problems if there were foreign sbrk calls.
 2895    */
 2896   current_brk = (char *) (MORECORE (0));
 2897   if (current_brk == (char *) (av->top) + top_size)
 2898     {
 2899       /*
 2900          Attempt to release memory. We ignore MORECORE return value,
 2901          and instead call again to find out where new end of memory is.
 2902          This avoids problems if first call releases less than we asked,
 2903          of if failure somehow altered brk value. (We could still
 2904          encounter problems if it altered brk in some very bad way,
 2905          but the only thing we can do is adjust anyway, which will cause
 2906          some downstream failure.)
 2907        */
 2908 
 2909       MORECORE (-extra);
 2910       new_brk = (char *) (MORECORE (0));
 2911 
 2912       LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra);
 2913 
 2914       if (new_brk != (char *) MORECORE_FAILURE)
 2915         {
 2916           released = (long) (current_brk - new_brk);
 2917 
 2918           if (released != 0)
 2919             {
 2920               /* Success. Adjust top. */
 2921               av->system_mem -= released;
 2922               set_head (av->top, (top_size - released) | PREV_INUSE);
 2923               check_malloc_state (av);
 2924               return 1;
 2925             }
 2926         }
 2927     }
 2928   return 0;
 2929 }
 2930 
 2931 static void
 2932 munmap_chunk (mchunkptr p)
 2933 {
 2934   size_t pagesize = GLRO (dl_pagesize);
 2935   INTERNAL_SIZE_T size = chunksize (p);
 2936 
 2937   assert (chunk_is_mmapped (p));
 2938 
 2939   uintptr_t mem = (uintptr_t) chunk2mem (p);
 2940   uintptr_t block = (uintptr_t) p - prev_size (p);
 2941   size_t total_size = prev_size (p) + size;
 2942   /* Unfortunately we have to do the compilers job by hand here.  Normally
 2943      we would test BLOCK and TOTAL-SIZE separately for compliance with the
 2944      page size.  But gcc does not recognize the optimization possibility
 2945      (in the moment at least) so we combine the two values into one before
 2946      the bit test.  */
 2947   if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
 2948       || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
 2949     malloc_printerr ("munmap_chunk(): invalid pointer");
 2950 
 2951   atomic_decrement (&mp_.n_mmaps);
 2952   atomic_add (&mp_.mmapped_mem, -total_size);
 2953 
 2954   /* If munmap failed the process virtual memory address space is in a
 2955      bad shape.  Just leave the block hanging around, the process will
 2956      terminate shortly anyway since not much can be done.  */
 2957   __munmap ((char *) block, total_size);
 2958 }
 2959 
 2960 #if HAVE_MREMAP
 2961 
 2962 static mchunkptr
 2963 mremap_chunk (mchunkptr p, size_t new_size)
 2964 {
 2965   size_t pagesize = GLRO (dl_pagesize);
 2966   INTERNAL_SIZE_T offset = prev_size (p);
 2967   INTERNAL_SIZE_T size = chunksize (p);
 2968   char *cp;
 2969 
 2970   assert (chunk_is_mmapped (p));
 2971 
 2972   uintptr_t block = (uintptr_t) p - offset;
 2973   uintptr_t mem = (uintptr_t) chunk2mem(p);
 2974   size_t total_size = offset + size;
 2975   if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
 2976       || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
 2977     malloc_printerr("mremap_chunk(): invalid pointer");
 2978 
 2979   /* Note the extra SIZE_SZ overhead as in mmap_chunk(). */
 2980   new_size = ALIGN_UP (new_size + offset + SIZE_SZ, pagesize);
 2981 
 2982   /* No need to remap if the number of pages does not change.  */
 2983   if (total_size == new_size)
 2984     return p;
 2985 
 2986   cp = (char *) __mremap ((char *) block, total_size, new_size,
 2987                           MREMAP_MAYMOVE);
 2988 
 2989   if (cp == MAP_FAILED)
 2990     return 0;
 2991 
 2992   p = (mchunkptr) (cp + offset);
 2993 
 2994   assert (aligned_OK (chunk2mem (p)));
 2995 
 2996   assert (prev_size (p) == offset);
 2997   set_head (p, (new_size - offset) | IS_MMAPPED);
 2998 
 2999   INTERNAL_SIZE_T new;
 3000   new = atomic_exchange_and_add (&mp_.mmapped_mem, new_size - size - offset)
 3001         + new_size - size - offset;
 3002   atomic_max (&mp_.max_mmapped_mem, new);
 3003   return p;
 3004 }
 3005 #endif /* HAVE_MREMAP */
 3006 
 3007 /*------------------------ Public wrappers. --------------------------------*/
 3008 
 3009 #if USE_TCACHE
 3010 
 3011 /* We overlay this structure on the user-data portion of a chunk when
 3012    the chunk is stored in the per-thread cache.  */
 3013 typedef struct tcache_entry
 3014 {
 3015   struct tcache_entry *next;
 3016   /* This field exists to detect double frees.  */
 3017   uintptr_t key;
 3018 } tcache_entry;
 3019 
 3020 /* There is one of these for each thread, which contains the
 3021    per-thread cache (hence "tcache_perthread_struct").  Keeping
 3022    overall size low is mildly important.  Note that COUNTS and ENTRIES
 3023    are redundant (we could have just counted the linked list each
 3024    time), this is for performance reasons.  */
 3025 typedef struct tcache_perthread_struct
 3026 {
 3027   uint16_t counts[TCACHE_MAX_BINS];
 3028   tcache_entry *entries[TCACHE_MAX_BINS];
 3029 } tcache_perthread_struct;
 3030 
 3031 static __thread bool tcache_shutting_down = false;
 3032 static __thread tcache_perthread_struct *tcache = NULL;
 3033 
 3034 /* Process-wide key to try and catch a double-free in the same thread.  */
 3035 static uintptr_t tcache_key;
 3036 
 3037 /* The value of tcache_key does not really have to be a cryptographically
 3038    secure random number.  It only needs to be arbitrary enough so that it does
 3039    not collide with values present in applications.  If a collision does happen
 3040    consistently enough, it could cause a degradation in performance since the
 3041    entire list is checked to check if the block indeed has been freed the
 3042    second time.  The odds of this happening are exceedingly low though, about 1
 3043    in 2^wordsize.  There is probably a higher chance of the performance
 3044    degradation being due to a double free where the first free happened in a
 3045    different thread; that's a case this check does not cover.  */
 3046 static void
 3047 tcache_key_initialize (void)
 3048 {
 3049   if (__getrandom (&tcache_key, sizeof(tcache_key), GRND_NONBLOCK)
 3050       != sizeof (tcache_key))
 3051     {
 3052       tcache_key = random_bits ();
 3053 #if __WORDSIZE == 64
 3054       tcache_key = (tcache_key << 32) | random_bits ();
 3055 #endif
 3056     }
 3057 }
 3058 
 3059 /* Caller must ensure that we know tc_idx is valid and there's room
 3060    for more chunks.  */
 3061 static __always_inline void
 3062 tcache_put (mchunkptr chunk, size_t tc_idx)
 3063 {
 3064   tcache_entry *e = (tcache_entry *) chunk2mem (chunk);
 3065 
 3066   /* Mark this chunk as "in the tcache" so the test in _int_free will
 3067      detect a double free.  */
 3068   e->key = tcache_key;
 3069 
 3070   e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]);
 3071   tcache->entries[tc_idx] = e;
 3072   ++(tcache->counts[tc_idx]);
 3073 }
 3074 
 3075 /* Caller must ensure that we know tc_idx is valid and there's
 3076    available chunks to remove.  */
 3077 static __always_inline void *
 3078 tcache_get (size_t tc_idx)
 3079 {
 3080   tcache_entry *e = tcache->entries[tc_idx];
 3081   if (__glibc_unlikely (!aligned_OK (e)))
 3082     malloc_printerr ("malloc(): unaligned tcache chunk detected");
 3083   tcache->entries[tc_idx] = REVEAL_PTR (e->next);
 3084   --(tcache->counts[tc_idx]);
 3085   e->key = 0;
 3086   return (void *) e;
 3087 }
 3088 
 3089 static void
 3090 tcache_thread_shutdown (void)
 3091 {
 3092   int i;
 3093   tcache_perthread_struct *tcache_tmp = tcache;
 3094 
 3095   tcache_shutting_down = true;
 3096 
 3097   if (!tcache)
 3098     return;
 3099 
 3100   /* Disable the tcache and prevent it from being reinitialized.  */
 3101   tcache = NULL;
 3102 
 3103   /* Free all of the entries and the tcache itself back to the arena
 3104      heap for coalescing.  */
 3105   for (i = 0; i < TCACHE_MAX_BINS; ++i)
 3106     {
 3107       while (tcache_tmp->entries[i])
 3108     {
 3109       tcache_entry *e = tcache_tmp->entries[i];
 3110       if (__glibc_unlikely (!aligned_OK (e)))
 3111         malloc_printerr ("tcache_thread_shutdown(): "
 3112                  "unaligned tcache chunk detected");
 3113       tcache_tmp->entries[i] = REVEAL_PTR (e->next);
 3114       __libc_free (e);
 3115     }
 3116     }
 3117 
 3118   __libc_free (tcache_tmp);
 3119 }
 3120 
 3121 static void
 3122 tcache_init(void)
 3123 {
 3124   mstate ar_ptr;
 3125   void *victim = 0;
 3126   const size_t bytes = sizeof (tcache_perthread_struct);
 3127 
 3128   if (tcache_shutting_down)
 3129     return;
 3130 
 3131   arena_get (ar_ptr, bytes);
 3132   victim = _int_malloc (ar_ptr, bytes);
 3133   if (!victim && ar_ptr != NULL)
 3134     {
 3135       ar_ptr = arena_get_retry (ar_ptr, bytes);
 3136       victim = _int_malloc (ar_ptr, bytes);
 3137     }
 3138 
 3139 
 3140   if (ar_ptr != NULL)
 3141     __libc_lock_unlock (ar_ptr->mutex);
 3142 
 3143   /* In a low memory situation, we may not be able to allocate memory
 3144      - in which case, we just keep trying later.  However, we
 3145      typically do this very early, so either there is sufficient
 3146      memory, or there isn't enough memory to do non-trivial
 3147      allocations anyway.  */
 3148   if (victim)
 3149     {
 3150       tcache = (tcache_perthread_struct *) victim;
 3151       memset (tcache, 0, sizeof (tcache_perthread_struct));
 3152     }
 3153 
 3154 }
 3155 
 3156 # define MAYBE_INIT_TCACHE() \
 3157   if (__glibc_unlikely (tcache == NULL)) \
 3158     tcache_init();
 3159 
 3160 #else  /* !USE_TCACHE */
 3161 # define MAYBE_INIT_TCACHE()
 3162 
 3163 static void
 3164 tcache_thread_shutdown (void)
 3165 {
 3166   /* Nothing to do if there is no thread cache.  */
 3167 }
 3168 
 3169 #endif /* !USE_TCACHE  */
 3170 
 3171 #if IS_IN (libc)
 3172 void *
 3173 __libc_malloc (size_t bytes)
 3174 {
 3175   mstate ar_ptr;
 3176   void *victim;
 3177 
 3178   _Static_assert (PTRDIFF_MAX <= SIZE_MAX / 2,
 3179                   "PTRDIFF_MAX is not more than half of SIZE_MAX");
 3180 
 3181   if (!__malloc_initialized)
 3182     ptmalloc_init ();
 3183 #if USE_TCACHE
 3184   /* int_free also calls request2size, be careful to not pad twice.  */
 3185   size_t tbytes;
 3186   if (!checked_request2size (bytes, &tbytes))
 3187     {
 3188       __set_errno (ENOMEM);
 3189       return NULL;
 3190     }
 3191   size_t tc_idx = csize2tidx (tbytes);
 3192 
 3193   MAYBE_INIT_TCACHE ();
 3194 
 3195   DIAG_PUSH_NEEDS_COMMENT;
 3196   if (tc_idx < mp_.tcache_bins
 3197       && tcache
 3198       && tcache->counts[tc_idx] > 0)
 3199     {
 3200       victim = tcache_get (tc_idx);
 3201       return tag_new_usable (victim);
 3202     }
 3203   DIAG_POP_NEEDS_COMMENT;
 3204 #endif
 3205 
 3206   if (SINGLE_THREAD_P)
 3207     {
 3208       victim = tag_new_usable (_int_malloc (&main_arena, bytes));
 3209       assert (!victim || chunk_is_mmapped (mem2chunk (victim)) ||
 3210           &main_arena == arena_for_chunk (mem2chunk (victim)));
 3211       return victim;
 3212     }
 3213 
 3214   arena_get (ar_ptr, bytes);
 3215 
 3216   victim = _int_malloc (ar_ptr, bytes);
 3217   /* Retry with another arena only if we were able to find a usable arena
 3218      before.  */
 3219   if (!victim && ar_ptr != NULL)
 3220     {
 3221       LIBC_PROBE (memory_malloc_retry, 1, bytes);
 3222       ar_ptr = arena_get_retry (ar_ptr, bytes);
 3223       victim = _int_malloc (ar_ptr, bytes);
 3224     }
 3225 
 3226   if (ar_ptr != NULL)
 3227     __libc_lock_unlock (ar_ptr->mutex);
 3228 
 3229   victim = tag_new_usable (victim);
 3230 
 3231   assert (!victim || chunk_is_mmapped (mem2chunk (victim)) ||
 3232           ar_ptr == arena_for_chunk (mem2chunk (victim)));
 3233   return victim;
 3234 }
 3235 libc_hidden_def (__libc_malloc)
 3236 
 3237 void
 3238 __libc_free (void *mem)
 3239 {
 3240   mstate ar_ptr;
 3241   mchunkptr p;                          /* chunk corresponding to mem */
 3242 
 3243   if (mem == 0)                              /* free(0) has no effect */
 3244     return;
 3245 
 3246   /* Quickly check that the freed pointer matches the tag for the memory.
 3247      This gives a useful double-free detection.  */
 3248   if (__glibc_unlikely (mtag_enabled))
 3249     *(volatile char *)mem;
 3250 
 3251   int err = errno;
 3252 
 3253   p = mem2chunk (mem);
 3254 
 3255   if (chunk_is_mmapped (p))                       /* release mmapped memory. */
 3256     {
 3257       /* See if the dynamic brk/mmap threshold needs adjusting.
 3258      Dumped fake mmapped chunks do not affect the threshold.  */
 3259       if (!mp_.no_dyn_threshold
 3260           && chunksize_nomask (p) > mp_.mmap_threshold
 3261           && chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
 3262         {
 3263           mp_.mmap_threshold = chunksize (p);
 3264           mp_.trim_threshold = 2 * mp_.mmap_threshold;
 3265           LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2,
 3266                       mp_.mmap_threshold, mp_.trim_threshold);
 3267         }
 3268       munmap_chunk (p);
 3269     }
 3270   else
 3271     {
 3272       MAYBE_INIT_TCACHE ();
 3273 
 3274       /* Mark the chunk as belonging to the library again.  */
 3275       (void)tag_region (chunk2mem (p), memsize (p));
 3276 
 3277       ar_ptr = arena_for_chunk (p);
 3278       _int_free (ar_ptr, p, 0);
 3279     }
 3280 
 3281   __set_errno (err);
 3282 }
 3283 libc_hidden_def (__libc_free)
 3284 
 3285 void *
 3286 __libc_realloc (void *oldmem, size_t bytes)
 3287 {
 3288   mstate ar_ptr;
 3289   INTERNAL_SIZE_T nb;         /* padded request size */
 3290 
 3291   void *newp;             /* chunk to return */
 3292 
 3293   if (!__malloc_initialized)
 3294     ptmalloc_init ();
 3295 
 3296 #if REALLOC_ZERO_BYTES_FREES
 3297   if (bytes == 0 && oldmem != NULL)
 3298     {
 3299       __libc_free (oldmem); return 0;
 3300     }
 3301 #endif
 3302 
 3303   /* realloc of null is supposed to be same as malloc */
 3304   if (oldmem == 0)
 3305     return __libc_malloc (bytes);
 3306 
 3307   /* Perform a quick check to ensure that the pointer's tag matches the
 3308      memory's tag.  */
 3309   if (__glibc_unlikely (mtag_enabled))
 3310     *(volatile char*) oldmem;
 3311 
 3312   /* chunk corresponding to oldmem */
 3313   const mchunkptr oldp = mem2chunk (oldmem);
 3314   /* its size */
 3315   const INTERNAL_SIZE_T oldsize = chunksize (oldp);
 3316 
 3317   if (chunk_is_mmapped (oldp))
 3318     ar_ptr = NULL;
 3319   else
 3320     {
 3321       MAYBE_INIT_TCACHE ();
 3322       ar_ptr = arena_for_chunk (oldp);
 3323     }
 3324 
 3325   /* Little security check which won't hurt performance: the allocator
 3326      never wrapps around at the end of the address space.  Therefore
 3327      we can exclude some size values which might appear here by
 3328      accident or by "design" from some intruder.  */
 3329   if ((__builtin_expect ((uintptr_t) oldp > (uintptr_t) -oldsize, 0)
 3330        || __builtin_expect (misaligned_chunk (oldp), 0)))
 3331       malloc_printerr ("realloc(): invalid pointer");
 3332 
 3333   if (!checked_request2size (bytes, &nb))
 3334     {
 3335       __set_errno (ENOMEM);
 3336       return NULL;
 3337     }
 3338 
 3339   if (chunk_is_mmapped (oldp))
 3340     {
 3341       void *newmem;
 3342 
 3343 #if HAVE_MREMAP
 3344       newp = mremap_chunk (oldp, nb);
 3345       if (newp)
 3346     {
 3347       void *newmem = chunk2mem_tag (newp);
 3348       /* Give the new block a different tag.  This helps to ensure
 3349          that stale handles to the previous mapping are not
 3350          reused.  There's a performance hit for both us and the
 3351          caller for doing this, so we might want to
 3352          reconsider.  */
 3353       return tag_new_usable (newmem);
 3354     }
 3355 #endif
 3356       /* Note the extra SIZE_SZ overhead. */
 3357       if (oldsize - SIZE_SZ >= nb)
 3358         return oldmem;                         /* do nothing */
 3359 
 3360       /* Must alloc, copy, free. */
 3361       newmem = __libc_malloc (bytes);
 3362       if (newmem == 0)
 3363         return 0;              /* propagate failure */
 3364 
 3365       memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ);
 3366       munmap_chunk (oldp);
 3367       return newmem;
 3368     }
 3369 
 3370   if (SINGLE_THREAD_P)
 3371     {
 3372       newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
 3373       assert (!newp || chunk_is_mmapped (mem2chunk (newp)) ||
 3374           ar_ptr == arena_for_chunk (mem2chunk (newp)));
 3375 
 3376       return newp;
 3377     }
 3378 
 3379   __libc_lock_lock (ar_ptr->mutex);
 3380 
 3381   newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
 3382 
 3383   __libc_lock_unlock (ar_ptr->mutex);
 3384   assert (!newp || chunk_is_mmapped (mem2chunk (newp)) ||
 3385           ar_ptr == arena_for_chunk (mem2chunk (newp)));
 3386 
 3387   if (newp == NULL)
 3388     {
 3389       /* Try harder to allocate memory in other arenas.  */
 3390       LIBC_PROBE (memory_realloc_retry, 2, bytes, oldmem);
 3391       newp = __libc_malloc (bytes);
 3392       if (newp != NULL)
 3393         {
 3394       size_t sz = memsize (oldp);
 3395       memcpy (newp, oldmem, sz);
 3396       (void) tag_region (chunk2mem (oldp), sz);
 3397           _int_free (ar_ptr, oldp, 0);
 3398         }
 3399     }
 3400 
 3401   return newp;
 3402 }
 3403 libc_hidden_def (__libc_realloc)
 3404 
 3405 void *
 3406 __libc_memalign (size_t alignment, size_t bytes)
 3407 {
 3408   if (!__malloc_initialized)
 3409     ptmalloc_init ();
 3410 
 3411   void *address = RETURN_ADDRESS (0);
 3412   return _mid_memalign (alignment, bytes, address);
 3413 }
 3414 
 3415 static void *
 3416 _mid_memalign (size_t alignment, size_t bytes, void *address)
 3417 {
 3418   mstate ar_ptr;
 3419   void *p;
 3420 
 3421   /* If we need less alignment than we give anyway, just relay to malloc.  */
 3422   if (alignment <= MALLOC_ALIGNMENT)
 3423     return __libc_malloc (bytes);
 3424 
 3425   /* Otherwise, ensure that it is at least a minimum chunk size */
 3426   if (alignment < MINSIZE)
 3427     alignment = MINSIZE;
 3428 
 3429   /* If the alignment is greater than SIZE_MAX / 2 + 1 it cannot be a
 3430      power of 2 and will cause overflow in the check below.  */
 3431   if (alignment > SIZE_MAX / 2 + 1)
 3432     {
 3433       __set_errno (EINVAL);
 3434       return 0;
 3435     }
 3436 
 3437 
 3438   /* Make sure alignment is power of 2.  */
 3439   if (!powerof2 (alignment))
 3440     {
 3441       size_t a = MALLOC_ALIGNMENT * 2;
 3442       while (a < alignment)
 3443         a <<= 1;
 3444       alignment = a;
 3445     }
 3446 
 3447   if (SINGLE_THREAD_P)
 3448     {
 3449       p = _int_memalign (&main_arena, alignment, bytes);
 3450       assert (!p || chunk_is_mmapped (mem2chunk (p)) ||
 3451           &main_arena == arena_for_chunk (mem2chunk (p)));
 3452       return tag_new_usable (p);
 3453     }
 3454 
 3455   arena_get (ar_ptr, bytes + alignment + MINSIZE);
 3456 
 3457   p = _int_memalign (ar_ptr, alignment, bytes);
 3458   if (!p && ar_ptr != NULL)
 3459     {
 3460       LIBC_PROBE (memory_memalign_retry, 2, bytes, alignment);
 3461       ar_ptr = arena_get_retry (ar_ptr, bytes);
 3462       p = _int_memalign (ar_ptr, alignment, bytes);
 3463     }
 3464 
 3465   if (ar_ptr != NULL)
 3466     __libc_lock_unlock (ar_ptr->mutex);
 3467 
 3468   assert (!p || chunk_is_mmapped (mem2chunk (p)) ||
 3469           ar_ptr == arena_for_chunk (mem2chunk (p)));
 3470   return tag_new_usable (p);
 3471 }
 3472 /* For ISO C11.  */
 3473 weak_alias (__libc_memalign, aligned_alloc)
 3474 libc_hidden_def (__libc_memalign)
 3475 
 3476 void *
 3477 __libc_valloc (size_t bytes)
 3478 {
 3479   if (!__malloc_initialized)
 3480     ptmalloc_init ();
 3481 
 3482   void *address = RETURN_ADDRESS (0);
 3483   size_t pagesize = GLRO (dl_pagesize);
 3484   return _mid_memalign (pagesize, bytes, address);
 3485 }
 3486 
 3487 void *
 3488 __libc_pvalloc (size_t bytes)
 3489 {
 3490   if (!__malloc_initialized)
 3491     ptmalloc_init ();
 3492 
 3493   void *address = RETURN_ADDRESS (0);
 3494   size_t pagesize = GLRO (dl_pagesize);
 3495   size_t rounded_bytes;
 3496   /* ALIGN_UP with overflow check.  */
 3497   if (__glibc_unlikely (__builtin_add_overflow (bytes,
 3498                         pagesize - 1,
 3499                         &rounded_bytes)))
 3500     {
 3501       __set_errno (ENOMEM);
 3502       return 0;
 3503     }
 3504   rounded_bytes = rounded_bytes & -(pagesize - 1);
 3505 
 3506   return _mid_memalign (pagesize, rounded_bytes, address);
 3507 }
 3508 
 3509 void *
 3510 __libc_calloc (size_t n, size_t elem_size)
 3511 {
 3512   mstate av;
 3513   mchunkptr oldtop;
 3514   INTERNAL_SIZE_T sz, oldtopsize;
 3515   void *mem;
 3516   unsigned long clearsize;
 3517   unsigned long nclears;
 3518   INTERNAL_SIZE_T *d;
 3519   ptrdiff_t bytes;
 3520 
 3521   if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
 3522     {
 3523        __set_errno (ENOMEM);
 3524        return NULL;
 3525     }
 3526 
 3527   sz = bytes;
 3528 
 3529   if (!__malloc_initialized)
 3530     ptmalloc_init ();
 3531 
 3532   MAYBE_INIT_TCACHE ();
 3533 
 3534   if (SINGLE_THREAD_P)
 3535     av = &main_arena;
 3536   else
 3537     arena_get (av, sz);
 3538 
 3539   if (av)
 3540     {
 3541       /* Check if we hand out the top chunk, in which case there may be no
 3542      need to clear. */
 3543 #if MORECORE_CLEARS
 3544       oldtop = top (av);
 3545       oldtopsize = chunksize (top (av));
 3546 # if MORECORE_CLEARS < 2
 3547       /* Only newly allocated memory is guaranteed to be cleared.  */
 3548       if (av == &main_arena &&
 3549       oldtopsize < mp_.sbrk_base + av->max_system_mem - (char *) oldtop)
 3550     oldtopsize = (mp_.sbrk_base + av->max_system_mem - (char *) oldtop);
 3551 # endif
 3552       if (av != &main_arena)
 3553     {
 3554       heap_info *heap = heap_for_ptr (oldtop);
 3555       if (oldtopsize < (char *) heap + heap->mprotect_size - (char *) oldtop)
 3556         oldtopsize = (char *) heap + heap->mprotect_size - (char *) oldtop;
 3557     }
 3558 #endif
 3559     }
 3560   else
 3561     {
 3562       /* No usable arenas.  */
 3563       oldtop = 0;
 3564       oldtopsize = 0;
 3565     }
 3566   mem = _int_malloc (av, sz);
 3567 
 3568   assert (!mem || chunk_is_mmapped (mem2chunk (mem)) ||
 3569           av == arena_for_chunk (mem2chunk (mem)));
 3570 
 3571   if (!SINGLE_THREAD_P)
 3572     {
 3573       if (mem == 0 && av != NULL)
 3574     {
 3575       LIBC_PROBE (memory_calloc_retry, 1, sz);
 3576       av = arena_get_retry (av, sz);
 3577       mem = _int_malloc (av, sz);
 3578     }
 3579 
 3580       if (av != NULL)
 3581     __libc_lock_unlock (av->mutex);
 3582     }
 3583 
 3584   /* Allocation failed even after a retry.  */
 3585   if (mem == 0)
 3586     return 0;
 3587 
 3588   mchunkptr p = mem2chunk (mem);
 3589 
 3590   /* If we are using memory tagging, then we need to set the tags
 3591      regardless of MORECORE_CLEARS, so we zero the whole block while
 3592      doing so.  */
 3593   if (__glibc_unlikely (mtag_enabled))
 3594     return tag_new_zero_region (mem, memsize (p));
 3595 
 3596   INTERNAL_SIZE_T csz = chunksize (p);
 3597 
 3598   /* Two optional cases in which clearing not necessary */
 3599   if (chunk_is_mmapped (p))
 3600     {
 3601       if (__builtin_expect (perturb_byte, 0))
 3602         return memset (mem, 0, sz);
 3603 
 3604       return mem;
 3605     }
 3606 
 3607 #if MORECORE_CLEARS
 3608   if (perturb_byte == 0 && (p == oldtop && csz > oldtopsize))
 3609     {
 3610       /* clear only the bytes from non-freshly-sbrked memory */
 3611       csz = oldtopsize;
 3612     }
 3613 #endif
 3614 
 3615   /* Unroll clear of <= 36 bytes (72 if 8byte sizes).  We know that
 3616      contents have an odd number of INTERNAL_SIZE_T-sized words;
 3617      minimally 3.  */
 3618   d = (INTERNAL_SIZE_T *) mem;
 3619   clearsize = csz - SIZE_SZ;
 3620   nclears = clearsize / sizeof (INTERNAL_SIZE_T);
 3621   assert (nclears >= 3);
 3622 
 3623   if (nclears > 9)
 3624     return memset (d, 0, clearsize);
 3625 
 3626   else
 3627     {
 3628       *(d + 0) = 0;
 3629       *(d + 1) = 0;
 3630       *(d + 2) = 0;
 3631       if (nclears > 4)
 3632         {
 3633           *(d + 3) = 0;
 3634           *(d + 4) = 0;
 3635           if (nclears > 6)
 3636             {
 3637               *(d + 5) = 0;
 3638               *(d + 6) = 0;
 3639               if (nclears > 8)
 3640                 {
 3641                   *(d + 7) = 0;
 3642                   *(d + 8) = 0;
 3643                 }
 3644             }
 3645         }
 3646     }
 3647 
 3648   return mem;
 3649 }
 3650 #endif /* IS_IN (libc) */
 3651 
 3652 /*
 3653    ------------------------------ malloc ------------------------------
 3654  */
 3655 
 3656 static void *
 3657 _int_malloc (mstate av, size_t bytes)
 3658 {
 3659   INTERNAL_SIZE_T nb;               /* normalized request size */
 3660   unsigned int idx;                 /* associated bin index */
 3661   mbinptr bin;                      /* associated bin */
 3662 
 3663   mchunkptr victim;                 /* inspected/selected chunk */
 3664   INTERNAL_SIZE_T size;             /* its size */
 3665   int victim_index;                 /* its bin index */
 3666 
 3667   mchunkptr remainder;              /* remainder from a split */
 3668   unsigned long remainder_size;     /* its size */
 3669 
 3670   unsigned int block;               /* bit map traverser */
 3671   unsigned int bit;                 /* bit map traverser */
 3672   unsigned int map;                 /* current word of binmap */
 3673 
 3674   mchunkptr fwd;                    /* misc temp for linking */
 3675   mchunkptr bck;                    /* misc temp for linking */
 3676 
 3677 #if USE_TCACHE
 3678   size_t tcache_unsorted_count;     /* count of unsorted chunks processed */
 3679 #endif
 3680 
 3681   /*
 3682      Convert request size to internal form by adding SIZE_SZ bytes
 3683      overhead plus possibly more to obtain necessary alignment and/or
 3684      to obtain a size of at least MINSIZE, the smallest allocatable
 3685      size. Also, checked_request2size returns false for request sizes
 3686      that are so large that they wrap around zero when padded and
 3687      aligned.
 3688    */
 3689 
 3690   if (!checked_request2size (bytes, &nb))
 3691     {
 3692       __set_errno (ENOMEM);
 3693       return NULL;
 3694     }
 3695 
 3696   /* There are no usable arenas.  Fall back to sysmalloc to get a chunk from
 3697      mmap.  */
 3698   if (__glibc_unlikely (av == NULL))
 3699     {
 3700       void *p = sysmalloc (nb, av);
 3701       if (p != NULL)
 3702     alloc_perturb (p, bytes);
 3703       return p;
 3704     }
 3705 
 3706   /*
 3707      If the size qualifies as a fastbin, first check corresponding bin.
 3708      This code is safe to execute even if av is not yet initialized, so we
 3709      can try it without checking, which saves some time on this fast path.
 3710    */
 3711 
 3712 #define REMOVE_FB(fb, victim, pp)           \
 3713   do                            \
 3714     {                           \
 3715       victim = pp;                  \
 3716       if (victim == NULL)               \
 3717     break;                      \
 3718       pp = REVEAL_PTR (victim->fd);                                     \
 3719       if (__glibc_unlikely (pp != NULL && misaligned_chunk (pp)))       \
 3720     malloc_printerr ("malloc(): unaligned fastbin chunk detected"); \
 3721     }                           \
 3722   while ((pp = catomic_compare_and_exchange_val_acq (fb, pp, victim)) \
 3723      != victim);                    \
 3724 
 3725   if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
 3726     {
 3727       idx = fastbin_index (nb);
 3728       mfastbinptr *fb = &fastbin (av, idx);
 3729       mchunkptr pp;
 3730       victim = *fb;
 3731 
 3732       if (victim != NULL)
 3733     {
 3734       if (__glibc_unlikely (misaligned_chunk (victim)))
 3735         malloc_printerr ("malloc(): unaligned fastbin chunk detected 2");
 3736 
 3737       if (SINGLE_THREAD_P)
 3738         *fb = REVEAL_PTR (victim->fd);
 3739       else
 3740         REMOVE_FB (fb, pp, victim);
 3741       if (__glibc_likely (victim != NULL))
 3742         {
 3743           size_t victim_idx = fastbin_index (chunksize (victim));
 3744           if (__builtin_expect (victim_idx != idx, 0))
 3745         malloc_printerr ("malloc(): memory corruption (fast)");
 3746           check_remalloced_chunk (av, victim, nb);
 3747 #if USE_TCACHE
 3748           /* While we're here, if we see other chunks of the same size,
 3749          stash them in the tcache.  */
 3750           size_t tc_idx = csize2tidx (nb);
 3751           if (tcache && tc_idx < mp_.tcache_bins)
 3752         {
 3753           mchunkptr tc_victim;
 3754 
 3755           /* While bin not empty and tcache not full, copy chunks.  */
 3756           while (tcache->counts[tc_idx] < mp_.tcache_count
 3757              && (tc_victim = *fb) != NULL)
 3758             {
 3759               if (__glibc_unlikely (misaligned_chunk (tc_victim)))
 3760             malloc_printerr ("malloc(): unaligned fastbin chunk detected 3");
 3761               if (SINGLE_THREAD_P)
 3762             *fb = REVEAL_PTR (tc_victim->fd);
 3763               else
 3764             {
 3765               REMOVE_FB (fb, pp, tc_victim);
 3766               if (__glibc_unlikely (tc_victim == NULL))
 3767                 break;
 3768             }
 3769               tcache_put (tc_victim, tc_idx);
 3770             }
 3771         }
 3772 #endif
 3773           void *p = chunk2mem (victim);
 3774           alloc_perturb (p, bytes);
 3775           return p;
 3776         }
 3777     }
 3778     }
 3779 
 3780   /*
 3781      If a small request, check regular bin.  Since these "smallbins"
 3782      hold one size each, no searching within bins is necessary.
 3783      (For a large request, we need to wait until unsorted chunks are
 3784      processed to find best fit. But for small ones, fits are exact
 3785      anyway, so we can check now, which is faster.)
 3786    */
 3787 
 3788   if (in_smallbin_range (nb))
 3789     {
 3790       idx = smallbin_index (nb);
 3791       bin = bin_at (av, idx);
 3792 
 3793       if ((victim = last (bin)) != bin)
 3794         {
 3795           bck = victim->bk;
 3796       if (__glibc_unlikely (bck->fd != victim))
 3797         malloc_printerr ("malloc(): smallbin double linked list corrupted");
 3798           set_inuse_bit_at_offset (victim, nb);
 3799           bin->bk = bck;
 3800           bck->fd = bin;
 3801 
 3802           if (av != &main_arena)
 3803         set_non_main_arena (victim);
 3804           check_malloced_chunk (av, victim, nb);
 3805 #if USE_TCACHE
 3806       /* While we're here, if we see other chunks of the same size,
 3807          stash them in the tcache.  */
 3808       size_t tc_idx = csize2tidx (nb);
 3809       if (tcache && tc_idx < mp_.tcache_bins)
 3810         {
 3811           mchunkptr tc_victim;
 3812 
 3813           /* While bin not empty and tcache not full, copy chunks over.  */
 3814           while (tcache->counts[tc_idx] < mp_.tcache_count
 3815              && (tc_victim = last (bin)) != bin)
 3816         {
 3817           if (tc_victim != 0)
 3818             {
 3819               bck = tc_victim->bk;
 3820               set_inuse_bit_at_offset (tc_victim, nb);
 3821               if (av != &main_arena)
 3822             set_non_main_arena (tc_victim);
 3823               bin->bk = bck;
 3824               bck->fd = bin;
 3825 
 3826               tcache_put (tc_victim, tc_idx);
 3827                 }
 3828         }
 3829         }
 3830 #endif
 3831           void *p = chunk2mem (victim);
 3832           alloc_perturb (p, bytes);
 3833           return p;
 3834         }
 3835     }
 3836 
 3837   /*
 3838      If this is a large request, consolidate fastbins before continuing.
 3839      While it might look excessive to kill all fastbins before
 3840      even seeing if there is space available, this avoids
 3841      fragmentation problems normally associated with fastbins.
 3842      Also, in practice, programs tend to have runs of either small or
 3843      large requests, but less often mixtures, so consolidation is not
 3844      invoked all that often in most programs. And the programs that
 3845      it is called frequently in otherwise tend to fragment.
 3846    */
 3847 
 3848   else
 3849     {
 3850       idx = largebin_index (nb);
 3851       if (atomic_load_relaxed (&av->have_fastchunks))
 3852         malloc_consolidate (av);
 3853     }
 3854 
 3855   /*
 3856      Process recently freed or remaindered chunks, taking one only if
 3857      it is exact fit, or, if this a small request, the chunk is remainder from
 3858      the most recent non-exact fit.  Place other traversed chunks in
 3859      bins.  Note that this step is the only place in any routine where
 3860      chunks are placed in bins.
 3861 
 3862      The outer loop here is needed because we might not realize until
 3863      near the end of malloc that we should have consolidated, so must
 3864      do so and retry. This happens at most once, and only when we would
 3865      otherwise need to expand memory to service a "small" request.
 3866    */
 3867 
 3868 #if USE_TCACHE
 3869   INTERNAL_SIZE_T tcache_nb = 0;
 3870   size_t tc_idx = csize2tidx (nb);
 3871   if (tcache && tc_idx < mp_.tcache_bins)
 3872     tcache_nb = nb;
 3873   int return_cached = 0;
 3874 
 3875   tcache_unsorted_count = 0;
 3876 #endif
 3877 
 3878   for (;; )
 3879     {
 3880       int iters = 0;
 3881       while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av))
 3882         {
 3883           bck = victim->bk;
 3884           size = chunksize (victim);
 3885           mchunkptr next = chunk_at_offset (victim, size);
 3886 
 3887           if (__glibc_unlikely (size <= CHUNK_HDR_SZ)
 3888               || __glibc_unlikely (size > av->system_mem))
 3889             malloc_printerr ("malloc(): invalid size (unsorted)");
 3890           if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ)
 3891               || __glibc_unlikely (chunksize_nomask (next) > av->system_mem))
 3892             malloc_printerr ("malloc(): invalid next size (unsorted)");
 3893           if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size))
 3894             malloc_printerr ("malloc(): mismatching next->prev_size (unsorted)");
 3895           if (__glibc_unlikely (bck->fd != victim)
 3896               || __glibc_unlikely (victim->fd != unsorted_chunks (av)))
 3897             malloc_printerr ("malloc(): unsorted double linked list corrupted");
 3898           if (__glibc_unlikely (prev_inuse (next)))
 3899             malloc_printerr ("malloc(): invalid next->prev_inuse (unsorted)");
 3900 
 3901           /*
 3902              If a small request, try to use last remainder if it is the
 3903              only chunk in unsorted bin.  This helps promote locality for
 3904              runs of consecutive small requests. This is the only
 3905              exception to best-fit, and applies only when there is
 3906              no exact fit for a small chunk.
 3907            */
 3908 
 3909           if (in_smallbin_range (nb) &&
 3910               bck == unsorted_chunks (av) &&
 3911               victim == av->last_remainder &&
 3912               (unsigned long) (size) > (unsigned long) (nb + MINSIZE))
 3913             {
 3914               /* split and reattach remainder */
 3915               remainder_size = size - nb;
 3916               remainder = chunk_at_offset (victim, nb);
 3917               unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder;
 3918               av->last_remainder = remainder;
 3919               remainder->bk = remainder->fd = unsorted_chunks (av);
 3920               if (!in_smallbin_range (remainder_size))
 3921                 {
 3922                   remainder->fd_nextsize = NULL;
 3923                   remainder->bk_nextsize = NULL;
 3924                 }
 3925 
 3926               set_head (victim, nb | PREV_INUSE |
 3927                         (av != &main_arena ? NON_MAIN_ARENA : 0));
 3928               set_head (remainder, remainder_size | PREV_INUSE);
 3929               set_foot (remainder, remainder_size);
 3930 
 3931               check_malloced_chunk (av, victim, nb);
 3932               void *p = chunk2mem (victim);
 3933               alloc_perturb (p, bytes);
 3934               return p;
 3935             }
 3936 
 3937           /* remove from unsorted list */
 3938           if (__glibc_unlikely (bck->fd != victim))
 3939             malloc_printerr ("malloc(): corrupted unsorted chunks 3");
 3940           unsorted_chunks (av)->bk = bck;
 3941           bck->fd = unsorted_chunks (av);
 3942 
 3943           /* Take now instead of binning if exact fit */
 3944 
 3945           if (size == nb)
 3946             {
 3947               set_inuse_bit_at_offset (victim, size);
 3948               if (av != &main_arena)
 3949         set_non_main_arena (victim);
 3950 #if USE_TCACHE
 3951           /* Fill cache first, return to user only if cache fills.
 3952          We may return one of these chunks later.  */
 3953           if (tcache_nb
 3954           && tcache->counts[tc_idx] < mp_.tcache_count)
 3955         {
 3956           tcache_put (victim, tc_idx);
 3957           return_cached = 1;
 3958           continue;
 3959         }
 3960           else
 3961         {
 3962 #endif
 3963               check_malloced_chunk (av, victim, nb);
 3964               void *p = chunk2mem (victim);
 3965               alloc_perturb (p, bytes);
 3966               return p;
 3967 #if USE_TCACHE
 3968         }
 3969 #endif
 3970             }
 3971 
 3972           /* place chunk in bin */
 3973 
 3974           if (in_smallbin_range (size))
 3975             {
 3976               victim_index = smallbin_index (size);
 3977               bck = bin_at (av, victim_index);
 3978               fwd = bck->fd;
 3979             }
 3980           else
 3981             {
 3982               victim_index = largebin_index (size);
 3983               bck = bin_at (av, victim_index);
 3984               fwd = bck->fd;
 3985 
 3986               /* maintain large bins in sorted order */
 3987               if (fwd != bck)
 3988                 {
 3989                   /* Or with inuse bit to speed comparisons */
 3990                   size |= PREV_INUSE;
 3991                   /* if smaller than smallest, bypass loop below */
 3992                   assert (chunk_main_arena (bck->bk));
 3993                   if ((unsigned long) (size)
 3994               < (unsigned long) chunksize_nomask (bck->bk))
 3995                     {
 3996                       fwd = bck;
 3997                       bck = bck->bk;
 3998 
 3999                       victim->fd_nextsize = fwd->fd;
 4000                       victim->bk_nextsize = fwd->fd->bk_nextsize;
 4001                       fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim;
 4002                     }
 4003                   else
 4004                     {
 4005                       assert (chunk_main_arena (fwd));
 4006                       while ((unsigned long) size < chunksize_nomask (fwd))
 4007                         {
 4008                           fwd = fwd->fd_nextsize;
 4009               assert (chunk_main_arena (fwd));
 4010                         }
 4011 
 4012                       if ((unsigned long) size
 4013               == (unsigned long) chunksize_nomask (fwd))
 4014                         /* Always insert in the second position.  */
 4015                         fwd = fwd->fd;
 4016                       else
 4017                         {
 4018                           victim->fd_nextsize = fwd;
 4019                           victim->bk_nextsize = fwd->bk_nextsize;
 4020                           if (__glibc_unlikely (fwd->bk_nextsize->fd_nextsize != fwd))
 4021                             malloc_printerr ("malloc(): largebin double linked list corrupted (nextsize)");
 4022                           fwd->bk_nextsize = victim;
 4023                           victim->bk_nextsize->fd_nextsize = victim;
 4024                         }
 4025                       bck = fwd->bk;
 4026                       if (bck->fd != fwd)
 4027                         malloc_printerr ("malloc(): largebin double linked list corrupted (bk)");
 4028                     }
 4029                 }
 4030               else
 4031                 victim->fd_nextsize = victim->bk_nextsize = victim;
 4032             }
 4033 
 4034           mark_bin (av, victim_index);
 4035           victim->bk = bck;
 4036           victim->fd = fwd;
 4037           fwd->bk = victim;
 4038           bck->fd = victim;
 4039 
 4040 #if USE_TCACHE
 4041       /* If we've processed as many chunks as we're allowed while
 4042      filling the cache, return one of the cached ones.  */
 4043       ++tcache_unsorted_count;
 4044       if (return_cached
 4045       && mp_.tcache_unsorted_limit > 0
 4046       && tcache_unsorted_count > mp_.tcache_unsorted_limit)
 4047     {
 4048       return tcache_get (tc_idx);
 4049     }
 4050 #endif
 4051 
 4052 #define MAX_ITERS       10000
 4053           if (++iters >= MAX_ITERS)
 4054             break;
 4055         }
 4056 
 4057 #if USE_TCACHE
 4058       /* If all the small chunks we found ended up cached, return one now.  */
 4059       if (return_cached)
 4060     {
 4061       return tcache_get (tc_idx);
 4062     }
 4063 #endif
 4064 
 4065       /*
 4066          If a large request, scan through the chunks of current bin in
 4067          sorted order to find smallest that fits.  Use the skip list for this.
 4068        */
 4069 
 4070       if (!in_smallbin_range (nb))
 4071         {
 4072           bin = bin_at (av, idx);
 4073 
 4074           /* skip scan if empty or largest chunk is too small */
 4075           if ((victim = first (bin)) != bin
 4076           && (unsigned long) chunksize_nomask (victim)
 4077             >= (unsigned long) (nb))
 4078             {
 4079               victim = victim->bk_nextsize;
 4080               while (((unsigned long) (size = chunksize (victim)) <
 4081                       (unsigned long) (nb)))
 4082                 victim = victim->bk_nextsize;
 4083 
 4084               /* Avoid removing the first entry for a size so that the skip
 4085                  list does not have to be rerouted.  */
 4086               if (victim != last (bin)
 4087           && chunksize_nomask (victim)
 4088             == chunksize_nomask (victim->fd))
 4089                 victim = victim->fd;
 4090 
 4091               remainder_size = size - nb;
 4092               unlink_chunk (av, victim);
 4093 
 4094               /* Exhaust */
 4095               if (remainder_size < MINSIZE)
 4096                 {
 4097                   set_inuse_bit_at_offset (victim, size);
 4098                   if (av != &main_arena)
 4099             set_non_main_arena (victim);
 4100                 }
 4101               /* Split */
 4102               else
 4103                 {
 4104                   remainder = chunk_at_offset (victim, nb);
 4105                   /* We cannot assume the unsorted list is empty and therefore
 4106                      have to perform a complete insert here.  */
 4107                   bck = unsorted_chunks (av);
 4108                   fwd = bck->fd;
 4109           if (__glibc_unlikely (fwd->bk != bck))
 4110             malloc_printerr ("malloc(): corrupted unsorted chunks");
 4111                   remainder->bk = bck;
 4112                   remainder->fd = fwd;
 4113                   bck->fd = remainder;
 4114                   fwd->bk = remainder;
 4115                   if (!in_smallbin_range (remainder_size))
 4116                     {
 4117                       remainder->fd_nextsize = NULL;
 4118                       remainder->bk_nextsize = NULL;
 4119                     }
 4120                   set_head (victim, nb | PREV_INUSE |
 4121                             (av != &main_arena ? NON_MAIN_ARENA : 0));
 4122                   set_head (remainder, remainder_size | PREV_INUSE);
 4123                   set_foot (remainder, remainder_size);
 4124                 }
 4125               check_malloced_chunk (av, victim, nb);
 4126               void *p = chunk2mem (victim);
 4127               alloc_perturb (p, bytes);
 4128               return p;
 4129             }
 4130         }
 4131 
 4132       /*
 4133          Search for a chunk by scanning bins, starting with next largest
 4134          bin. This search is strictly by best-fit; i.e., the smallest
 4135          (with ties going to approximately the least recently used) chunk
 4136          that fits is selected.
 4137 
 4138          The bitmap avoids needing to check that most blocks are nonempty.
 4139          The particular case of skipping all bins during warm-up phases
 4140          when no chunks have been returned yet is faster than it might look.
 4141        */
 4142 
 4143       ++idx;
 4144       bin = bin_at (av, idx);
 4145       block = idx2block (idx);
 4146       map = av->binmap[block];
 4147       bit = idx2bit (idx);
 4148 
 4149       for (;; )
 4150         {
 4151           /* Skip rest of block if there are no more set bits in this block.  */
 4152           if (bit > map || bit == 0)
 4153             {
 4154               do
 4155                 {
 4156                   if (++block >= BINMAPSIZE) /* out of bins */
 4157                     goto use_top;
 4158                 }
 4159               while ((map = av->binmap[block]) == 0);
 4160 
 4161               bin = bin_at (av, (block << BINMAPSHIFT));
 4162               bit = 1;
 4163             }
 4164 
 4165           /* Advance to bin with set bit. There must be one. */
 4166           while ((bit & map) == 0)
 4167             {
 4168               bin = next_bin (bin);
 4169               bit <<= 1;
 4170               assert (bit != 0);
 4171             }
 4172 
 4173           /* Inspect the bin. It is likely to be non-empty */
 4174           victim = last (bin);
 4175 
 4176           /*  If a false alarm (empty bin), clear the bit. */
 4177           if (victim == bin)
 4178             {
 4179               av->binmap[block] = map &= ~bit; /* Write through */
 4180               bin = next_bin (bin);
 4181               bit <<= 1;
 4182             }
 4183 
 4184           else
 4185             {
 4186               size = chunksize (victim);
 4187 
 4188               /*  We know the first chunk in this bin is big enough to use. */
 4189               assert ((unsigned long) (size) >= (unsigned long) (nb));
 4190 
 4191               remainder_size = size - nb;
 4192 
 4193               /* unlink */
 4194               unlink_chunk (av, victim);
 4195 
 4196               /* Exhaust */
 4197               if (remainder_size < MINSIZE)
 4198                 {
 4199                   set_inuse_bit_at_offset (victim, size);
 4200                   if (av != &main_arena)
 4201             set_non_main_arena (victim);
 4202                 }
 4203 
 4204               /* Split */
 4205               else
 4206                 {
 4207                   remainder = chunk_at_offset (victim, nb);
 4208 
 4209                   /* We cannot assume the unsorted list is empty and therefore
 4210                      have to perform a complete insert here.  */
 4211                   bck = unsorted_chunks (av);
 4212                   fwd = bck->fd;
 4213           if (__glibc_unlikely (fwd->bk != bck))
 4214             malloc_printerr ("malloc(): corrupted unsorted chunks 2");
 4215                   remainder->bk = bck;
 4216                   remainder->fd = fwd;
 4217                   bck->fd = remainder;
 4218                   fwd->bk = remainder;
 4219 
 4220                   /* advertise as last remainder */
 4221                   if (in_smallbin_range (nb))
 4222                     av->last_remainder = remainder;
 4223                   if (!in_smallbin_range (remainder_size))
 4224                     {
 4225                       remainder->fd_nextsize = NULL;
 4226                       remainder->bk_nextsize = NULL;
 4227                     }
 4228                   set_head (victim, nb | PREV_INUSE |
 4229                             (av != &main_arena ? NON_MAIN_ARENA : 0));
 4230                   set_head (remainder, remainder_size | PREV_INUSE);
 4231                   set_foot (remainder, remainder_size);
 4232                 }
 4233               check_malloced_chunk (av, victim, nb);
 4234               void *p = chunk2mem (victim);
 4235               alloc_perturb (p, bytes);
 4236               return p;
 4237             }
 4238         }
 4239 
 4240     use_top:
 4241       /*
 4242          If large enough, split off the chunk bordering the end of memory
 4243          (held in av->top). Note that this is in accord with the best-fit
 4244          search rule.  In effect, av->top is treated as larger (and thus
 4245          less well fitting) than any other available chunk since it can
 4246          be extended to be as large as necessary (up to system
 4247          limitations).
 4248 
 4249          We require that av->top always exists (i.e., has size >=
 4250          MINSIZE) after initialization, so if it would otherwise be
 4251          exhausted by current request, it is replenished. (The main
 4252          reason for ensuring it exists is that we may need MINSIZE space
 4253          to put in fenceposts in sysmalloc.)
 4254        */
 4255 
 4256       victim = av->top;
 4257       size = chunksize (victim);
 4258 
 4259       if (__glibc_unlikely (size > av->system_mem))
 4260         malloc_printerr ("malloc(): corrupted top size");
 4261 
 4262       if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
 4263         {
 4264           remainder_size = size - nb;
 4265           remainder = chunk_at_offset (victim, nb);
 4266           av->top = remainder;
 4267           set_head (victim, nb | PREV_INUSE |
 4268                     (av != &main_arena ? NON_MAIN_ARENA : 0));
 4269           set_head (remainder, remainder_size | PREV_INUSE);
 4270 
 4271           check_malloced_chunk (av, victim, nb);
 4272           void *p = chunk2mem (victim);
 4273           alloc_perturb (p, bytes);
 4274           return p;
 4275         }
 4276 
 4277       /* When we are using atomic ops to free fast chunks we can get
 4278          here for all block sizes.  */
 4279       else if (atomic_load_relaxed (&av->have_fastchunks))
 4280         {
 4281           malloc_consolidate (av);
 4282           /* restore original bin index */
 4283           if (in_smallbin_range (nb))
 4284             idx = smallbin_index (nb);
 4285           else
 4286             idx = largebin_index (nb);
 4287         }
 4288 
 4289       /*
 4290          Otherwise, relay to handle system-dependent cases
 4291        */
 4292       else
 4293         {
 4294           void *p = sysmalloc (nb, av);
 4295           if (p != NULL)
 4296             alloc_perturb (p, bytes);
 4297           return p;
 4298         }
 4299     }
 4300 }
 4301 
 4302 /*
 4303    ------------------------------ free ------------------------------
 4304  */
 4305 
 4306 static void
 4307 _int_free (mstate av, mchunkptr p, int have_lock)
 4308 {
 4309   INTERNAL_SIZE_T size;        /* its size */
 4310   mfastbinptr *fb;             /* associated fastbin */
 4311   mchunkptr nextchunk;         /* next contiguous chunk */
 4312   INTERNAL_SIZE_T nextsize;    /* its size */
 4313   int nextinuse;               /* true if nextchunk is used */
 4314   INTERNAL_SIZE_T prevsize;    /* size of previous contiguous chunk */
 4315   mchunkptr bck;               /* misc temp for linking */
 4316   mchunkptr fwd;               /* misc temp for linking */
 4317 
 4318   size = chunksize (p);
 4319 
 4320   /* Little security check which won't hurt performance: the
 4321      allocator never wrapps around at the end of the address space.
 4322      Therefore we can exclude some size values which might appear
 4323      here by accident or by "design" from some intruder.  */
 4324   if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, 0)
 4325       || __builtin_expect (misaligned_chunk (p), 0))
 4326     malloc_printerr ("free(): invalid pointer");
 4327   /* We know that each chunk is at least MINSIZE bytes in size or a
 4328      multiple of MALLOC_ALIGNMENT.  */
 4329   if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size)))
 4330     malloc_printerr ("free(): invalid size");
 4331 
 4332   check_inuse_chunk(av, p);
 4333 
 4334 #if USE_TCACHE
 4335   {
 4336     size_t tc_idx = csize2tidx (size);
 4337     if (tcache != NULL && tc_idx < mp_.tcache_bins)
 4338       {
 4339     /* Check to see if it's already in the tcache.  */
 4340     tcache_entry *e = (tcache_entry *) chunk2mem (p);
 4341 
 4342     /* This test succeeds on double free.  However, we don't 100%
 4343        trust it (it also matches random payload data at a 1 in
 4344        2^<size_t> chance), so verify it's not an unlikely
 4345        coincidence before aborting.  */
 4346     if (__glibc_unlikely (e->key == tcache_key))
 4347       {
 4348         tcache_entry *tmp;
 4349         size_t cnt = 0;
 4350         LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx);
 4351         for (tmp = tcache->entries[tc_idx];
 4352          tmp;
 4353          tmp = REVEAL_PTR (tmp->next), ++cnt)
 4354           {
 4355         if (cnt >= mp_.tcache_count)
 4356           malloc_printerr ("free(): too many chunks detected in tcache");
 4357         if (__glibc_unlikely (!aligned_OK (tmp)))
 4358           malloc_printerr ("free(): unaligned chunk detected in tcache 2");
 4359         if (tmp == e)
 4360           malloc_printerr ("free(): double free detected in tcache 2");
 4361         /* If we get here, it was a coincidence.  We've wasted a
 4362            few cycles, but don't abort.  */
 4363           }
 4364       }
 4365 
 4366     if (tcache->counts[tc_idx] < mp_.tcache_count)
 4367       {
 4368         tcache_put (p, tc_idx);
 4369         return;
 4370       }
 4371       }
 4372   }
 4373 #endif
 4374 
 4375   /*
 4376     If eligible, place chunk on a fastbin so it can be found
 4377     and used quickly in malloc.
 4378   */
 4379 
 4380   if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
 4381 
 4382 #if TRIM_FASTBINS
 4383       /*
 4384     If TRIM_FASTBINS set, don't place chunks
 4385     bordering top into fastbins
 4386       */
 4387       && (chunk_at_offset(p, size) != av->top)
 4388 #endif
 4389       ) {
 4390 
 4391     if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size))
 4392               <= CHUNK_HDR_SZ, 0)
 4393     || __builtin_expect (chunksize (chunk_at_offset (p, size))
 4394                  >= av->system_mem, 0))
 4395       {
 4396     bool fail = true;
 4397     /* We might not have a lock at this point and concurrent modifications
 4398        of system_mem might result in a false positive.  Redo the test after
 4399        getting the lock.  */
 4400     if (!have_lock)
 4401       {
 4402         __libc_lock_lock (av->mutex);
 4403         fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ
 4404             || chunksize (chunk_at_offset (p, size)) >= av->system_mem);
 4405         __libc_lock_unlock (av->mutex);
 4406       }
 4407 
 4408     if (fail)
 4409       malloc_printerr ("free(): invalid next size (fast)");
 4410       }
 4411 
 4412     free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
 4413 
 4414     atomic_store_relaxed (&av->have_fastchunks, true);
 4415     unsigned int idx = fastbin_index(size);
 4416     fb = &fastbin (av, idx);
 4417 
 4418     /* Atomically link P to its fastbin: P->FD = *FB; *FB = P;  */
 4419     mchunkptr old = *fb, old2;
 4420 
 4421     if (SINGLE_THREAD_P)
 4422       {
 4423     /* Check that the top of the bin is not the record we are going to
 4424        add (i.e., double free).  */
 4425     if (__builtin_expect (old == p, 0))
 4426       malloc_printerr ("double free or corruption (fasttop)");
 4427     p->fd = PROTECT_PTR (&p->fd, old);
 4428     *fb = p;
 4429       }
 4430     else
 4431       do
 4432     {
 4433       /* Check that the top of the bin is not the record we are going to
 4434          add (i.e., double free).  */
 4435       if (__builtin_expect (old == p, 0))
 4436         malloc_printerr ("double free or corruption (fasttop)");
 4437       old2 = old;
 4438       p->fd = PROTECT_PTR (&p->fd, old);
 4439     }
 4440       while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2))
 4441          != old2);
 4442 
 4443     /* Check that size of fastbin chunk at the top is the same as
 4444        size of the chunk that we are adding.  We can dereference OLD
 4445        only if we have the lock, otherwise it might have already been
 4446        allocated again.  */
 4447     if (have_lock && old != NULL
 4448     && __builtin_expect (fastbin_index (chunksize (old)) != idx, 0))
 4449       malloc_printerr ("invalid fastbin entry (free)");
 4450   }
 4451 
 4452   /*
 4453     Consolidate other non-mmapped chunks as they arrive.
 4454   */
 4455 
 4456   else if (!chunk_is_mmapped(p)) {
 4457 
 4458     /* If we're single-threaded, don't lock the arena.  */
 4459     if (SINGLE_THREAD_P)
 4460       have_lock = true;
 4461 
 4462     if (!have_lock)
 4463       __libc_lock_lock (av->mutex);
 4464 
 4465     nextchunk = chunk_at_offset(p, size);
 4466 
 4467     /* Lightweight tests: check whether the block is already the
 4468        top block.  */
 4469     if (__glibc_unlikely (p == av->top))
 4470       malloc_printerr ("double free or corruption (top)");
 4471     /* Or whether the next chunk is beyond the boundaries of the arena.  */
 4472     if (__builtin_expect (contiguous (av)
 4473               && (char *) nextchunk
 4474               >= ((char *) av->top + chunksize(av->top)), 0))
 4475     malloc_printerr ("double free or corruption (out)");
 4476     /* Or whether the block is actually not marked used.  */
 4477     if (__glibc_unlikely (!prev_inuse(nextchunk)))
 4478       malloc_printerr ("double free or corruption (!prev)");
 4479 
 4480     nextsize = chunksize(nextchunk);
 4481     if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0)
 4482     || __builtin_expect (nextsize >= av->system_mem, 0))
 4483       malloc_printerr ("free(): invalid next size (normal)");
 4484 
 4485     free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ);
 4486 
 4487     /* consolidate backward */
 4488     if (!prev_inuse(p)) {
 4489       prevsize = prev_size (p);
 4490       size += prevsize;
 4491       p = chunk_at_offset(p, -((long) prevsize));
 4492       if (__glibc_unlikely (chunksize(p) != prevsize))
 4493         malloc_printerr ("corrupted size vs. prev_size while consolidating");
 4494       unlink_chunk (av, p);
 4495     }
 4496 
 4497     if (nextchunk != av->top) {
 4498       /* get and clear inuse bit */
 4499       nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
 4500 
 4501       /* consolidate forward */
 4502       if (!nextinuse) {
 4503     unlink_chunk (av, nextchunk);
 4504     size += nextsize;
 4505       } else
 4506     clear_inuse_bit_at_offset(nextchunk, 0);
 4507 
 4508       /*
 4509     Place the chunk in unsorted chunk list. Chunks are
 4510     not placed into regular bins until after they have
 4511     been given one chance to be used in malloc.
 4512       */
 4513 
 4514       bck = unsorted_chunks(av);
 4515       fwd = bck->fd;
 4516       if (__glibc_unlikely (fwd->bk != bck))
 4517     malloc_printerr ("free(): corrupted unsorted chunks");
 4518       p->fd = fwd;
 4519       p->bk = bck;
 4520       if (!in_smallbin_range(size))
 4521     {
 4522       p->fd_nextsize = NULL;
 4523       p->bk_nextsize = NULL;
 4524     }
 4525       bck->fd = p;
 4526       fwd->bk = p;
 4527 
 4528       set_head(p, size | PREV_INUSE);
 4529       set_foot(p, size);
 4530 
 4531       check_free_chunk(av, p);
 4532     }
 4533 
 4534     /*
 4535       If the chunk borders the current high end of memory,
 4536       consolidate into top
 4537     */
 4538 
 4539     else {
 4540       size += nextsize;
 4541       set_head(p, size | PREV_INUSE);
 4542       av->top = p;
 4543       check_chunk(av, p);
 4544     }
 4545 
 4546     /*
 4547       If freeing a large space, consolidate possibly-surrounding
 4548       chunks. Then, if the total unused topmost memory exceeds trim
 4549       threshold, ask malloc_trim to reduce top.
 4550 
 4551       Unless max_fast is 0, we don't know if there are fastbins
 4552       bordering top, so we cannot tell for sure whether threshold
 4553       has been reached unless fastbins are consolidated.  But we
 4554       don't want to consolidate on each free.  As a compromise,
 4555       consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
 4556       is reached.
 4557     */
 4558 
 4559     if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
 4560       if (atomic_load_relaxed (&av->have_fastchunks))
 4561     malloc_consolidate(av);
 4562 
 4563       if (av == &main_arena) {
 4564 #ifndef MORECORE_CANNOT_TRIM
 4565     if ((unsigned long)(chunksize(av->top)) >=
 4566         (unsigned long)(mp_.trim_threshold))
 4567       systrim(mp_.top_pad, av);
 4568 #endif
 4569       } else {
 4570     /* Always try heap_trim(), even if the top chunk is not
 4571        large, because the corresponding heap might go away.  */
 4572     heap_info *heap = heap_for_ptr(top(av));
 4573 
 4574     assert(heap->ar_ptr == av);
 4575     heap_trim(heap, mp_.top_pad);
 4576       }
 4577     }
 4578 
 4579     if (!have_lock)
 4580       __libc_lock_unlock (av->mutex);
 4581   }
 4582   /*
 4583     If the chunk was allocated via mmap, release via munmap().
 4584   */
 4585 
 4586   else {
 4587     munmap_chunk (p);
 4588   }
 4589 }
 4590 
 4591 /*
 4592   ------------------------- malloc_consolidate -------------------------
 4593 
 4594   malloc_consolidate is a specialized version of free() that tears
 4595   down chunks held in fastbins.  Free itself cannot be used for this
 4596   purpose since, among other things, it might place chunks back onto
 4597   fastbins.  So, instead, we need to use a minor variant of the same
 4598   code.
 4599 */
 4600 
 4601 static void malloc_consolidate(mstate av)
 4602 {
 4603   mfastbinptr*    fb;                 /* current fastbin being consolidated */
 4604   mfastbinptr*    maxfb;              /* last fastbin (for loop control) */
 4605   mchunkptr       p;                  /* current chunk being consolidated */
 4606   mchunkptr       nextp;              /* next chunk to consolidate */
 4607   mchunkptr       unsorted_bin;       /* bin header */
 4608   mchunkptr       first_unsorted;     /* chunk to link to */
 4609 
 4610   /* These have same use as in free() */
 4611   mchunkptr       nextchunk;
 4612   INTERNAL_SIZE_T size;
 4613   INTERNAL_SIZE_T nextsize;
 4614   INTERNAL_SIZE_T prevsize;
 4615   int             nextinuse;
 4616 
 4617   atomic_store_relaxed (&av->have_fastchunks, false);
 4618 
 4619   unsorted_bin = unsorted_chunks(av);
 4620 
 4621   /*
 4622     Remove each chunk from fast bin and consolidate it, placing it
 4623     then in unsorted bin. Among other reasons for doing this,
 4624     placing in unsorted bin avoids needing to calculate actual bins
 4625     until malloc is sure that chunks aren't immediately going to be
 4626     reused anyway.
 4627   */
 4628 
 4629   maxfb = &fastbin (av, NFASTBINS - 1);
 4630   fb = &fastbin (av, 0);
 4631   do {
 4632     p = atomic_exchange_acq (fb, NULL);
 4633     if (p != 0) {
 4634       do {
 4635     {
 4636       if (__glibc_unlikely (misaligned_chunk (p)))
 4637         malloc_printerr ("malloc_consolidate(): "
 4638                  "unaligned fastbin chunk detected");
 4639 
 4640       unsigned int idx = fastbin_index (chunksize (p));
 4641       if ((&fastbin (av, idx)) != fb)
 4642         malloc_printerr ("malloc_consolidate(): invalid chunk size");
 4643     }
 4644 
 4645     check_inuse_chunk(av, p);
 4646     nextp = REVEAL_PTR (p->fd);
 4647 
 4648     /* Slightly streamlined version of consolidation code in free() */
 4649     size = chunksize (p);
 4650     nextchunk = chunk_at_offset(p, size);
 4651     nextsize = chunksize(nextchunk);
 4652 
 4653     if (!prev_inuse(p)) {
 4654       prevsize = prev_size (p);
 4655       size += prevsize;
 4656       p = chunk_at_offset(p, -((long) prevsize));
 4657       if (__glibc_unlikely (chunksize(p) != prevsize))
 4658         malloc_printerr ("corrupted size vs. prev_size in fastbins");
 4659       unlink_chunk (av, p);
 4660     }
 4661 
 4662     if (nextchunk != av->top) {
 4663       nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
 4664 
 4665       if (!nextinuse) {
 4666         size += nextsize;
 4667         unlink_chunk (av, nextchunk);
 4668       } else
 4669         clear_inuse_bit_at_offset(nextchunk, 0);
 4670 
 4671       first_unsorted = unsorted_bin->fd;
 4672       unsorted_bin->fd = p;
 4673       first_unsorted->bk = p;
 4674 
 4675       if (!in_smallbin_range (size)) {
 4676         p->fd_nextsize = NULL;
 4677         p->bk_nextsize = NULL;
 4678       }
 4679 
 4680       set_head(p, size | PREV_INUSE);
 4681       p->bk = unsorted_bin;
 4682       p->fd = first_unsorted;
 4683       set_foot(p, size);
 4684     }
 4685 
 4686     else {
 4687       size += nextsize;
 4688       set_head(p, size | PREV_INUSE);
 4689       av->top = p;
 4690     }
 4691 
 4692       } while ( (p = nextp) != 0);
 4693 
 4694     }
 4695   } while (fb++ != maxfb);
 4696 }
 4697 
 4698 /*
 4699   ------------------------------ realloc ------------------------------
 4700 */
 4701 
 4702 static void *
 4703 _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize,
 4704          INTERNAL_SIZE_T nb)
 4705 {
 4706   mchunkptr        newp;            /* chunk to return */
 4707   INTERNAL_SIZE_T  newsize;         /* its size */
 4708   void*          newmem;          /* corresponding user mem */
 4709 
 4710   mchunkptr        next;            /* next contiguous chunk after oldp */
 4711 
 4712   mchunkptr        remainder;       /* extra space at end of newp */
 4713   unsigned long    remainder_size;  /* its size */
 4714 
 4715   /* oldmem size */
 4716   if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, 0)
 4717       || __builtin_expect (oldsize >= av->system_mem, 0))
 4718     malloc_printerr ("realloc(): invalid old size");
 4719 
 4720   check_inuse_chunk (av, oldp);
 4721 
 4722   /* All callers already filter out mmap'ed chunks.  */
 4723   assert (!chunk_is_mmapped (oldp));
 4724 
 4725   next = chunk_at_offset (oldp, oldsize);
 4726   INTERNAL_SIZE_T nextsize = chunksize (next);
 4727   if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, 0)
 4728       || __builtin_expect (nextsize >= av->system_mem, 0))
 4729     malloc_printerr ("realloc(): invalid next size");
 4730 
 4731   if ((unsigned long) (oldsize) >= (unsigned long) (nb))
 4732     {
 4733       /* already big enough; split below */
 4734       newp = oldp;
 4735       newsize = oldsize;
 4736     }
 4737 
 4738   else
 4739     {
 4740       /* Try to expand forward into top */
 4741       if (next == av->top &&
 4742           (unsigned long) (newsize = oldsize + nextsize) >=
 4743           (unsigned long) (nb + MINSIZE))
 4744         {
 4745           set_head_size (oldp, nb | (av != &main_arena ? NON_MAIN_ARENA : 0));
 4746           av->top = chunk_at_offset (oldp, nb);
 4747           set_head (av->top, (newsize - nb) | PREV_INUSE);
 4748           check_inuse_chunk (av, oldp);
 4749           return tag_new_usable (chunk2mem (oldp));
 4750         }
 4751 
 4752       /* Try to expand forward into next chunk;  split off remainder below */
 4753       else if (next != av->top &&
 4754                !inuse (next) &&
 4755                (unsigned long) (newsize = oldsize + nextsize) >=
 4756                (unsigned long) (nb))
 4757         {
 4758           newp = oldp;
 4759           unlink_chunk (av, next);
 4760         }
 4761 
 4762       /* allocate, copy, free */
 4763       else
 4764         {
 4765           newmem = _int_malloc (av, nb - MALLOC_ALIGN_MASK);
 4766           if (newmem == 0)
 4767             return 0; /* propagate failure */
 4768 
 4769           newp = mem2chunk (newmem);
 4770           newsize = chunksize (newp);
 4771 
 4772           /*
 4773              Avoid copy if newp is next chunk after oldp.
 4774            */
 4775           if (newp == next)
 4776             {
 4777               newsize += oldsize;
 4778               newp = oldp;
 4779             }
 4780           else
 4781             {
 4782           void *oldmem = chunk2mem (oldp);
 4783           size_t sz = memsize (oldp);
 4784           (void) tag_region (oldmem, sz);
 4785           newmem = tag_new_usable (newmem);
 4786           memcpy (newmem, oldmem, sz);
 4787           _int_free (av, oldp, 1);
 4788           check_inuse_chunk (av, newp);
 4789           return newmem;
 4790             }
 4791         }
 4792     }
 4793 
 4794   /* If possible, free extra space in old or extended chunk */
 4795 
 4796   assert ((unsigned long) (newsize) >= (unsigned long) (nb));
 4797 
 4798   remainder_size = newsize - nb;
 4799 
 4800   if (remainder_size < MINSIZE)   /* not enough extra to split off */
 4801     {
 4802       set_head_size (newp, newsize | (av != &main_arena ? NON_MAIN_ARENA : 0));
 4803       set_inuse_bit_at_offset (newp, newsize);
 4804     }
 4805   else   /* split remainder */
 4806     {
 4807       remainder = chunk_at_offset (newp, nb);
 4808       /* Clear any user-space tags before writing the header.  */
 4809       remainder = tag_region (remainder, remainder_size);
 4810       set_head_size (newp, nb | (av != &main_arena ? NON_MAIN_ARENA : 0));
 4811       set_head (remainder, remainder_size | PREV_INUSE |
 4812                 (av != &main_arena ? NON_MAIN_ARENA : 0));
 4813       /* Mark remainder as inuse so free() won't complain */
 4814       set_inuse_bit_at_offset (remainder, remainder_size);
 4815       _int_free (av, remainder, 1);
 4816     }
 4817 
 4818   check_inuse_chunk (av, newp);
 4819   return tag_new_usable (chunk2mem (newp));
 4820 }
 4821 
 4822 /*
 4823    ------------------------------ memalign ------------------------------
 4824  */
 4825 
 4826 static void *
 4827 _int_memalign (mstate av, size_t alignment, size_t bytes)
 4828 {
 4829   INTERNAL_SIZE_T nb;             /* padded  request size */
 4830   char *m;                        /* memory returned by malloc call */
 4831   mchunkptr p;                    /* corresponding chunk */
 4832   char *brk;                      /* alignment point within p */
 4833   mchunkptr newp;                 /* chunk to return */
 4834   INTERNAL_SIZE_T newsize;        /* its size */
 4835   INTERNAL_SIZE_T leadsize;       /* leading space before alignment point */
 4836   mchunkptr remainder;            /* spare room at end to split off */
 4837   unsigned long remainder_size;   /* its size */
 4838   INTERNAL_SIZE_T size;
 4839 
 4840 
 4841 
 4842   if (!checked_request2size (bytes, &nb))
 4843     {
 4844       __set_errno (ENOMEM);
 4845       return NULL;
 4846     }
 4847 
 4848   /*
 4849      Strategy: find a spot within that chunk that meets the alignment
 4850      request, and then possibly free the leading and trailing space.
 4851    */
 4852 
 4853   /* Call malloc with worst case padding to hit alignment. */
 4854 
 4855   m = (char *) (_int_malloc (av, nb + alignment + MINSIZE));
 4856 
 4857   if (m == 0)
 4858     return 0;           /* propagate failure */
 4859 
 4860   p = mem2chunk (m);
 4861 
 4862   if ((((unsigned long) (m)) % alignment) != 0)   /* misaligned */
 4863 
 4864     { /*
 4865                 Find an aligned spot inside chunk.  Since we need to give back
 4866                 leading space in a chunk of at least MINSIZE, if the first
 4867                 calculation places us at a spot with less than MINSIZE leader,
 4868                 we can move to the next aligned spot -- we've allocated enough
 4869                 total room so that this is always possible.
 4870                  */
 4871       brk = (char *) mem2chunk (((unsigned long) (m + alignment - 1)) &
 4872                                 - ((signed long) alignment));
 4873       if ((unsigned long) (brk - (char *) (p)) < MINSIZE)
 4874         brk += alignment;
 4875 
 4876       newp = (mchunkptr) brk;
 4877       leadsize = brk - (char *) (p);
 4878       newsize = chunksize (p) - leadsize;
 4879 
 4880       /* For mmapped chunks, just adjust offset */
 4881       if (chunk_is_mmapped (p))
 4882         {
 4883           set_prev_size (newp, prev_size (p) + leadsize);
 4884           set_head (newp, newsize | IS_MMAPPED);
 4885           return chunk2mem (newp);
 4886         }
 4887 
 4888       /* Otherwise, give back leader, use the rest */
 4889       set_head (newp, newsize | PREV_INUSE |
 4890                 (av != &main_arena ? NON_MAIN_ARENA : 0));
 4891       set_inuse_bit_at_offset (newp, newsize);
 4892       set_head_size (p, leadsize | (av != &main_arena ? NON_MAIN_ARENA : 0));
 4893       _int_free (av, p, 1);
 4894       p = newp;
 4895 
 4896       assert (newsize >= nb &&
 4897               (((unsigned long) (chunk2mem (p))) % alignment) == 0);
 4898     }
 4899 
 4900   /* Also give back spare room at the end */
 4901   if (!chunk_is_mmapped (p))
 4902     {
 4903       size = chunksize (p);
 4904       if ((unsigned long) (size) > (unsigned long) (nb + MINSIZE))
 4905         {
 4906           remainder_size = size - nb;
 4907           remainder = chunk_at_offset (p, nb);
 4908           set_head (remainder, remainder_size | PREV_INUSE |
 4909                     (av != &main_arena ? NON_MAIN_ARENA : 0));
 4910           set_head_size (p, nb);
 4911           _int_free (av, remainder, 1);
 4912         }
 4913     }
 4914 
 4915   check_inuse_chunk (av, p);
 4916   return chunk2mem (p);
 4917 }
 4918 
 4919 
 4920 /*
 4921    ------------------------------ malloc_trim ------------------------------
 4922  */
 4923 
 4924 static int
 4925 mtrim (mstate av, size_t pad)
 4926 {
 4927   /* Ensure all blocks are consolidated.  */
 4928   malloc_consolidate (av);
 4929 
 4930   const size_t ps = GLRO (dl_pagesize);
 4931   int psindex = bin_index (ps);
 4932   const size_t psm1 = ps - 1;
 4933 
 4934   int result = 0;
 4935   for (int i = 1; i < NBINS; ++i)
 4936     if (i == 1 || i >= psindex)
 4937       {
 4938         mbinptr bin = bin_at (av, i);
 4939 
 4940         for (mchunkptr p = last (bin); p != bin; p = p->bk)
 4941           {
 4942             INTERNAL_SIZE_T size = chunksize (p);
 4943 
 4944             if (size > psm1 + sizeof (struct malloc_chunk))
 4945               {
 4946                 /* See whether the chunk contains at least one unused page.  */
 4947                 char *paligned_mem = (char *) (((uintptr_t) p
 4948                                                 + sizeof (struct malloc_chunk)
 4949                                                 + psm1) & ~psm1);
 4950 
 4951                 assert ((char *) chunk2mem (p) + 2 * CHUNK_HDR_SZ
 4952             <= paligned_mem);
 4953                 assert ((char *) p + size > paligned_mem);
 4954 
 4955                 /* This is the size we could potentially free.  */
 4956                 size -= paligned_mem - (char *) p;
 4957 
 4958                 if (size > psm1)
 4959                   {
 4960 #if MALLOC_DEBUG
 4961                     /* When debugging we simulate destroying the memory
 4962                        content.  */
 4963                     memset (paligned_mem, 0x89, size & ~psm1);
 4964 #endif
 4965                     __madvise (paligned_mem, size & ~psm1, MADV_DONTNEED);
 4966 
 4967                     result = 1;
 4968                   }
 4969               }
 4970           }
 4971       }
 4972 
 4973 #ifndef MORECORE_CANNOT_TRIM
 4974   return result | (av == &main_arena ? systrim (pad, av) : 0);
 4975 
 4976 #else
 4977   return result;
 4978 #endif
 4979 }
 4980 
 4981 
 4982 int
 4983 __malloc_trim (size_t s)
 4984 {
 4985   int result = 0;
 4986 
 4987   if (!__malloc_initialized)
 4988     ptmalloc_init ();
 4989 
 4990   mstate ar_ptr = &main_arena;
 4991   do
 4992     {
 4993       __libc_lock_lock (ar_ptr->mutex);
 4994       result |= mtrim (ar_ptr, s);
 4995       __libc_lock_unlock (ar_ptr->mutex);
 4996 
 4997       ar_ptr = ar_ptr->next;
 4998     }
 4999   while (ar_ptr != &main_arena);
 5000 
 5001   return result;
 5002 }
 5003 
 5004 
 5005 /*
 5006    ------------------------- malloc_usable_size -------------------------
 5007  */
 5008 
 5009 static size_t
 5010 musable (void *mem)
 5011 {
 5012   mchunkptr p;
 5013   if (mem != 0)
 5014     {
 5015       size_t result = 0;
 5016 
 5017       p = mem2chunk (mem);
 5018 
 5019       if (chunk_is_mmapped (p))
 5020     result = chunksize (p) - CHUNK_HDR_SZ;
 5021       else if (inuse (p))
 5022     result = memsize (p);
 5023 
 5024       return result;
 5025     }
 5026   return 0;
 5027 }
 5028 
 5029 #if IS_IN (libc)
 5030 size_t
 5031 __malloc_usable_size (void *m)
 5032 {
 5033   size_t result;
 5034 
 5035   result = musable (m);
 5036   return result;
 5037 }
 5038 #endif
 5039 
 5040 /*
 5041    ------------------------------ mallinfo ------------------------------
 5042    Accumulate malloc statistics for arena AV into M.
 5043  */
 5044 static void
 5045 int_mallinfo (mstate av, struct mallinfo2 *m)
 5046 {
 5047   size_t i;
 5048   mbinptr b;
 5049   mchunkptr p;
 5050   INTERNAL_SIZE_T avail;
 5051   INTERNAL_SIZE_T fastavail;
 5052   int nblocks;
 5053   int nfastblocks;
 5054 
 5055   check_malloc_state (av);
 5056 
 5057   /* Account for top */
 5058   avail = chunksize (av->top);
 5059   nblocks = 1;  /* top always exists */
 5060 
 5061   /* traverse fastbins */
 5062   nfastblocks = 0;
 5063   fastavail = 0;
 5064 
 5065   for (i = 0; i < NFASTBINS; ++i)
 5066     {
 5067       for (p = fastbin (av, i);
 5068        p != 0;
 5069        p = REVEAL_PTR (p->fd))
 5070         {
 5071       if (__glibc_unlikely (misaligned_chunk (p)))
 5072         malloc_printerr ("int_mallinfo(): "
 5073                  "unaligned fastbin chunk detected");
 5074           ++nfastblocks;
 5075           fastavail += chunksize (p);
 5076         }
 5077     }
 5078 
 5079   avail += fastavail;
 5080 
 5081   /* traverse regular bins */
 5082   for (i = 1; i < NBINS; ++i)
 5083     {
 5084       b = bin_at (av, i);
 5085       for (p = last (b); p != b; p = p->bk)
 5086         {
 5087           ++nblocks;
 5088           avail += chunksize (p);
 5089         }
 5090     }
 5091 
 5092   m->smblks += nfastblocks;
 5093   m->ordblks += nblocks;
 5094   m->fordblks += avail;
 5095   m->uordblks += av->system_mem - avail;
 5096   m->arena += av->system_mem;
 5097   m->fsmblks += fastavail;
 5098   if (av == &main_arena)
 5099     {
 5100       m->hblks = mp_.n_mmaps;
 5101       m->hblkhd = mp_.mmapped_mem;
 5102       m->usmblks = 0;
 5103       m->keepcost = chunksize (av->top);
 5104     }
 5105 }
 5106 
 5107 
 5108 struct mallinfo2
 5109 __libc_mallinfo2 (void)
 5110 {
 5111   struct mallinfo2 m;
 5112   mstate ar_ptr;
 5113 
 5114   if (!__malloc_initialized)
 5115     ptmalloc_init ();
 5116 
 5117   memset (&m, 0, sizeof (m));
 5118   ar_ptr = &main_arena;
 5119   do
 5120     {
 5121       __libc_lock_lock (ar_ptr->mutex);
 5122       int_mallinfo (ar_ptr, &m);
 5123       __libc_lock_unlock (ar_ptr->mutex);
 5124 
 5125       ar_ptr = ar_ptr->next;
 5126     }
 5127   while (ar_ptr != &main_arena);
 5128 
 5129   return m;
 5130 }
 5131 libc_hidden_def (__libc_mallinfo2)
 5132 
 5133 struct mallinfo
 5134 __libc_mallinfo (void)
 5135 {
 5136   struct mallinfo m;
 5137   struct mallinfo2 m2 = __libc_mallinfo2 ();
 5138 
 5139   m.arena = m2.arena;
 5140   m.ordblks = m2.ordblks;
 5141   m.smblks = m2.smblks;
 5142   m.hblks = m2.hblks;
 5143   m.hblkhd = m2.hblkhd;
 5144   m.usmblks = m2.usmblks;
 5145   m.fsmblks = m2.fsmblks;
 5146   m.uordblks = m2.uordblks;
 5147   m.fordblks = m2.fordblks;
 5148   m.keepcost = m2.keepcost;
 5149 
 5150   return m;
 5151 }
 5152 
 5153 
 5154 /*
 5155    ------------------------------ malloc_stats ------------------------------
 5156  */
 5157 
 5158 void
 5159 __malloc_stats (void)
 5160 {
 5161   int i;
 5162   mstate ar_ptr;
 5163   unsigned int in_use_b = mp_.mmapped_mem, system_b = in_use_b;
 5164 
 5165   if (!__malloc_initialized)
 5166     ptmalloc_init ();
 5167   _IO_flockfile (stderr);
 5168   int old_flags2 = stderr->_flags2;
 5169   stderr->_flags2 |= _IO_FLAGS2_NOTCANCEL;
 5170   for (i = 0, ar_ptr = &main_arena;; i++)
 5171     {
 5172       struct mallinfo2 mi;
 5173 
 5174       memset (&mi, 0, sizeof (mi));
 5175       __libc_lock_lock (ar_ptr->mutex);
 5176       int_mallinfo (ar_ptr, &mi);
 5177       fprintf (stderr, "Arena %d:\n", i);
 5178       fprintf (stderr, "system bytes     = %10u\n", (unsigned int) mi.arena);
 5179       fprintf (stderr, "in use bytes     = %10u\n", (unsigned int) mi.uordblks);
 5180 #if MALLOC_DEBUG > 1
 5181       if (i > 0)
 5182         dump_heap (heap_for_ptr (top (ar_ptr)));
 5183 #endif
 5184       system_b += mi.arena;
 5185       in_use_b += mi.uordblks;
 5186       __libc_lock_unlock (ar_ptr->mutex);
 5187       ar_ptr = ar_ptr->next;
 5188       if (ar_ptr == &main_arena)
 5189         break;
 5190     }
 5191   fprintf (stderr, "Total (incl. mmap):\n");
 5192   fprintf (stderr, "system bytes     = %10u\n", system_b);
 5193   fprintf (stderr, "in use bytes     = %10u\n", in_use_b);
 5194   fprintf (stderr, "max mmap regions = %10u\n", (unsigned int) mp_.max_n_mmaps);
 5195   fprintf (stderr, "max mmap bytes   = %10lu\n",
 5196            (unsigned long) mp_.max_mmapped_mem);
 5197   stderr->_flags2 = old_flags2;
 5198   _IO_funlockfile (stderr);
 5199 }
 5200 
 5201 
 5202 /*
 5203    ------------------------------ mallopt ------------------------------
 5204  */
 5205 static __always_inline int
 5206 do_set_trim_threshold (size_t value)
 5207 {
 5208   LIBC_PROBE (memory_mallopt_trim_threshold, 3, value, mp_.trim_threshold,
 5209           mp_.no_dyn_threshold);
 5210   mp_.trim_threshold = value;
 5211   mp_.no_dyn_threshold = 1;
 5212   return 1;
 5213 }
 5214 
 5215 static __always_inline int
 5216 do_set_top_pad (size_t value)
 5217 {
 5218   LIBC_PROBE (memory_mallopt_top_pad, 3, value, mp_.top_pad,
 5219           mp_.no_dyn_threshold);
 5220   mp_.top_pad = value;
 5221   mp_.no_dyn_threshold = 1;
 5222   return 1;
 5223 }
 5224 
 5225 static __always_inline int
 5226 do_set_mmap_threshold (size_t value)
 5227 {
 5228   /* Forbid setting the threshold too high.  */
 5229   if (value <= HEAP_MAX_SIZE / 2)
 5230     {
 5231       LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold,
 5232           mp_.no_dyn_threshold);
 5233       mp_.mmap_threshold = value;
 5234       mp_.no_dyn_threshold = 1;
 5235       return 1;
 5236     }
 5237   return 0;
 5238 }
 5239 
 5240 static __always_inline int
 5241 do_set_mmaps_max (int32_t value)
 5242 {
 5243   LIBC_PROBE (memory_mallopt_mmap_max, 3, value, mp_.n_mmaps_max,
 5244           mp_.no_dyn_threshold);
 5245   mp_.n_mmaps_max = value;
 5246   mp_.no_dyn_threshold = 1;
 5247   return 1;
 5248 }
 5249 
 5250 static __always_inline int
 5251 do_set_mallopt_check (int32_t value)
 5252 {
 5253   return 1;
 5254 }
 5255 
 5256 static __always_inline int
 5257 do_set_perturb_byte (int32_t value)
 5258 {
 5259   LIBC_PROBE (memory_mallopt_perturb, 2, value, perturb_byte);
 5260   perturb_byte = value;
 5261   return 1;
 5262 }
 5263 
 5264 static __always_inline int
 5265 do_set_arena_test (size_t value)
 5266 {
 5267   LIBC_PROBE (memory_mallopt_arena_test, 2, value, mp_.arena_test);
 5268   mp_.arena_test = value;
 5269   return 1;
 5270 }
 5271 
 5272 static __always_inline int
 5273 do_set_arena_max (size_t value)
 5274 {
 5275   LIBC_PROBE (memory_mallopt_arena_max, 2, value, mp_.arena_max);
 5276   mp_.arena_max = value;
 5277   return 1;
 5278 }
 5279 
 5280 #if USE_TCACHE
 5281 static __always_inline int
 5282 do_set_tcache_max (size_t value)
 5283 {
 5284   if (value <= MAX_TCACHE_SIZE)
 5285     {
 5286       LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes);
 5287       mp_.tcache_max_bytes = value;
 5288       mp_.tcache_bins = csize2tidx (request2size(value)) + 1;
 5289       return 1;
 5290     }
 5291   return 0;
 5292 }
 5293 
 5294 static __always_inline int
 5295 do_set_tcache_count (size_t value)
 5296 {
 5297   if (value <= MAX_TCACHE_COUNT)
 5298     {
 5299       LIBC_PROBE (memory_tunable_tcache_count, 2, value, mp_.tcache_count);
 5300       mp_.tcache_count = value;
 5301       return 1;
 5302     }
 5303   return 0;
 5304 }
 5305 
 5306 static __always_inline int
 5307 do_set_tcache_unsorted_limit (size_t value)
 5308 {
 5309   LIBC_PROBE (memory_tunable_tcache_unsorted_limit, 2, value, mp_.tcache_unsorted_limit);
 5310   mp_.tcache_unsorted_limit = value;
 5311   return 1;
 5312 }
 5313 #endif
 5314 
 5315 static inline int
 5316 __always_inline
 5317 do_set_mxfast (size_t value)
 5318 {
 5319   if (value <= MAX_FAST_SIZE)
 5320     {
 5321       LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ());
 5322       set_max_fast (value);
 5323       return 1;
 5324     }
 5325   return 0;
 5326 }
 5327 
 5328 int
 5329 __libc_mallopt (int param_number, int value)
 5330 {
 5331   mstate av = &main_arena;
 5332   int res = 1;
 5333 
 5334   if (!__malloc_initialized)
 5335     ptmalloc_init ();
 5336   __libc_lock_lock (av->mutex);
 5337 
 5338   LIBC_PROBE (memory_mallopt, 2, param_number, value);
 5339 
 5340   /* We must consolidate main arena before changing max_fast
 5341      (see definition of set_max_fast).  */
 5342   malloc_consolidate (av);
 5343 
 5344   /* Many of these helper functions take a size_t.  We do not worry
 5345      about overflow here, because negative int values will wrap to
 5346      very large size_t values and the helpers have sufficient range
 5347      checking for such conversions.  Many of these helpers are also
 5348      used by the tunables macros in arena.c.  */
 5349 
 5350   switch (param_number)
 5351     {
 5352     case M_MXFAST:
 5353       res = do_set_mxfast (value);
 5354       break;
 5355 
 5356     case M_TRIM_THRESHOLD:
 5357       res = do_set_trim_threshold (value);
 5358       break;
 5359 
 5360     case M_TOP_PAD:
 5361       res = do_set_top_pad (value);
 5362       break;
 5363 
 5364     case M_MMAP_THRESHOLD:
 5365       res = do_set_mmap_threshold (value);
 5366       break;
 5367 
 5368     case M_MMAP_MAX:
 5369       res = do_set_mmaps_max (value);
 5370       break;
 5371 
 5372     case M_CHECK_ACTION:
 5373       res = do_set_mallopt_check (value);
 5374       break;
 5375 
 5376     case M_PERTURB:
 5377       res = do_set_perturb_byte (value);
 5378       break;
 5379 
 5380     case M_ARENA_TEST:
 5381       if (value > 0)
 5382     res = do_set_arena_test (value);
 5383       break;
 5384 
 5385     case M_ARENA_MAX:
 5386       if (value > 0)
 5387     res = do_set_arena_max (value);
 5388       break;
 5389     }
 5390   __libc_lock_unlock (av->mutex);
 5391   return res;
 5392 }
 5393 libc_hidden_def (__libc_mallopt)
 5394 
 5395 
 5396 /*
 5397    -------------------- Alternative MORECORE functions --------------------
 5398  */
 5399 
 5400 
 5401 /*
 5402    General Requirements for MORECORE.
 5403 
 5404    The MORECORE function must have the following properties:
 5405 
 5406    If MORECORE_CONTIGUOUS is false:
 5407 
 5408  * MORECORE must allocate in multiples of pagesize. It will
 5409       only be called with arguments that are multiples of pagesize.
 5410 
 5411  * MORECORE(0) must return an address that is at least
 5412       MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
 5413 
 5414    else (i.e. If MORECORE_CONTIGUOUS is true):
 5415 
 5416  * Consecutive calls to MORECORE with positive arguments
 5417       return increasing addresses, indicating that space has been
 5418       contiguously extended.
 5419 
 5420  * MORECORE need not allocate in multiples of pagesize.
 5421       Calls to MORECORE need not have args of multiples of pagesize.
 5422 
 5423  * MORECORE need not page-align.
 5424 
 5425    In either case:
 5426 
 5427  * MORECORE may allocate more memory than requested. (Or even less,
 5428       but this will generally result in a malloc failure.)
 5429 
 5430  * MORECORE must not allocate memory when given argument zero, but
 5431       instead return one past the end address of memory from previous
 5432       nonzero call. This malloc does NOT call MORECORE(0)
 5433       until at least one call with positive arguments is made, so
 5434       the initial value returned is not important.
 5435 
 5436  * Even though consecutive calls to MORECORE need not return contiguous
 5437       addresses, it must be OK for malloc'ed chunks to span multiple
 5438       regions in those cases where they do happen to be contiguous.
 5439 
 5440  * MORECORE need not handle negative arguments -- it may instead
 5441       just return MORECORE_FAILURE when given negative arguments.
 5442       Negative arguments are always multiples of pagesize. MORECORE
 5443       must not misinterpret negative args as large positive unsigned
 5444       args. You can suppress all such calls from even occurring by defining
 5445       MORECORE_CANNOT_TRIM,
 5446 
 5447    There is some variation across systems about the type of the
 5448    argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
 5449    actually be size_t, because sbrk supports negative args, so it is
 5450    normally the signed type of the same width as size_t (sometimes
 5451    declared as "intptr_t", and sometimes "ptrdiff_t").  It doesn't much
 5452    matter though. Internally, we use "long" as arguments, which should
 5453    work across all reasonable possibilities.
 5454 
 5455    Additionally, if MORECORE ever returns failure for a positive
 5456    request, then mmap is used as a noncontiguous system allocator. This
 5457    is a useful backup strategy for systems with holes in address spaces
 5458    -- in this case sbrk cannot contiguously expand the heap, but mmap
 5459    may be able to map noncontiguous space.
 5460 
 5461    If you'd like mmap to ALWAYS be used, you can define MORECORE to be
 5462    a function that always returns MORECORE_FAILURE.
 5463 
 5464    If you are using this malloc with something other than sbrk (or its
 5465    emulation) to supply memory regions, you probably want to set
 5466    MORECORE_CONTIGUOUS as false.  As an example, here is a custom
 5467    allocator kindly contributed for pre-OSX macOS.  It uses virtually
 5468    but not necessarily physically contiguous non-paged memory (locked
 5469    in, present and won't get swapped out).  You can use it by
 5470    uncommenting this section, adding some #includes, and setting up the
 5471    appropriate defines above:
 5472 
 5473  *#define MORECORE osMoreCore
 5474  *#define MORECORE_CONTIGUOUS 0
 5475 
 5476    There is also a shutdown routine that should somehow be called for
 5477    cleanup upon program exit.
 5478 
 5479  *#define MAX_POOL_ENTRIES 100
 5480  *#define MINIMUM_MORECORE_SIZE  (64 * 1024)
 5481    static int next_os_pool;
 5482    void *our_os_pools[MAX_POOL_ENTRIES];
 5483 
 5484    void *osMoreCore(int size)
 5485    {
 5486     void *ptr = 0;
 5487     static void *sbrk_top = 0;
 5488 
 5489     if (size > 0)
 5490     {
 5491       if (size < MINIMUM_MORECORE_SIZE)
 5492          size = MINIMUM_MORECORE_SIZE;
 5493       if (CurrentExecutionLevel() == kTaskLevel)
 5494          ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
 5495       if (ptr == 0)
 5496       {
 5497         return (void *) MORECORE_FAILURE;
 5498       }
 5499       // save ptrs so they can be freed during cleanup
 5500       our_os_pools[next_os_pool] = ptr;
 5501       next_os_pool++;
 5502       ptr = (void *) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
 5503       sbrk_top = (char *) ptr + size;
 5504       return ptr;
 5505     }
 5506     else if (size < 0)
 5507     {
 5508       // we don't currently support shrink behavior
 5509       return (void *) MORECORE_FAILURE;
 5510     }
 5511     else
 5512     {
 5513       return sbrk_top;
 5514     }
 5515    }
 5516 
 5517    // cleanup any allocated memory pools
 5518    // called as last thing before shutting down driver
 5519 
 5520    void osCleanupMem(void)
 5521    {
 5522     void **ptr;
 5523 
 5524     for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
 5525       if (*ptr)
 5526       {
 5527          PoolDeallocate(*ptr);
 5528  * ptr = 0;
 5529       }
 5530    }
 5531 
 5532  */
 5533 
 5534 
 5535 /* Helper code.  */
 5536 
 5537 extern char **__libc_argv attribute_hidden;
 5538 
 5539 static void
 5540 malloc_printerr (const char *str)
 5541 {
 5542 #if IS_IN (libc)
 5543   __libc_message (do_abort, "%s\n", str);
 5544 #else
 5545   __libc_fatal (str);
 5546 #endif
 5547   __builtin_unreachable ();
 5548 }
 5549 
 5550 #if IS_IN (libc)
 5551 /* We need a wrapper function for one of the additions of POSIX.  */
 5552 int
 5553 __posix_memalign (void **memptr, size_t alignment, size_t size)
 5554 {
 5555   void *mem;
 5556 
 5557   if (!__malloc_initialized)
 5558     ptmalloc_init ();
 5559 
 5560   /* Test whether the SIZE argument is valid.  It must be a power of
 5561      two multiple of sizeof (void *).  */
 5562   if (alignment % sizeof (void *) != 0
 5563       || !powerof2 (alignment / sizeof (void *))
 5564       || alignment == 0)
 5565     return EINVAL;
 5566 
 5567 
 5568   void *address = RETURN_ADDRESS (0);
 5569   mem = _mid_memalign (alignment, size, address);
 5570 
 5571   if (mem != NULL)
 5572     {
 5573       *memptr = mem;
 5574       return 0;
 5575     }
 5576 
 5577   return ENOMEM;
 5578 }
 5579 weak_alias (__posix_memalign, posix_memalign)
 5580 #endif
 5581 
 5582 
 5583 int
 5584 __malloc_info (int options, FILE *fp)
 5585 {
 5586   /* For now, at least.  */
 5587   if (options != 0)
 5588     return EINVAL;
 5589 
 5590   int n = 0;
 5591   size_t total_nblocks = 0;
 5592   size_t total_nfastblocks = 0;
 5593   size_t total_avail = 0;
 5594   size_t total_fastavail = 0;
 5595   size_t total_system = 0;
 5596   size_t total_max_system = 0;
 5597   size_t total_aspace = 0;
 5598   size_t total_aspace_mprotect = 0;
 5599 
 5600 
 5601 
 5602   if (!__malloc_initialized)
 5603     ptmalloc_init ();
 5604 
 5605   fputs ("<malloc version=\"1\">\n", fp);
 5606 
 5607   /* Iterate over all arenas currently in use.  */
 5608   mstate ar_ptr = &main_arena;
 5609   do
 5610     {
 5611       fprintf (fp, "<heap nr=\"%d\">\n<sizes>\n", n++);
 5612 
 5613       size_t nblocks = 0;
 5614       size_t nfastblocks = 0;
 5615       size_t avail = 0;
 5616       size_t fastavail = 0;
 5617       struct
 5618       {
 5619     size_t from;
 5620     size_t to;
 5621     size_t total;
 5622     size_t count;
 5623       } sizes[NFASTBINS + NBINS - 1];
 5624 #define nsizes (sizeof (sizes) / sizeof (sizes[0]))
 5625 
 5626       __libc_lock_lock (ar_ptr->mutex);
 5627 
 5628       /* Account for top chunk.  The top-most available chunk is
 5629      treated specially and is never in any bin. See "initial_top"
 5630      comments.  */
 5631       avail = chunksize (ar_ptr->top);
 5632       nblocks = 1;  /* Top always exists.  */
 5633 
 5634       for (size_t i = 0; i < NFASTBINS; ++i)
 5635     {
 5636       mchunkptr p = fastbin (ar_ptr, i);
 5637       if (p != NULL)
 5638         {
 5639           size_t nthissize = 0;
 5640           size_t thissize = chunksize (p);
 5641 
 5642           while (p != NULL)
 5643         {
 5644           if (__glibc_unlikely (misaligned_chunk (p)))
 5645             malloc_printerr ("__malloc_info(): "
 5646                      "unaligned fastbin chunk detected");
 5647           ++nthissize;
 5648           p = REVEAL_PTR (p->fd);
 5649         }
 5650 
 5651           fastavail += nthissize * thissize;
 5652           nfastblocks += nthissize;
 5653           sizes[i].from = thissize - (MALLOC_ALIGNMENT - 1);
 5654           sizes[i].to = thissize;
 5655           sizes[i].count = nthissize;
 5656         }
 5657       else
 5658         sizes[i].from = sizes[i].to = sizes[i].count = 0;
 5659 
 5660       sizes[i].total = sizes[i].count * sizes[i].to;
 5661     }
 5662 
 5663 
 5664       mbinptr bin;
 5665       struct malloc_chunk *r;
 5666 
 5667       for (size_t i = 1; i < NBINS; ++i)
 5668     {
 5669       bin = bin_at (ar_ptr, i);
 5670       r = bin->fd;
 5671       sizes[NFASTBINS - 1 + i].from = ~((size_t) 0);
 5672       sizes[NFASTBINS - 1 + i].to = sizes[NFASTBINS - 1 + i].total
 5673                       = sizes[NFASTBINS - 1 + i].count = 0;
 5674 
 5675       if (r != NULL)
 5676         while (r != bin)
 5677           {
 5678         size_t r_size = chunksize_nomask (r);
 5679         ++sizes[NFASTBINS - 1 + i].count;
 5680         sizes[NFASTBINS - 1 + i].total += r_size;
 5681         sizes[NFASTBINS - 1 + i].from
 5682           = MIN (sizes[NFASTBINS - 1 + i].from, r_size);
 5683         sizes[NFASTBINS - 1 + i].to = MAX (sizes[NFASTBINS - 1 + i].to,
 5684                            r_size);
 5685 
 5686         r = r->fd;
 5687           }
 5688 
 5689       if (sizes[NFASTBINS - 1 + i].count == 0)
 5690         sizes[NFASTBINS - 1 + i].from = 0;
 5691       nblocks += sizes[NFASTBINS - 1 + i].count;
 5692       avail += sizes[NFASTBINS - 1 + i].total;
 5693     }
 5694 
 5695       size_t heap_size = 0;
 5696       size_t heap_mprotect_size = 0;
 5697       size_t heap_count = 0;
 5698       if (ar_ptr != &main_arena)
 5699     {
 5700       /* Iterate over the arena heaps from back to front.  */
 5701       heap_info *heap = heap_for_ptr (top (ar_ptr));
 5702       do
 5703         {
 5704           heap_size += heap->size;
 5705           heap_mprotect_size += heap->mprotect_size;
 5706           heap = heap->prev;
 5707           ++heap_count;
 5708         }
 5709       while (heap != NULL);
 5710     }
 5711 
 5712       __libc_lock_unlock (ar_ptr->mutex);
 5713 
 5714       total_nfastblocks += nfastblocks;
 5715       total_fastavail += fastavail;
 5716 
 5717       total_nblocks += nblocks;
 5718       total_avail += avail;
 5719 
 5720       for (size_t i = 0; i < nsizes; ++i)
 5721     if (sizes[i].count != 0 && i != NFASTBINS)
 5722       fprintf (fp, "\
 5723   <size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
 5724            sizes[i].from, sizes[i].to, sizes[i].total, sizes[i].count);
 5725 
 5726       if (sizes[NFASTBINS].count != 0)
 5727     fprintf (fp, "\
 5728   <unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
 5729          sizes[NFASTBINS].from, sizes[NFASTBINS].to,
 5730          sizes[NFASTBINS].total, sizes[NFASTBINS].count);
 5731 
 5732       total_system += ar_ptr->system_mem;
 5733       total_max_system += ar_ptr->max_system_mem;
 5734 
 5735       fprintf (fp,
 5736            "</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
 5737            "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
 5738            "<system type=\"current\" size=\"%zu\"/>\n"
 5739            "<system type=\"max\" size=\"%zu\"/>\n",
 5740            nfastblocks, fastavail, nblocks, avail,
 5741            ar_ptr->system_mem, ar_ptr->max_system_mem);
 5742 
 5743       if (ar_ptr != &main_arena)
 5744     {
 5745       fprintf (fp,
 5746            "<aspace type=\"total\" size=\"%zu\"/>\n"
 5747            "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
 5748            "<aspace type=\"subheaps\" size=\"%zu\"/>\n",
 5749            heap_size, heap_mprotect_size, heap_count);
 5750       total_aspace += heap_size;
 5751       total_aspace_mprotect += heap_mprotect_size;
 5752     }
 5753       else
 5754     {
 5755       fprintf (fp,
 5756            "<aspace type=\"total\" size=\"%zu\"/>\n"
 5757            "<aspace type=\"mprotect\" size=\"%zu\"/>\n",
 5758            ar_ptr->system_mem, ar_ptr->system_mem);
 5759       total_aspace += ar_ptr->system_mem;
 5760       total_aspace_mprotect += ar_ptr->system_mem;
 5761     }
 5762 
 5763       fputs ("</heap>\n", fp);
 5764       ar_ptr = ar_ptr->next;
 5765     }
 5766   while (ar_ptr != &main_arena);
 5767 
 5768   fprintf (fp,
 5769        "<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
 5770        "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
 5771        "<total type=\"mmap\" count=\"%d\" size=\"%zu\"/>\n"
 5772        "<system type=\"current\" size=\"%zu\"/>\n"
 5773        "<system type=\"max\" size=\"%zu\"/>\n"
 5774        "<aspace type=\"total\" size=\"%zu\"/>\n"
 5775        "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
 5776        "</malloc>\n",
 5777        total_nfastblocks, total_fastavail, total_nblocks, total_avail,
 5778        mp_.n_mmaps, mp_.mmapped_mem,
 5779        total_system, total_max_system,
 5780        total_aspace, total_aspace_mprotect);
 5781 
 5782   return 0;
 5783 }
 5784 #if IS_IN (libc)
 5785 weak_alias (__malloc_info, malloc_info)
 5786 
 5787 strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
 5788 strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
 5789 strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
 5790 strong_alias (__libc_memalign, __memalign)
 5791 weak_alias (__libc_memalign, memalign)
 5792 strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
 5793 strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
 5794 strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
 5795 strong_alias (__libc_mallinfo, __mallinfo)
 5796 weak_alias (__libc_mallinfo, mallinfo)
 5797 strong_alias (__libc_mallinfo2, __mallinfo2)
 5798 weak_alias (__libc_mallinfo2, mallinfo2)
 5799 strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
 5800 
 5801 weak_alias (__malloc_stats, malloc_stats)
 5802 weak_alias (__malloc_usable_size, malloc_usable_size)
 5803 weak_alias (__malloc_trim, malloc_trim)
 5804 #endif
 5805 
 5806 #if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_26)
 5807 compat_symbol (libc, __libc_free, cfree, GLIBC_2_0);
 5808 #endif
 5809 
 5810 /* ------------------------------------------------------------
 5811    History:
 5812 
 5813    [see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]
 5814 
 5815  */
 5816 /*
 5817  * Local variables:
 5818  * c-basic-offset: 2
 5819  * End:
 5820  */