"Fossies" - the Fresh Open Source Software Archive

Member "apt-1.9.4/apt-pkg/contrib/mmap.cc" (19 Sep 2019, 14702 Bytes) of package /linux/misc/apt-1.9.4.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "mmap.cc" see the Fossies "Dox" file reference documentation.

    1 // -*- mode: cpp; mode: fold -*-
    2 // Description                              /*{{{*/
    3 /* ######################################################################
    4    
    5    MMap Class - Provides 'real' mmap or a faked mmap using read().
    6 
    7    MMap cover class.
    8 
    9    Some broken versions of glibc2 (libc6) have a broken definition
   10    of mmap that accepts a char * -- all other systems (and libc5) use
   11    void *. We can't safely do anything here that would be portable, so
   12    libc6 generates warnings -- which should be errors, g++ isn't properly
   13    strict.
   14    
   15    ##################################################################### */
   16                                     /*}}}*/
   17 // Include Files                            /*{{{*/
   18 #define _DEFAULT_SOURCE
   19 #include <config.h>
   20 
   21 #include <apt-pkg/error.h>
   22 #include <apt-pkg/fileutl.h>
   23 #include <apt-pkg/macros.h>
   24 #include <apt-pkg/mmap.h>
   25 
   26 #include <cstring>
   27 #include <string>
   28 #include <errno.h>
   29 #include <stdlib.h>
   30 #include <unistd.h>
   31 
   32 #include <apti18n.h>
   33                                     /*}}}*/
   34 
   35 // MMap::MMap - Constructor                     /*{{{*/
   36 // ---------------------------------------------------------------------
   37 /* */
   38 MMap::MMap(FileFd &F,unsigned long Flags) : Flags(Flags), iSize(0),
   39                      Base(nullptr), SyncToFd(nullptr)
   40 {
   41    if ((Flags & NoImmMap) != NoImmMap)
   42       Map(F);
   43 }
   44                                     /*}}}*/
   45 // MMap::MMap - Constructor                     /*{{{*/
   46 // ---------------------------------------------------------------------
   47 /* */
   48 MMap::MMap(unsigned long Flags) : Flags(Flags), iSize(0),
   49                      Base(nullptr), SyncToFd(nullptr)
   50 {
   51 }
   52                                     /*}}}*/
   53 // MMap::~MMap - Destructor                     /*{{{*/
   54 // ---------------------------------------------------------------------
   55 /* */
   56 MMap::~MMap()
   57 {
   58    Close();
   59 }
   60                                     /*}}}*/
   61 // MMap::Map - Perform the mapping                  /*{{{*/
   62 // ---------------------------------------------------------------------
   63 /* */
   64 bool MMap::Map(FileFd &Fd)
   65 {
   66    iSize = Fd.Size();
   67 
   68    // Set the permissions.
   69    int Prot = PROT_READ;
   70    int Map = MAP_SHARED;
   71    if ((Flags & ReadOnly) != ReadOnly)
   72       Prot |= PROT_WRITE;
   73    if ((Flags & Public) != Public)
   74       Map = MAP_PRIVATE;
   75    
   76    if (iSize == 0)
   77       return _error->Error(_("Can't mmap an empty file"));
   78 
   79    // We can't mmap compressed fd's directly, so we need to read it completely
   80    if (Fd.IsCompressed() == true)
   81    {
   82       if ((Flags & ReadOnly) != ReadOnly)
   83      return _error->Error("Compressed file %s can only be mapped readonly", Fd.Name().c_str());
   84       Base = malloc(iSize);
   85       if (unlikely(Base == nullptr))
   86      return _error->Errno("MMap-compressed-malloc", _("Couldn't make mmap of %llu bytes"), iSize);
   87       SyncToFd = new FileFd();
   88       if (Fd.Seek(0L) == false || Fd.Read(Base, iSize) == false)
   89      return _error->Error("Compressed file %s can't be read into mmap", Fd.Name().c_str());
   90       return true;
   91    }
   92 
   93    // Map it.
   94    Base = (Flags & Fallback) ? MAP_FAILED : mmap(0,iSize,Prot,Map,Fd.Fd(),0);
   95    if (Base == MAP_FAILED)
   96    {
   97       if (errno == ENODEV || errno == EINVAL || (Flags & Fallback))
   98       {
   99      // The filesystem doesn't support this particular kind of mmap.
  100      // So we allocate a buffer and read the whole file into it.
  101      if ((Flags & ReadOnly) == ReadOnly)
  102      {
  103         // for readonly, we don't need sync, so make it simple
  104         Base = malloc(iSize);
  105         if (unlikely(Base == nullptr))
  106            return _error->Errno("MMap-malloc", _("Couldn't make mmap of %llu bytes"), iSize);
  107         SyncToFd = new FileFd();
  108         return Fd.Read(Base, iSize);
  109      }
  110      // FIXME: Writing to compressed fd's ?
  111      int const dupped_fd = dup(Fd.Fd());
  112      if (dupped_fd == -1)
  113         return _error->Errno("mmap", _("Couldn't duplicate file descriptor %i"), Fd.Fd());
  114 
  115      Base = calloc(iSize, 1);
  116      if (unlikely(Base == nullptr))
  117         return _error->Errno("MMap-calloc", _("Couldn't make mmap of %llu bytes"), iSize);
  118      SyncToFd = new FileFd (dupped_fd);
  119      if (!SyncToFd->Seek(0L) || !SyncToFd->Read(Base, iSize))
  120         return false;
  121       }
  122       else
  123      return _error->Errno("MMap-mmap", _("Couldn't make mmap of %llu bytes"), iSize);
  124      }
  125 
  126    return true;
  127 }
  128                                     /*}}}*/
  129 // MMap::Close - Close the map                      /*{{{*/
  130 // ---------------------------------------------------------------------
  131 /* */
  132 bool MMap::Close(bool DoSync)
  133 {
  134    if ((Flags & UnMapped) == UnMapped || validData() == false || iSize == 0)
  135       return true;
  136    
  137    if (DoSync == true)
  138       Sync();
  139 
  140    if (SyncToFd != NULL)
  141    {
  142       free(Base);
  143       delete SyncToFd;
  144       SyncToFd = NULL;
  145    }
  146    else
  147    {
  148       if (munmap((char *)Base, iSize) != 0)
  149      _error->WarningE("mmap", _("Unable to close mmap"));
  150    }
  151 
  152    iSize = 0;
  153    Base = 0;
  154    return true;
  155 }
  156                                     /*}}}*/
  157 // MMap::Sync - Synchronize the map with the disk           /*{{{*/
  158 // ---------------------------------------------------------------------
  159 /* This is done in synchronous mode - the docs indicate that this will 
  160    not return till all IO is complete */
  161 bool MMap::Sync()
  162 {
  163    if ((Flags & UnMapped) == UnMapped)
  164       return true;
  165 
  166    if ((Flags & ReadOnly) != ReadOnly)
  167    {
  168       if (SyncToFd != NULL)
  169       {
  170      if (!SyncToFd->Seek(0) || !SyncToFd->Write(Base, iSize))
  171         return false;
  172       }
  173       else
  174       {
  175 #ifdef _POSIX_SYNCHRONIZED_IO
  176      if (msync((char *)Base, iSize, MS_SYNC) < 0)
  177         return _error->Errno("msync", _("Unable to synchronize mmap"));
  178 #endif
  179       }
  180    }
  181    return true;
  182 }
  183                                     /*}}}*/
  184 // MMap::Sync - Synchronize a section of the file to disk       /*{{{*/
  185 // ---------------------------------------------------------------------
  186 /* */
  187 bool MMap::Sync(unsigned long Start,unsigned long Stop)
  188 {
  189    if ((Flags & UnMapped) == UnMapped)
  190       return true;
  191 
  192    if ((Flags & ReadOnly) != ReadOnly)
  193    {
  194       if (SyncToFd != 0)
  195       {
  196      if (!SyncToFd->Seek(0) ||
  197          !SyncToFd->Write (((char *)Base)+Start, Stop-Start))
  198         return false;
  199       }
  200       else
  201       {
  202 #ifdef _POSIX_SYNCHRONIZED_IO
  203      unsigned long long const PSize = sysconf(_SC_PAGESIZE);
  204      if (msync((char *)Base+(Start/PSize)*PSize, Stop - Start, MS_SYNC) < 0)
  205         return _error->Errno("msync", _("Unable to synchronize mmap"));
  206 #endif
  207       }
  208    }
  209    return true;
  210 }
  211                                     /*}}}*/
  212 
  213 // DynamicMMap::DynamicMMap - Constructor               /*{{{*/
  214 // ---------------------------------------------------------------------
  215 /* */
  216 DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Workspace,
  217              unsigned long const &Grow, unsigned long const &Limit) :
  218         MMap(F,Flags | NoImmMap), Fd(&F), WorkSpace(Workspace),
  219         GrowFactor(Grow), Limit(Limit)
  220 {
  221    // disable Moveable if we don't grow
  222    if (Grow == 0)
  223       this->Flags &= ~Moveable;
  224 
  225 #ifndef __linux__
  226    // kfreebsd doesn't have mremap, so we use the fallback
  227    if ((this->Flags & Moveable) == Moveable)
  228       this->Flags |= Fallback;
  229 #endif
  230 
  231    unsigned long long EndOfFile = Fd->Size();
  232    if (EndOfFile > WorkSpace)
  233       WorkSpace = EndOfFile;
  234    else if(WorkSpace > 0)
  235    {
  236       Fd->Seek(WorkSpace - 1);
  237       char C = 0;
  238       Fd->Write(&C,sizeof(C));
  239    }
  240    
  241    Map(F);
  242    iSize = EndOfFile;
  243 }
  244                                     /*}}}*/
  245 // DynamicMMap::DynamicMMap - Constructor for a non-file backed map /*{{{*/
  246 // ---------------------------------------------------------------------
  247 /* We try here to use mmap to reserve some space - this is much more
  248    cooler than the fallback solution to simply allocate a char array
  249    and could come in handy later than we are able to grow such an mmap */
  250 DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace,
  251              unsigned long const &Grow, unsigned long const &Limit) :
  252         MMap(Flags | NoImmMap | UnMapped), Fd(0), WorkSpace(WorkSpace),
  253         GrowFactor(Grow), Limit(Limit)
  254 {
  255     // disable Moveable if we don't grow
  256     if (Grow == 0)
  257         this->Flags &= ~Moveable;
  258 
  259 #ifndef __linux__
  260     // kfreebsd doesn't have mremap, so we use the fallback
  261     if ((this->Flags & Moveable) == Moveable)
  262         this->Flags |= Fallback;
  263 #endif
  264 
  265 #ifdef _POSIX_MAPPED_FILES
  266     if ((this->Flags & Fallback) != Fallback) {
  267         // Set the permissions.
  268         int Prot = PROT_READ;
  269 #ifdef MAP_ANONYMOUS
  270         int Map = MAP_PRIVATE | MAP_ANONYMOUS;
  271 #else
  272         int Map = MAP_PRIVATE | MAP_ANON;
  273 #endif
  274         if ((this->Flags & ReadOnly) != ReadOnly)
  275             Prot |= PROT_WRITE;
  276         if ((this->Flags & Public) == Public)
  277 #ifdef MAP_ANONYMOUS
  278             Map = MAP_SHARED | MAP_ANONYMOUS;
  279 #else
  280             Map = MAP_SHARED | MAP_ANON;
  281 #endif
  282 
  283         // use anonymous mmap() to get the memory
  284         Base = (unsigned char*) mmap(0, WorkSpace, Prot, Map, -1, 0);
  285 
  286         if(Base == MAP_FAILED)
  287             _error->Errno("DynamicMMap",_("Couldn't make mmap of %lu bytes"),WorkSpace);
  288 
  289         iSize = 0;
  290         return;
  291     }
  292 #endif
  293     // fallback to a static allocated space
  294     Base = calloc(WorkSpace, 1);
  295     iSize = 0;
  296 }
  297                                     /*}}}*/
  298 // DynamicMMap::~DynamicMMap - Destructor               /*{{{*/
  299 // ---------------------------------------------------------------------
  300 /* We truncate the file to the size of the memory data set */
  301 DynamicMMap::~DynamicMMap()
  302 {
  303    if (Fd == 0)
  304    {
  305       if (validData() == false)
  306      return;
  307 #ifdef _POSIX_MAPPED_FILES
  308       munmap(Base, WorkSpace);
  309 #else
  310       free(Base);
  311 #endif
  312       return;
  313    }
  314    
  315    unsigned long long EndOfFile = iSize;
  316    iSize = WorkSpace;
  317    Close(false);
  318    if(ftruncate(Fd->Fd(),EndOfFile) < 0)
  319       _error->Errno("ftruncate", _("Failed to truncate file"));
  320 }  
  321                                     /*}}}*/
  322 // DynamicMMap::RawAllocate - Allocate a raw chunk of unaligned space   /*{{{*/
  323 // ---------------------------------------------------------------------
  324 /* This allocates a block of memory aligned to the given size */
  325 unsigned long DynamicMMap::RawAllocate(unsigned long long Size,unsigned long Aln)
  326 {
  327    unsigned long long Result = iSize;
  328    if (Aln != 0)
  329       Result += Aln - (iSize%Aln);
  330 
  331    iSize = Result + Size;
  332 
  333    // try to grow the buffer
  334    while(Result + Size > WorkSpace)
  335    {
  336       if(!Grow())
  337       {
  338      _error->Fatal(_("Dynamic MMap ran out of room. Please increase the size "
  339              "of APT::Cache-Start. Current value: %lu. (man 5 apt.conf)"), WorkSpace);
  340      return 0;
  341       }
  342    }
  343    return Result;
  344 }
  345                                     /*}}}*/
  346 // DynamicMMap::Allocate - Pooled aligned allocation            /*{{{*/
  347 // ---------------------------------------------------------------------
  348 /* This allocates an Item of size ItemSize so that it is aligned to its
  349    size in the file. */
  350 unsigned long DynamicMMap::Allocate(unsigned long ItemSize)
  351 {
  352    if (unlikely(ItemSize == 0))
  353    {
  354       _error->Fatal("Can't allocate an item of size zero");
  355       return 0;
  356    }
  357 
  358    // Look for a matching pool entry
  359    Pool *I;
  360    Pool *Empty = 0;
  361    for (I = Pools; I != Pools + PoolCount; ++I)
  362    {
  363       if (I->ItemSize == 0)
  364      Empty = I;
  365       if (I->ItemSize == ItemSize)
  366      break;
  367    }
  368    // No pool is allocated, use an unallocated one
  369    if (I == Pools + PoolCount)
  370    {
  371       // Woops, we ran out, the calling code should allocate more.
  372       if (Empty == 0)
  373       {
  374      _error->Error("Ran out of allocation pools");
  375      return 0;
  376       }
  377       
  378       I = Empty;
  379       I->ItemSize = ItemSize;
  380       I->Count = 0;
  381    }
  382 
  383    unsigned long Result = 0;
  384    // Out of space, allocate some more
  385    if (I->Count == 0)
  386    {
  387       const unsigned long size = 20*1024;
  388       I->Count = size/ItemSize;
  389       Pool* oldPools = Pools;
  390       _error->PushToStack();
  391       Result = RawAllocate(size,ItemSize);
  392       bool const newError = _error->PendingError();
  393       _error->MergeWithStack();
  394       if (Pools != oldPools)
  395      I += Pools - oldPools;
  396 
  397       // Does the allocation failed ?
  398       if (Result == 0 && newError)
  399      return 0;
  400       I->Start = Result;
  401    }
  402    else
  403       Result = I->Start;
  404 
  405    I->Count--;
  406    I->Start += ItemSize;
  407    return Result/ItemSize;
  408 }
  409                                     /*}}}*/
  410 // DynamicMMap::WriteString - Write a string to the file        /*{{{*/
  411 // ---------------------------------------------------------------------
  412 /* Strings are aligned to 16 bytes */
  413 unsigned long DynamicMMap::WriteString(const char *String,
  414                        unsigned long Len)
  415 {
  416    if (Len == std::numeric_limits<unsigned long>::max())
  417       Len = strlen(String);
  418 
  419    _error->PushToStack();
  420    unsigned long Result = RawAllocate(Len+1+sizeof(uint16_t),sizeof(uint16_t));
  421    bool const newError = _error->PendingError();
  422    _error->MergeWithStack();
  423 
  424    if (Base == NULL || (Result == 0 && newError))
  425       return 0;
  426 
  427    if (Len >= std::numeric_limits<uint16_t>::max())
  428       abort();
  429 
  430    uint16_t LenToWrite = Len;
  431    memcpy((char *)Base + Result, &LenToWrite, sizeof(LenToWrite));
  432    Result += + sizeof(LenToWrite);
  433 
  434    memcpy((char *)Base + Result,String,Len);
  435    ((char *)Base)[Result + Len] = 0;
  436    return Result;
  437 }
  438                                     /*}}}*/
  439 // DynamicMMap::Grow - Grow the mmap                    /*{{{*/
  440 // ---------------------------------------------------------------------
  441 /* This method is a wrapper around different methods to (try to) grow
  442    a mmap (or our char[]-fallback). Encounterable environments:
  443    1. Moveable + !Fallback + linux -> mremap with MREMAP_MAYMOVE
  444    2. Moveable + !Fallback + !linux -> not possible (forbidden by constructor)
  445    3. Moveable + Fallback -> realloc
  446    4. !Moveable + !Fallback + linux -> mremap alone - which will fail in 99,9%
  447    5. !Moveable + !Fallback + !linux -> not possible (forbidden by constructor)
  448    6. !Moveable + Fallback -> not possible
  449    [ While Moveable and Fallback stands for the equally named flags and
  450      "linux" indicates a linux kernel instead of a freebsd kernel. ]
  451    So what you can see here is, that a MMAP which want to be growable need
  452    to be moveable to have a real chance but that this method will at least try
  453    the nearly impossible 4 to grow it before it finally give up: Never say never. */
  454 bool DynamicMMap::Grow() {
  455     if (Limit != 0 && WorkSpace >= Limit)
  456         return _error->Error(_("Unable to increase the size of the MMap as the "
  457                                "limit of %lu bytes is already reached."), Limit);
  458     if (GrowFactor <= 0)
  459         return _error->Error(_("Unable to increase size of the MMap as automatic growing is disabled by user."));
  460 
  461     unsigned long long const newSize = WorkSpace + GrowFactor;
  462 
  463     if(Fd != 0) {
  464         Fd->Seek(newSize - 1);
  465         char C = 0;
  466         Fd->Write(&C,sizeof(C));
  467     }
  468 
  469     unsigned long const poolOffset = Pools - ((Pool*) Base);
  470 
  471     if ((Flags & Fallback) != Fallback) {
  472 #if defined(_POSIX_MAPPED_FILES) && defined(__linux__)
  473    #ifdef MREMAP_MAYMOVE
  474 
  475         if ((Flags & Moveable) == Moveable)
  476             Base = mremap(Base, WorkSpace, newSize, MREMAP_MAYMOVE);
  477         else
  478    #endif
  479             Base = mremap(Base, WorkSpace, newSize, 0);
  480 
  481         if(Base == MAP_FAILED)
  482             return false;
  483 #else
  484         return false;
  485 #endif
  486     } else {
  487         if ((Flags & Moveable) != Moveable)
  488             return false;
  489 
  490         Base = realloc(Base, newSize);
  491         if (Base == NULL)
  492             return false;
  493         else
  494             /* Set new memory to 0 */
  495             memset((char*)Base + WorkSpace, 0, newSize - WorkSpace);
  496     }
  497 
  498     Pools =(Pool*) Base + poolOffset;
  499     WorkSpace = newSize;
  500     return true;
  501 }
  502                                     /*}}}*/