"Fossies" - the Fresh Open Source Software Archive

Member "apt-1.9.4/ftparchive/cachedb.cc" (19 Sep 2019, 16342 Bytes) of package /linux/misc/apt-1.9.4.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "cachedb.cc" see the Fossies "Dox" file reference documentation.

    1 // -*- mode: cpp; mode: fold -*-
    2 // Description                              /*{{{*/
    3 /* ######################################################################
    4 
    5    CacheDB
    6    
    7    Simple uniform interface to a cache database.
    8    
    9    ##################################################################### */
   10                                     /*}}}*/
   11 // Include Files                            /*{{{*/
   12 #include <config.h>
   13 
   14 #include <apt-pkg/configuration.h>
   15 #include <apt-pkg/debfile.h>
   16 #include <apt-pkg/error.h>
   17 #include <apt-pkg/fileutl.h>
   18 #include <apt-pkg/gpgv.h>
   19 #include <apt-pkg/hashes.h>
   20 #include <apt-pkg/md5.h>
   21 #include <apt-pkg/sha1.h>
   22 #include <apt-pkg/sha2.h>
   23 #include <apt-pkg/strutl.h>
   24 
   25 #include <ctype.h>
   26 #include <netinet/in.h> // htonl, etc
   27 #include <stddef.h>
   28 #include <strings.h>
   29 #include <sys/stat.h>
   30 
   31 #include "cachedb.h"
   32 
   33 #include <apti18n.h>
   34                                     /*}}}*/
   35 
   36 CacheDB::CacheDB(std::string const &DB)
   37    : Dbp(0), Fd(NULL), DebFile(0)
   38 {
   39    TmpKey[0]='\0';
   40    ReadyDB(DB);
   41 }
   42 
   43 CacheDB::~CacheDB()
   44 {
   45    ReadyDB();
   46    delete DebFile;
   47    CloseFile();
   48 }
   49 
   50 // CacheDB::ReadyDB - Ready the DB2                 /*{{{*/
   51 // ---------------------------------------------------------------------
   52 /* This opens the DB2 file for caching package information */
   53 bool CacheDB::ReadyDB(std::string const &DB)
   54 {
   55    int err;
   56 
   57    ReadOnly = _config->FindB("APT::FTPArchive::ReadOnlyDB",false);
   58    
   59    // Close the old DB
   60    if (Dbp != 0) 
   61       Dbp->close(Dbp,0);
   62    
   63    /* Check if the DB was disabled while running and deal with a 
   64       corrupted DB */
   65    if (DBFailed() == true)
   66    {
   67       _error->Warning(_("DB was corrupted, file renamed to %s.old"),DBFile.c_str());
   68       rename(DBFile.c_str(),(DBFile+".old").c_str());
   69    }
   70    
   71    DBLoaded = false;
   72    Dbp = 0;
   73    DBFile = std::string();
   74    
   75    if (DB.empty())
   76       return true;
   77 
   78    db_create(&Dbp, NULL, 0);
   79    if ((err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_BTREE,
   80                         (ReadOnly?DB_RDONLY:DB_CREATE),
   81                         0644)) != 0)
   82    {
   83       if (err == DB_OLD_VERSION)
   84       {
   85           _error->Warning(_("DB is old, attempting to upgrade %s"),DBFile.c_str());
   86       err = Dbp->upgrade(Dbp, DB.c_str(), 0);
   87       if (!err)
   88          err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_HASH,
   89                             (ReadOnly?DB_RDONLY:DB_CREATE), 0644);
   90 
   91       }
   92       // the database format has changed from DB_HASH to DB_BTREE in 
   93       // apt 0.6.44
   94       if (err == EINVAL)
   95       {
   96      _error->Error(_("DB format is invalid. If you upgraded from an older version of apt, please remove and re-create the database."));
   97       }
   98       if (err)
   99       {
  100           Dbp = 0;
  101           return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err));
  102       }
  103    }
  104 
  105    DBFile = DB;
  106    DBLoaded = true;
  107    return true;
  108 }
  109                                     /*}}}*/
  110 // CacheDB::OpenFile - Open the file                    /*{{{*/
  111 // ---------------------------------------------------------------------
  112 /* */
  113 bool CacheDB::OpenFile()
  114 {
  115    // always close existing file first
  116    CloseFile();
  117 
  118    // open a new file
  119    Fd = new FileFd(FileName,FileFd::ReadOnly);
  120    if (_error->PendingError() == true)
  121    {
  122       CloseFile();
  123       return false;
  124    }
  125    return true;
  126 }
  127                                     /*}}}*/
  128 // CacheDB::CloseFile - Close the file                  /*{{{*/
  129 void CacheDB::CloseFile()
  130 {
  131    if(Fd != NULL)
  132    {
  133       delete Fd;
  134       Fd = NULL;
  135    }
  136 }
  137                                     /*}}}*/
  138 // CacheDB::OpenDebFile - Open a debfile                /*{{{*/
  139 bool CacheDB::OpenDebFile()
  140 {
  141    // always close existing file first
  142    CloseDebFile();
  143 
  144    // first open the fd, then pass it to the debDebFile
  145    if(OpenFile() == false)
  146       return false;
  147    DebFile = new debDebFile(*Fd);
  148    if (_error->PendingError() == true)
  149       return false;
  150    return true;
  151 }
  152                                     /*}}}*/
  153 // CacheDB::CloseDebFile - Close a debfile again            /*{{{*/
  154 void CacheDB::CloseDebFile()
  155 {
  156    CloseFile();
  157 
  158    if(DebFile != NULL)
  159    {
  160       delete DebFile;
  161       DebFile = NULL;
  162    }
  163 }
  164                                     /*}}}*/
  165 // CacheDB::GetFileStat - Get stats from the file           /*{{{*/
  166 // ---------------------------------------------------------------------
  167 /* This gets the size from the database if it's there.  If we need
  168  * to look at the file, also get the mtime from the file. */
  169 bool CacheDB::GetFileStat(bool const &doStat)
  170 {
  171    if ((CurStat.Flags & FlSize) == FlSize && doStat == false)
  172       return true;
  173 
  174    /* Get it from the file. */
  175    if (OpenFile() == false)
  176       return false;
  177    
  178    // Stat the file
  179    struct stat St;
  180    if (fstat(Fd->Fd(),&St) != 0)
  181    {
  182       CloseFile();
  183       return _error->Errno("fstat",
  184                            _("Failed to stat %s"),FileName.c_str());
  185    }
  186    CurStat.FileSize = St.st_size;
  187    CurStat.mtime = htonl(St.st_mtime);
  188    CurStat.Flags |= FlSize;
  189    
  190    return true;
  191 }
  192                                     /*}}}*/
  193 // CacheDB::GetCurStatCompatOldFormat                       /*{{{*/
  194 // ---------------------------------------------------------------------
  195 /* Read the old (32bit FileSize) StateStore format from disk */
  196 bool CacheDB::GetCurStatCompatOldFormat()
  197 {
  198    InitQueryStats();
  199    Data.data = &CurStatOldFormat;
  200    Data.flags = DB_DBT_USERMEM;
  201    Data.ulen = sizeof(CurStatOldFormat);
  202    if (Get() == false)
  203    {
  204       CurStat.Flags = 0;
  205    } else {
  206       CurStat.Flags = CurStatOldFormat.Flags;
  207       CurStat.mtime = CurStatOldFormat.mtime;
  208       CurStat.FileSize = CurStatOldFormat.FileSize;
  209       memcpy(CurStat.MD5, CurStatOldFormat.MD5, sizeof(CurStat.MD5));
  210       memcpy(CurStat.SHA1, CurStatOldFormat.SHA1, sizeof(CurStat.SHA1));
  211       memcpy(CurStat.SHA256, CurStatOldFormat.SHA256, sizeof(CurStat.SHA256));
  212    }
  213    return true;
  214 }
  215                                     /*}}}*/
  216 // CacheDB::GetCurStatCompatOldFormat                       /*{{{*/
  217 // ---------------------------------------------------------------------
  218 /* Read the new (64bit FileSize) StateStore format from disk */
  219 bool CacheDB::GetCurStatCompatNewFormat()
  220 {
  221    InitQueryStats();
  222    Data.data = &CurStat;
  223    Data.flags = DB_DBT_USERMEM;
  224    Data.ulen = sizeof(CurStat);
  225    if (Get() == false)
  226    {
  227       CurStat.Flags = 0;
  228    }
  229    return true;
  230 }
  231                                     /*}}}*/
  232 // CacheDB::GetCurStat - Set the CurStat variable.          /*{{{*/
  233 // ---------------------------------------------------------------------
  234 /* Sets the CurStat variable.  Either to 0 if no database is used
  235  * or to the value in the database if one is used */
  236 bool CacheDB::GetCurStat()
  237 {
  238    memset(&CurStat,0,sizeof(CurStat));
  239    
  240    if (DBLoaded)
  241    {
  242       // do a first query to just get the size of the data on disk
  243       InitQueryStats();
  244       Data.data = &CurStat;
  245       Data.flags = DB_DBT_USERMEM;
  246       Data.ulen = 0;
  247       Get();
  248 
  249       if (Data.size == 0)
  250       {
  251          // nothing needs to be done, we just have not data for this deb
  252       }
  253       // check if the record is written in the old format (32bit filesize)
  254       else if(Data.size == sizeof(CurStatOldFormat))
  255       {
  256          GetCurStatCompatOldFormat();
  257       }
  258       else if(Data.size == sizeof(CurStat))
  259       {
  260          GetCurStatCompatNewFormat();
  261       } else {
  262          return _error->Error("Cache record size mismatch (%ul)", Data.size);
  263       }
  264 
  265       CurStat.Flags = ntohl(CurStat.Flags);
  266       CurStat.FileSize = ntohl(CurStat.FileSize);
  267    }      
  268    return true;
  269 }
  270                                     /*}}}*/
  271 // CacheDB::GetFileInfo - Get all the info about the file       /*{{{*/
  272 // ---------------------------------------------------------------------
  273 bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents,
  274                 bool const &GenContentsOnly, bool const DoSource, unsigned int const DoHashes,
  275                           bool const &checkMtime)
  276 {
  277    this->FileName = FileName;
  278 
  279    if (GetCurStat() == false)
  280       return false;
  281    OldStat = CurStat;
  282 
  283    if (GetFileStat(checkMtime) == false)
  284       return false;
  285 
  286    /* if mtime changed, update CurStat from disk */
  287    if (checkMtime == true && OldStat.mtime != CurStat.mtime)
  288       CurStat.Flags = FlSize;
  289 
  290    Stats.Bytes += CurStat.FileSize;
  291    ++Stats.Packages;
  292 
  293    if ((DoControl && LoadControl() == false)
  294      || (DoContents && LoadContents(GenContentsOnly) == false)
  295      || (DoSource && LoadSource() == false)
  296      || (DoHashes != 0 && GetHashes(false, DoHashes) == false)
  297       )
  298    {
  299       return false;
  300    }
  301 
  302    return true;
  303 }
  304                                     /*}}}*/
  305 bool CacheDB::LoadSource()                      /*{{{*/
  306 {
  307    // Try to read the control information out of the DB.
  308    if ((CurStat.Flags & FlSource) == FlSource)
  309    {
  310       // Lookup the control information
  311       InitQuerySource();
  312       if (Get() == true && Dsc.TakeDsc(Data.data, Data.size) == true)
  313       {
  314         return true;
  315       }
  316       CurStat.Flags &= ~FlSource;
  317    }
  318    if (OpenFile() == false)
  319       return false;
  320 
  321    Stats.Misses++;
  322    if (Dsc.Read(FileName) == false)
  323       return false;
  324 
  325    if (Dsc.Length == 0)
  326       return _error->Error(_("Failed to read .dsc"));
  327 
  328    // Write back the control information
  329    InitQuerySource();
  330    if (Put(Dsc.Data.c_str(), Dsc.Length) == true)
  331       CurStat.Flags |= FlSource;
  332 
  333    return true;
  334 }
  335                                     /*}}}*/
  336 // CacheDB::LoadControl - Load Control information          /*{{{*/
  337 // ---------------------------------------------------------------------
  338 /* */
  339 bool CacheDB::LoadControl()
  340 {
  341    // Try to read the control information out of the DB.
  342    if ((CurStat.Flags & FlControl) == FlControl)
  343    {
  344       // Lookup the control information
  345       InitQueryControl();
  346       if (Get() == true && Control.TakeControl(Data.data,Data.size) == true)
  347         return true;
  348       CurStat.Flags &= ~FlControl;
  349    }
  350    
  351    if(OpenDebFile() == false)
  352       return false;
  353    
  354    Stats.Misses++;
  355    if (Control.Read(*DebFile) == false)
  356       return false;
  357 
  358    if (Control.Control == 0)
  359       return _error->Error(_("Archive has no control record"));
  360    
  361    // Write back the control information
  362    InitQueryControl();
  363    if (Put(Control.Control,Control.Length) == true)
  364       CurStat.Flags |= FlControl;
  365    return true;
  366 }
  367                                     /*}}}*/
  368 // CacheDB::LoadContents - Load the File Listing            /*{{{*/
  369 // ---------------------------------------------------------------------
  370 /* */
  371 bool CacheDB::LoadContents(bool const &GenOnly)
  372 {
  373    // Try to read the control information out of the DB.
  374    if ((CurStat.Flags & FlContents) == FlContents)
  375    {
  376       if (GenOnly == true)
  377      return true;
  378       
  379       // Lookup the contents information
  380       InitQueryContent();
  381       if (Get() == true)
  382       {
  383      if (Contents.TakeContents(Data.data,Data.size) == true)
  384         return true;
  385       }
  386       
  387       CurStat.Flags &= ~FlContents;
  388    }
  389    
  390    if(OpenDebFile() == false)
  391       return false;
  392 
  393    Stats.Misses++;
  394    if (Contents.Read(*DebFile) == false)
  395       return false;     
  396    
  397    // Write back the control information
  398    InitQueryContent();
  399    if (Put(Contents.Data,Contents.CurSize) == true)
  400       CurStat.Flags |= FlContents;
  401    return true;
  402 }
  403                                     /*}}}*/
  404 // CacheDB::GetHashes - Get the hashes                  /*{{{*/
  405 static std::string bytes2hex(uint8_t *bytes, size_t length) {
  406    char buf[3];
  407    std::string space;
  408 
  409    space.reserve(length*2 + 1);
  410    for (size_t i = 0; i < length; i++) {
  411       snprintf(buf, sizeof(buf), "%02x", bytes[i]);
  412       space.append(buf);
  413    }
  414    return space;
  415 }
  416 
  417 static inline unsigned char xdig2num(char const &dig) {
  418    if (isdigit(dig)) return dig - '0';
  419    if ('a' <= dig && dig <= 'f') return dig - 'a' + 10;
  420    if ('A' <= dig && dig <= 'F') return dig - 'A' + 10;
  421    return 0;
  422 }
  423 
  424 static void hex2bytes(uint8_t *bytes, const char *hex, int length) {
  425    while (length-- > 0) {
  426       *bytes = 0;
  427       if (isxdigit(hex[0]) && isxdigit(hex[1])) {
  428       *bytes = xdig2num(hex[0]) * 16 + xdig2num(hex[1]);
  429       hex += 2;
  430       }
  431       bytes++;
  432    } 
  433 }
  434 bool CacheDB::GetHashes(bool const GenOnly, unsigned int const DoHashes)
  435 {
  436    unsigned int notCachedHashes = 0;
  437    if ((CurStat.Flags & FlMD5) != FlMD5)
  438    {
  439       notCachedHashes = notCachedHashes | Hashes::MD5SUM;
  440    }
  441    if ((CurStat.Flags & FlSHA1) != FlSHA1)
  442    {
  443       notCachedHashes = notCachedHashes | Hashes::SHA1SUM;
  444    }
  445    if ((CurStat.Flags & FlSHA256) != FlSHA256)
  446    {
  447       notCachedHashes = notCachedHashes | Hashes::SHA256SUM;
  448    }
  449    if ((CurStat.Flags & FlSHA512) != FlSHA512)
  450    {
  451       notCachedHashes = notCachedHashes | Hashes::SHA512SUM;
  452    }
  453    unsigned int FlHashes = DoHashes & notCachedHashes;
  454    HashesList.clear();
  455 
  456    if (FlHashes != 0)
  457    {
  458       if (OpenFile() == false)
  459      return false;
  460 
  461       Hashes hashes(FlHashes);
  462       if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize) == false)
  463      return false;
  464 
  465       HashStringList hl = hashes.GetHashStringList();
  466       for (HashStringList::const_iterator hs = hl.begin(); hs != hl.end(); ++hs)
  467       {
  468      HashesList.push_back(*hs);
  469      if (strcasecmp(hs->HashType().c_str(), "SHA512") == 0)
  470      {
  471         Stats.SHA512Bytes += CurStat.FileSize;
  472         hex2bytes(CurStat.SHA512, hs->HashValue().data(), sizeof(CurStat.SHA512));
  473         CurStat.Flags |= FlSHA512;
  474      }
  475      else if (strcasecmp(hs->HashType().c_str(), "SHA256") == 0)
  476      {
  477         Stats.SHA256Bytes += CurStat.FileSize;
  478         hex2bytes(CurStat.SHA256, hs->HashValue().data(), sizeof(CurStat.SHA256));
  479         CurStat.Flags |= FlSHA256;
  480      }
  481      else if (strcasecmp(hs->HashType().c_str(), "SHA1") == 0)
  482      {
  483         Stats.SHA1Bytes += CurStat.FileSize;
  484         hex2bytes(CurStat.SHA1, hs->HashValue().data(), sizeof(CurStat.SHA1));
  485         CurStat.Flags |= FlSHA1;
  486      }
  487      else if (strcasecmp(hs->HashType().c_str(), "MD5Sum") == 0)
  488      {
  489         Stats.MD5Bytes += CurStat.FileSize;
  490         hex2bytes(CurStat.MD5, hs->HashValue().data(), sizeof(CurStat.MD5));
  491         CurStat.Flags |= FlMD5;
  492      }
  493      else if (strcasecmp(hs->HashType().c_str(), "Checksum-FileSize") == 0)
  494      {
  495         // we store it in a different field already
  496      }
  497      else
  498         return _error->Error("Got unknown unrequested hashtype %s", hs->HashType().c_str());
  499       }
  500    }
  501    if (GenOnly == true)
  502       return true;
  503 
  504    bool ret = true;
  505 #define PUSH_BACK_HASH(FLAG, TYPE, VALUE) \
  506    if ((CurStat.Flags & FLAG) == FLAG) \
  507       ret &= HashesList.push_back(HashString(TYPE, bytes2hex(VALUE, sizeof(VALUE))));
  508    PUSH_BACK_HASH(FlMD5, "MD5Sum", CurStat.MD5);
  509    PUSH_BACK_HASH(FlSHA1, "SHA1", CurStat.SHA1);
  510    PUSH_BACK_HASH(FlSHA256, "SHA256", CurStat.SHA256);
  511    PUSH_BACK_HASH(FlSHA512, "SHA512", CurStat.SHA512);
  512    return ret;
  513 }
  514                                     /*}}}*/
  515 // CacheDB::Finish - Write back the cache structure         /*{{{*/
  516 // ---------------------------------------------------------------------
  517 /* */
  518 bool CacheDB::Finish()
  519 {
  520    // Optimize away some writes.
  521    if (CurStat.Flags == OldStat.Flags &&
  522        CurStat.mtime == OldStat.mtime)
  523       return true;
  524 
  525    // Write the stat information
  526    CurStat.Flags = htonl(CurStat.Flags);
  527    CurStat.FileSize = htonl(CurStat.FileSize);
  528    InitQueryStats();
  529    Put(&CurStat,sizeof(CurStat));
  530    CurStat.Flags = ntohl(CurStat.Flags);
  531    CurStat.FileSize = ntohl(CurStat.FileSize);
  532 
  533    return true;
  534 }
  535                                     /*}}}*/
  536 // CacheDB::Clean - Clean the Database                  /*{{{*/
  537 // ---------------------------------------------------------------------
  538 /* Tidy the database by removing files that no longer exist at all. */
  539 bool CacheDB::Clean()
  540 {
  541    if (DBLoaded == false)
  542       return true;
  543 
  544    /* I'm not sure what VERSION_MINOR should be here.. 2.4.14 certainly
  545       needs the lower one and 2.7.7 needs the upper.. */
  546    DBC *Cursor;
  547    if ((errno = Dbp->cursor(Dbp, NULL, &Cursor, 0)) != 0)
  548       return _error->Error(_("Unable to get a cursor"));
  549    
  550    DBT Key;
  551    DBT Data;
  552    memset(&Key,0,sizeof(Key));
  553    memset(&Data,0,sizeof(Data));
  554    while ((errno = Cursor->c_get(Cursor,&Key,&Data,DB_NEXT)) == 0)
  555    {
  556       const char *Colon = (char*)memrchr(Key.data, ':', Key.size);
  557       if (Colon)
  558       {
  559          if (stringcmp(Colon + 1, (char *)Key.data+Key.size,"st") == 0 ||
  560              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cl") == 0 ||
  561              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cs") == 0 ||
  562              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cn") == 0)
  563      {
  564             std::string FileName = std::string((const char *)Key.data,Colon);
  565             if (FileExists(FileName) == true) {
  566         continue;
  567             }
  568      }
  569       }
  570       Cursor->c_del(Cursor,0);
  571    }
  572    int res = Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL);
  573    if (res < 0)
  574       _error->Warning("compact failed with result %i", res);
  575 
  576    if(_config->FindB("Debug::APT::FTPArchive::Clean", false) == true)
  577       Dbp->stat_print(Dbp, 0);
  578 
  579 
  580    return true;
  581 }
  582                                     /*}}}*/