"Fossies" - the Fresh Open Source Software Archive

Member "apt-1.9.4/apt-pkg/acquire.cc" (19 Sep 2019, 46662 Bytes) of package /linux/misc/apt-1.9.4.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "acquire.cc" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 1.9.2_vs_1.9.3.

    1 // -*- mode: cpp; mode: fold -*-
    2 // Description                              /*{{{*/
    3 /* ######################################################################
    4 
    5    Acquire - File Acquiration
    6 
    7    The core element for the schedule system is the concept of a named
    8    queue. Each queue is unique and each queue has a name derived from the
    9    URI. The degree of paralization can be controlled by how the queue
   10    name is derived from the URI.
   11    
   12    ##################################################################### */
   13                                     /*}}}*/
   14 // Include Files                            /*{{{*/
   15 #include <config.h>
   16 
   17 #include <apt-pkg/acquire-item.h>
   18 #include <apt-pkg/acquire-worker.h>
   19 #include <apt-pkg/acquire.h>
   20 #include <apt-pkg/configuration.h>
   21 #include <apt-pkg/error.h>
   22 #include <apt-pkg/fileutl.h>
   23 #include <apt-pkg/strutl.h>
   24 
   25 #include <algorithm>
   26 #include <chrono>
   27 #include <iomanip>
   28 #include <iostream>
   29 #include <memory>
   30 #include <numeric>
   31 #include <sstream>
   32 #include <string>
   33 #include <vector>
   34 #include <cmath>
   35 
   36 #include <dirent.h>
   37 #include <errno.h>
   38 #include <fcntl.h>
   39 #include <grp.h>
   40 #include <pwd.h>
   41 #include <stdio.h>
   42 #include <stdlib.h>
   43 #include <string.h>
   44 #include <sys/select.h>
   45 #include <sys/stat.h>
   46 #include <sys/time.h>
   47 #include <unistd.h>
   48 
   49 #include <apti18n.h>
   50                                     /*}}}*/
   51 
   52 using namespace std;
   53 
   54 // Acquire::pkgAcquire - Constructor                    /*{{{*/
   55 // ---------------------------------------------------------------------
   56 /* We grab some runtime state from the configuration space */
   57 pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0),
   58                Debug(_config->FindB("Debug::pkgAcquire",false)),
   59                Running(false)
   60 {
   61    Initialize();
   62 }
   63 pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0),
   64                Configs(0), Log(NULL), ToFetch(0),
   65                Debug(_config->FindB("Debug::pkgAcquire",false)),
   66                Running(false)
   67 {
   68    Initialize();
   69    SetLog(Progress);
   70 }
   71 void pkgAcquire::Initialize()
   72 {
   73    string const Mode = _config->Find("Acquire::Queue-Mode","host");
   74    if (strcasecmp(Mode.c_str(),"host") == 0)
   75       QueueMode = QueueHost;
   76    if (strcasecmp(Mode.c_str(),"access") == 0)
   77       QueueMode = QueueAccess;
   78 }
   79                                     /*}}}*/
   80 // Acquire::GetLock - lock directory and prepare for action     /*{{{*/
   81 static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode)
   82 {
   83    if (_config->FindB("Debug::SetupAPTPartialDirectory::AssumeGood", false))
   84       return true;
   85    std::string const partial = parent + postfix;
   86    bool const partialExists = DirectoryExists(partial);
   87    if (partialExists == false)
   88    {
   89       mode_t const old_umask = umask(S_IWGRP | S_IWOTH);
   90       bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
   91         CreateAPTDirectoryIfNeeded(parent, partial) == false);
   92       umask(old_umask);
   93       if (creation_fail == true)
   94      return false;
   95    }
   96 
   97    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
   98    if (getuid() == 0)
   99    {
  100       if (SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
  101       {
  102      struct passwd const * const pw = getpwnam(SandboxUser.c_str());
  103      struct group const * const gr = getgrnam(ROOT_GROUP);
  104      if (pw != NULL && gr != NULL)
  105      {
  106         // chown the partial dir
  107         if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
  108            _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
  109      }
  110       }
  111       if (chmod(partial.c_str(), mode) != 0)
  112      _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
  113 
  114    }
  115    else if (chmod(partial.c_str(), mode) != 0)
  116    {
  117       // if we haven't created the dir and aren't root, it is kinda expected that chmod doesn't work
  118       if (partialExists == false)
  119      _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
  120    }
  121 
  122    _error->PushToStack();
  123    // remove 'old' FAILED files to stop us from collecting them for no reason
  124    for (auto const &Failed: GetListOfFilesInDir(partial, "FAILED", false, false))
  125       RemoveFile("SetupAPTPartialDirectory", Failed);
  126    _error->RevertToStack();
  127 
  128    return true;
  129 }
  130 bool pkgAcquire::GetLock(std::string const &Lock)
  131 {
  132    if (Lock.empty() == true)
  133       return false;
  134 
  135    // check for existence and possibly create auxiliary directories
  136    string const listDir = _config->FindDir("Dir::State::lists");
  137    string const archivesDir = _config->FindDir("Dir::Cache::Archives");
  138 
  139    if (Lock == listDir)
  140    {
  141       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false)
  142      return _error->Errno("Acquire", _("List directory %s is missing."), (listDir + "partial").c_str());
  143    }
  144    if (Lock == archivesDir)
  145    {
  146       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false)
  147      return _error->Errno("Acquire", _("Archives directory %s is missing."), (archivesDir + "partial").c_str());
  148    }
  149    if (Lock == listDir || Lock == archivesDir)
  150    {
  151       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
  152       {
  153      // not being able to create lists/auxfiles isn't critical as we will use a tmpdir then
  154       }
  155    }
  156 
  157    if (_config->FindB("Debug::NoLocking", false) == true)
  158       return true;
  159 
  160    // Lock the directory this acquire object will work in
  161    if (LockFD != -1)
  162       close(LockFD);
  163    LockFD = ::GetLock(flCombine(Lock, "lock"));
  164    if (LockFD == -1)
  165       return _error->Error(_("Unable to lock directory %s"), Lock.c_str());
  166 
  167    return true;
  168 }
  169                                     /*}}}*/
  170 // Acquire::~pkgAcquire - Destructor                    /*{{{*/
  171 // ---------------------------------------------------------------------
  172 /* Free our memory, clean up the queues (destroy the workers) */
  173 pkgAcquire::~pkgAcquire()
  174 {
  175    Shutdown();
  176 
  177    if (LockFD != -1)
  178       close(LockFD);
  179 
  180    while (Configs != 0)
  181    {
  182       MethodConfig *Jnk = Configs;
  183       Configs = Configs->Next;
  184       delete Jnk;
  185    }   
  186 }
  187                                     /*}}}*/
  188 // Acquire::Shutdown - Clean out the acquire object         /*{{{*/
  189 // ---------------------------------------------------------------------
  190 /* */
  191 void pkgAcquire::Shutdown()
  192 {
  193    while (Items.empty() == false)
  194    {
  195       if (Items[0]->Status == Item::StatFetching)
  196          Items[0]->Status = Item::StatError;
  197       delete Items[0];
  198    }
  199 
  200    while (Queues != 0)
  201    {
  202       Queue *Jnk = Queues;
  203       Queues = Queues->Next;
  204       delete Jnk;
  205    }   
  206 }
  207                                     /*}}}*/
  208 // Acquire::Add - Add a new item                    /*{{{*/
  209 // ---------------------------------------------------------------------
  210 /* This puts an item on the acquire list. This list is mainly for tracking
  211    item status */
  212 void pkgAcquire::Add(Item *Itm)
  213 {
  214    Items.push_back(Itm);
  215 }
  216                                     /*}}}*/
  217 // Acquire::Remove - Remove a item                  /*{{{*/
  218 // ---------------------------------------------------------------------
  219 /* Remove an item from the acquire list. This is usually not used.. */
  220 void pkgAcquire::Remove(Item *Itm)
  221 {
  222    Dequeue(Itm);
  223    
  224    for (ItemIterator I = Items.begin(); I != Items.end();)
  225    {
  226       if (*I == Itm)
  227       {
  228      Items.erase(I);
  229      I = Items.begin();
  230       }      
  231       else 
  232      ++I;
  233    }
  234 }
  235                                     /*}}}*/
  236 // Acquire::Add - Add a worker                      /*{{{*/
  237 // ---------------------------------------------------------------------
  238 /* A list of workers is kept so that the select loop can direct their FD
  239    usage. */
  240 void pkgAcquire::Add(Worker *Work)
  241 {
  242    Work->NextAcquire = Workers;
  243    Workers = Work;
  244 }
  245                                     /*}}}*/
  246 // Acquire::Remove - Remove a worker                    /*{{{*/
  247 // ---------------------------------------------------------------------
  248 /* A worker has died. This can not be done while the select loop is running
  249    as it would require that RunFds could handling a changing list state and
  250    it can't.. */
  251 void pkgAcquire::Remove(Worker *Work)
  252 {
  253    if (Running == true)
  254       abort();
  255    
  256    Worker **I = &Workers;
  257    for (; *I != 0;)
  258    {
  259       if (*I == Work)
  260      *I = (*I)->NextAcquire;
  261       else
  262      I = &(*I)->NextAcquire;
  263    }
  264 }
  265                                     /*}}}*/
  266 // Acquire::Enqueue - Queue an URI for fetching             /*{{{*/
  267 // ---------------------------------------------------------------------
  268 /* This is the entry point for an item. An item calls this function when
  269    it is constructed which creates a queue (based on the current queue
  270    mode) and puts the item in that queue. If the system is running then
  271    the queue might be started. */
  272 static bool CheckForBadItemAndFailIt(pkgAcquire::Item * const Item,
  273       pkgAcquire::MethodConfig const * const Config, pkgAcquireStatus * const Log)
  274 {
  275    auto SavedDesc = Item->GetItemDesc();
  276    if (Item->IsRedirectionLoop(SavedDesc.URI))
  277    {
  278       std::string const Message = "400 URI Failure"
  279      "\nURI: " + SavedDesc.URI +
  280      "\nFilename: " + Item->DestFile +
  281      "\nFailReason: RedirectionLoop";
  282 
  283       Item->Status = pkgAcquire::Item::StatError;
  284       Item->Failed(Message, Config);
  285       if (Log != nullptr)
  286      Log->Fail(SavedDesc);
  287       return true;
  288    }
  289 
  290    HashStringList const hsl = Item->GetExpectedHashes();
  291    if (hsl.usable() == false && Item->HashesRequired() &&
  292      _config->Exists("Acquire::ForceHash") == false)
  293    {
  294       std::string const Message = "400 URI Failure"
  295      "\nURI: " + SavedDesc.URI +
  296      "\nFilename: " + Item->DestFile +
  297      "\nFailReason: WeakHashSums";
  298 
  299       Item->Status = pkgAcquire::Item::StatAuthError;
  300       Item->Failed(Message, Config);
  301       if (Log != nullptr)
  302      Log->Fail(SavedDesc);
  303       return true;
  304    }
  305    return false;
  306 }
  307 void pkgAcquire::Enqueue(ItemDesc &Item)
  308 {
  309    // Determine which queue to put the item in
  310    const MethodConfig *Config = nullptr;
  311    string Name = QueueName(Item.URI,Config);
  312    if (Name.empty() == true)
  313    {
  314       Item.Owner->Status = pkgAcquire::Item::StatError;
  315       return;
  316    }
  317 
  318    /* the check for running avoids that we produce errors
  319       in logging before we actually have started, which would
  320       be easier to implement but would confuse users/implementations
  321       so we check the items skipped here in #Startup */
  322    if (Running && CheckForBadItemAndFailIt(Item.Owner, Config, Log))
  323       return;
  324 
  325    // Find the queue structure
  326    Queue *I = Queues;
  327    for (; I != 0 && I->Name != Name; I = I->Next);
  328    if (I == 0)
  329    {
  330       I = new Queue(Name,this);
  331       I->Next = Queues;
  332       Queues = I;
  333       
  334       if (Running == true)
  335      I->Startup();
  336    }
  337 
  338    // See if this is a local only URI
  339    if (Config->LocalOnly == true && Item.Owner->Complete == false)
  340       Item.Owner->Local = true;
  341    Item.Owner->Status = Item::StatIdle;
  342    
  343    // Queue it into the named queue
  344    if(I->Enqueue(Item)) 
  345       ToFetch++;
  346             
  347    // Some trace stuff
  348    if (Debug == true)
  349    {
  350       clog << "Fetching " << Item.URI << endl;
  351       clog << " to " << Item.Owner->DestFile << endl;
  352       clog << " Queue is: " << Name << endl;
  353    }
  354 }
  355                                     /*}}}*/
  356 // Acquire::Dequeue - Remove an item from all queues            /*{{{*/
  357 // ---------------------------------------------------------------------
  358 /* This is called when an item is finished being fetched. It removes it
  359    from all the queues */
  360 void pkgAcquire::Dequeue(Item *Itm)
  361 {
  362    Queue *I = Queues;
  363    bool Res = false;
  364    if (Debug == true)
  365       clog << "Dequeuing " << Itm->DestFile << endl;
  366 
  367    for (; I != 0; I = I->Next)
  368    {
  369       if (I->Dequeue(Itm))
  370       {
  371          Res = true;
  372      if (Debug == true)
  373         clog << "Dequeued from " << I->Name << endl;
  374       }
  375    }
  376 
  377    if (Res == true)
  378       ToFetch--;
  379 }
  380                                     /*}}}*/
  381 // Acquire::QueueName - Return the name of the queue for this URI   /*{{{*/
  382 // ---------------------------------------------------------------------
  383 /* The string returned depends on the configuration settings and the
  384    method parameters. Given something like http://foo.org/bar it can
  385    return http://foo.org or http */
  386 string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
  387 {
  388    constexpr int DEFAULT_HOST_LIMIT = 10;
  389    URI U(Uri);
  390 
  391    // Note that this gets written through the reference to the caller.
  392    Config = GetConfig(U.Access);
  393    if (Config == nullptr)
  394       return {};
  395 
  396    // Access mode forces all methods to be Single-Instance
  397    if (QueueMode == QueueAccess)
  398       return U.Access;
  399 
  400    // Single-Instance methods get exactly one queue per URI
  401    if (Config->SingleInstance == true)
  402       return U.Access;
  403 
  404    // Host-less methods like rred, store, …
  405    if (U.Host.empty())
  406    {
  407       int existing = 0;
  408       // check how many queues exist already and reuse empty ones
  409       auto const AccessSchema = U.Access + ':';
  410       for (Queue const *I = Queues; I != 0; I = I->Next)
  411      if (APT::String::Startswith(I->Name, AccessSchema))
  412      {
  413         if (I->Items == nullptr)
  414            return I->Name;
  415         ++existing;
  416      }
  417 
  418       int const Limit = _config->FindI("Acquire::QueueHost::Limit",
  419 #ifdef _SC_NPROCESSORS_ONLN
  420         sysconf(_SC_NPROCESSORS_ONLN) * 2
  421 #else
  422         DEFAULT_HOST_LIMIT
  423 #endif
  424       );
  425 
  426       // create a new worker if we don't have too many yet
  427       if (Limit <= 0 || existing < Limit)
  428      return AccessSchema + std::to_string(existing);
  429 
  430       // find the worker with the least to do
  431       // we already established that there are no empty and we can't spawn new
  432       Queue const *selected = nullptr;
  433       auto selected_backlog = std::numeric_limits<decltype(HashStringList().FileSize())>::max();
  434       for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next)
  435      if (APT::String::Startswith(Q->Name, AccessSchema))
  436      {
  437         decltype(selected_backlog) current_backlog = 0;
  438         for (auto const *I = Q->Items; I != nullptr; I = I->Next)
  439         {
  440            auto const hashes = I->Owner->GetExpectedHashes();
  441            if (not hashes.empty())
  442           current_backlog += hashes.FileSize();
  443            else
  444           current_backlog += I->Owner->FileSize;
  445         }
  446         if (current_backlog < selected_backlog)
  447         {
  448            selected = Q;
  449            selected_backlog = current_backlog;
  450         }
  451      }
  452 
  453       if (unlikely(selected == nullptr))
  454      return AccessSchema + "0";
  455       return selected->Name;
  456    }
  457    // most methods talking to remotes like http
  458    else
  459    {
  460       auto const FullQueueName = U.Access + ':' + U.Host;
  461       // if the queue already exists, re-use it
  462       for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next)
  463      if (Q->Name == FullQueueName)
  464         return FullQueueName;
  465 
  466       int existing = 0;
  467       // check how many queues exist already and reuse empty ones
  468       auto const AccessSchema = U.Access + ':';
  469       for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next)
  470      if (APT::String::Startswith(Q->Name, AccessSchema))
  471         ++existing;
  472 
  473       int const Limit = _config->FindI("Acquire::QueueHost::Limit", DEFAULT_HOST_LIMIT);
  474       // if we have too many hosts open use a single generic for the rest
  475       if (existing >= Limit)
  476      return U.Access;
  477 
  478       // we can still create new named queues
  479       return FullQueueName;
  480    }
  481 }
  482                                     /*}}}*/
  483 // Acquire::GetConfig - Fetch the configuration information     /*{{{*/
  484 // ---------------------------------------------------------------------
  485 /* This locates the configuration structure for an access method. If 
  486    a config structure cannot be found a Worker will be created to
  487    retrieve it */
  488 pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
  489 {
  490    // Search for an existing config
  491    MethodConfig *Conf;
  492    for (Conf = Configs; Conf != 0; Conf = Conf->Next)
  493       if (Conf->Access == Access)
  494      return Conf;
  495    
  496    // Create the new config class
  497    Conf = new MethodConfig;
  498    Conf->Access = Access;
  499 
  500    // Create the worker to fetch the configuration
  501    Worker Work(Conf);
  502    if (Work.Start() == false)
  503    {
  504       delete Conf;
  505       return nullptr;
  506    }
  507    Conf->Next = Configs;
  508    Configs = Conf;
  509 
  510    /* if a method uses DownloadLimit, we switch to SingleInstance mode */
  511    if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
  512       Conf->SingleInstance = true;
  513     
  514    return Conf;
  515 }
  516                                     /*}}}*/
  517 // Acquire::SetFds - Deal with readable FDs             /*{{{*/
  518 // ---------------------------------------------------------------------
  519 /* Collect FDs that have activity monitors into the fd sets */
  520 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
  521 {
  522    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
  523    {
  524       if (I->InReady == true && I->InFd >= 0)
  525       {
  526      if (Fd < I->InFd)
  527         Fd = I->InFd;
  528      FD_SET(I->InFd,RSet);
  529       }
  530       if (I->OutReady == true && I->OutFd >= 0)
  531       {
  532      if (Fd < I->OutFd)
  533         Fd = I->OutFd;
  534      FD_SET(I->OutFd,WSet);
  535       }
  536    }
  537 }
  538                                     /*}}}*/
  539 // Acquire::RunFds - compatibility remove on next abi/api break     /*{{{*/
  540 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
  541 {
  542    RunFdsSane(RSet, WSet);
  543 }
  544                                     /*}}}*/
  545 // Acquire::RunFdsSane - Deal with active FDs               /*{{{*/
  546 // ---------------------------------------------------------------------
  547 /* Dispatch active FDs over to the proper workers. It is very important
  548    that a worker never be erased while this is running! The queue class
  549    should never erase a worker except during shutdown processing. */
  550 bool pkgAcquire::RunFdsSane(fd_set *RSet,fd_set *WSet)
  551 {
  552    bool Res = true;
  553 
  554    for (Worker *I = Workers; I != 0; I = I->NextAcquire)
  555    {
  556       if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
  557      Res &= I->InFdReady();
  558       if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
  559      Res &= I->OutFdReady();
  560    }
  561 
  562    return Res;
  563 }
  564                                     /*}}}*/
  565 // Acquire::Run - Run the fetch sequence                /*{{{*/
  566 // ---------------------------------------------------------------------
  567 /* This runs the queues. It manages a select loop for all of the
  568    Worker tasks. The workers interact with the queues and items to
  569    manage the actual fetch. */
  570 static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite)
  571 {
  572    // you would think this is easily to answer with faccessat, right? Wrong!
  573    // It e.g. gets groups wrong, so the only thing which works reliable is trying
  574    // to open the file we want to open later on…
  575    if (unlikely(filename.empty()))
  576       return true;
  577 
  578    if (ReadWrite == false)
  579    {
  580       errno = 0;
  581       // can we read a file? Note that non-existing files are "fine"
  582       int const fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC);
  583       if (fd == -1 && errno == EACCES)
  584      return false;
  585       close(fd);
  586       return true;
  587    }
  588    else
  589    {
  590       // the file might not exist yet and even if it does we will fix permissions,
  591       // so important is here just that the directory it is in allows that
  592       std::string const dirname = flNotFile(filename);
  593       if (unlikely(dirname.empty()))
  594      return true;
  595 
  596       char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
  597       std::string const tmpfile_tpl = flCombine(dirname, filetag);
  598       std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
  599       int const fd = mkstemp(tmpfile.get());
  600       if (fd == -1 && errno == EACCES)
  601      return false;
  602       RemoveFile("IsAccessibleBySandboxUser", tmpfile.get());
  603       close(fd);
  604       return true;
  605    }
  606 }
  607 static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
  608 {
  609    if(getuid() != 0)
  610       return;
  611 
  612    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
  613    if (SandboxUser.empty() || SandboxUser == "root")
  614       return;
  615 
  616    struct passwd const * const pw = getpwnam(SandboxUser.c_str());
  617    if (pw == NULL)
  618    {
  619       _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str());
  620       _config->Set("APT::Sandbox::User", "");
  621       return;
  622    }
  623 
  624    gid_t const old_euid = geteuid();
  625    gid_t const old_egid = getegid();
  626 
  627    long const ngroups_max = sysconf(_SC_NGROUPS_MAX);
  628    std::unique_ptr<gid_t[]> old_gidlist(new gid_t[ngroups_max]);
  629    if (unlikely(old_gidlist == NULL))
  630       return;
  631    ssize_t old_gidlist_nr;
  632    if ((old_gidlist_nr = getgroups(ngroups_max, old_gidlist.get())) < 0)
  633    {
  634       _error->FatalE("getgroups", "getgroups %lu failed", ngroups_max);
  635       old_gidlist[0] = 0;
  636       old_gidlist_nr = 1;
  637    }
  638    if (setgroups(1, &pw->pw_gid))
  639       _error->FatalE("setgroups", "setgroups %u failed", pw->pw_gid);
  640 
  641    if (setegid(pw->pw_gid) != 0)
  642       _error->FatalE("setegid", "setegid %u failed", pw->pw_gid);
  643    if (seteuid(pw->pw_uid) != 0)
  644       _error->FatalE("seteuid", "seteuid %u failed", pw->pw_uid);
  645 
  646    for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin();
  647     I != Fetcher.ItemsEnd(); ++I)
  648    {
  649       // no need to drop privileges for a complete file
  650       if ((*I)->Complete == true || (*I)->Status != pkgAcquire::Item::StatIdle)
  651      continue;
  652 
  653       // if destination file is inaccessible all hope is lost for privilege dropping
  654       if (IsAccessibleBySandboxUser((*I)->DestFile, true) == false)
  655       {
  656      _error->WarningE("pkgAcquire::Run", _("Download is performed unsandboxed as root as file '%s' couldn't be accessed by user '%s'."),
  657            (*I)->DestFile.c_str(), SandboxUser.c_str());
  658      _config->Set("APT::Sandbox::User", "");
  659      break;
  660       }
  661 
  662       // if its the source file (e.g. local sources) we might be lucky
  663       // by dropping the dropping only for some methods.
  664       URI const source((*I)->DescURI());
  665       if (source.Access == "file" || source.Access == "copy")
  666       {
  667      std::string const conf = "Binary::" + source.Access + "::APT::Sandbox::User";
  668      if (_config->Exists(conf) == true)
  669         continue;
  670 
  671      if (IsAccessibleBySandboxUser(source.Path, false) == false)
  672      {
  673         _error->NoticeE("pkgAcquire::Run", _("Download is performed unsandboxed as root as file '%s' couldn't be accessed by user '%s'."),
  674           source.Path.c_str(), SandboxUser.c_str());
  675         _config->CndSet("Binary::file::APT::Sandbox::User", "root");
  676         _config->CndSet("Binary::copy::APT::Sandbox::User", "root");
  677      }
  678       }
  679    }
  680 
  681    if (seteuid(old_euid) != 0)
  682       _error->FatalE("seteuid", "seteuid %u failed", old_euid);
  683    if (setegid(old_egid) != 0)
  684       _error->FatalE("setegid", "setegid %u failed", old_egid);
  685    if (setgroups(old_gidlist_nr, old_gidlist.get()))
  686       _error->FatalE("setgroups", "setgroups %u failed", 0);
  687 }
  688 pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall)
  689 {
  690    _error->PushToStack();
  691    CheckDropPrivsMustBeDisabled(*this);
  692 
  693    Running = true;
  694    
  695    for (Queue *I = Queues; I != 0; I = I->Next)
  696       I->Startup();
  697    
  698    if (Log != 0)
  699       Log->Start();
  700    
  701    bool WasCancelled = false;
  702 
  703    // Run till all things have been acquired
  704    struct timeval tv;
  705    tv.tv_sec = 0;
  706    tv.tv_usec = PulseIntervall; 
  707    while (ToFetch > 0)
  708    {
  709       fd_set RFds;
  710       fd_set WFds;
  711       int Highest = 0;
  712       FD_ZERO(&RFds);
  713       FD_ZERO(&WFds);
  714       SetFds(Highest,&RFds,&WFds);
  715       
  716       int Res;
  717       do
  718       {
  719      Res = select(Highest+1,&RFds,&WFds,0,&tv);
  720       }
  721       while (Res < 0 && errno == EINTR);
  722       
  723       if (Res < 0)
  724       {
  725      _error->Errno("select","Select has failed");
  726      break;
  727       }
  728 
  729       if(RunFdsSane(&RFds,&WFds) == false)
  730          break;
  731 
  732       // Timeout, notify the log class
  733       if (Res == 0 || (Log != 0 && Log->Update == true))
  734       {
  735      tv.tv_usec = PulseIntervall;
  736      for (Worker *I = Workers; I != 0; I = I->NextAcquire)
  737         I->Pulse();
  738      if (Log != 0 && Log->Pulse(this) == false)
  739      {
  740         WasCancelled = true;
  741         break;
  742      }
  743       }      
  744    }   
  745 
  746    if (Log != 0)
  747       Log->Stop();
  748    
  749    // Shut down the acquire bits
  750    Running = false;
  751    for (Queue *I = Queues; I != 0; I = I->Next)
  752       I->Shutdown(false);
  753 
  754    // Shut down the items
  755    for (ItemIterator I = Items.begin(); I != Items.end(); ++I)
  756       (*I)->Finished();
  757 
  758    bool const newError = _error->PendingError();
  759    _error->MergeWithStack();
  760    if (newError)
  761       return Failed;
  762    if (WasCancelled)
  763       return Cancelled;
  764    return Continue;
  765 }
  766                                     /*}}}*/
  767 // Acquire::Bump - Called when an item is dequeued          /*{{{*/
  768 // ---------------------------------------------------------------------
  769 /* This routine bumps idle queues in hopes that they will be able to fetch
  770    the dequeued item */
  771 void pkgAcquire::Bump()
  772 {
  773    for (Queue *I = Queues; I != 0; I = I->Next)
  774       I->Bump();
  775 }
  776                                     /*}}}*/
  777 // Acquire::WorkerStep - Step to the next worker            /*{{{*/
  778 // ---------------------------------------------------------------------
  779 /* Not inlined to advoid including acquire-worker.h */
  780 pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
  781 {
  782    return I->NextAcquire;
  783 }
  784                                     /*}}}*/
  785 // Acquire::Clean - Cleans a directory                  /*{{{*/
  786 // ---------------------------------------------------------------------
  787 /* This is a bit simplistic, it looks at every file in the dir and sees
  788    if it is part of the download set. */
  789 bool pkgAcquire::Clean(string Dir)
  790 {
  791    // non-existing directories are by definition clean…
  792    if (DirectoryExists(Dir) == false)
  793       return true;
  794 
  795    if(Dir == "/")
  796       return _error->Error(_("Clean of %s is not supported"), Dir.c_str());
  797 
  798    int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
  799    if (dirfd == -1)
  800       return _error->Errno("open",_("Unable to read %s"),Dir.c_str());
  801    DIR * const D = fdopendir(dirfd);
  802    if (D == nullptr)
  803       return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
  804 
  805    for (struct dirent *E = readdir(D); E != nullptr; E = readdir(D))
  806    {
  807       // Skip some entries
  808       if (strcmp(E->d_name, "lock") == 0 ||
  809       strcmp(E->d_name, "partial") == 0 ||
  810       strcmp(E->d_name, "auxfiles") == 0 ||
  811       strcmp(E->d_name, "lost+found") == 0 ||
  812       strcmp(E->d_name, ".") == 0 ||
  813       strcmp(E->d_name, "..") == 0)
  814      continue;
  815 
  816       // Look in the get list and if not found nuke
  817       if (std::any_of(Items.cbegin(), Items.cend(),
  818          [&E](pkgAcquire::Item const * const I) {
  819         return flNotDir(I->DestFile) == E->d_name;
  820          }) == false)
  821       {
  822      RemoveFileAt("pkgAcquire::Clean", dirfd, E->d_name);
  823       }
  824    }
  825    closedir(D);
  826    return true;
  827 }
  828                                     /*}}}*/
  829 // Acquire::TotalNeeded - Number of bytes to fetch          /*{{{*/
  830 // ---------------------------------------------------------------------
  831 /* This is the total number of bytes needed */
  832 APT_PURE unsigned long long pkgAcquire::TotalNeeded()
  833 {
  834    return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
  835       [](unsigned long long const T, Item const * const I) {
  836      return T + I->FileSize;
  837    });
  838 }
  839                                     /*}}}*/
  840 // Acquire::FetchNeeded - Number of bytes needed to get         /*{{{*/
  841 // ---------------------------------------------------------------------
  842 /* This is the number of bytes that is not local */
  843 APT_PURE unsigned long long pkgAcquire::FetchNeeded()
  844 {
  845    return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
  846       [](unsigned long long const T, Item const * const I) {
  847      if (I->Local == false)
  848         return T + I->FileSize;
  849      else
  850         return T;
  851    });
  852 }
  853                                     /*}}}*/
  854 // Acquire::PartialPresent - Number of partial bytes we already have    /*{{{*/
  855 // ---------------------------------------------------------------------
  856 /* This is the number of bytes that is not local */
  857 APT_PURE unsigned long long pkgAcquire::PartialPresent()
  858 {
  859    return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
  860       [](unsigned long long const T, Item const * const I) {
  861      if (I->Local == false)
  862         return T + I->PartialSize;
  863      else
  864         return T;
  865    });
  866 }
  867                                     /*}}}*/
  868 // Acquire::UriBegin - Start iterator for the uri list          /*{{{*/
  869 // ---------------------------------------------------------------------
  870 /* */
  871 pkgAcquire::UriIterator pkgAcquire::UriBegin()
  872 {
  873    return UriIterator(Queues);
  874 }
  875                                     /*}}}*/
  876 // Acquire::UriEnd - End iterator for the uri list          /*{{{*/
  877 // ---------------------------------------------------------------------
  878 /* */
  879 pkgAcquire::UriIterator pkgAcquire::UriEnd()
  880 {
  881    return UriIterator(0);
  882 }
  883                                     /*}}}*/
  884 // Acquire::MethodConfig::MethodConfig - Constructor            /*{{{*/
  885 class pkgAcquire::MethodConfig::Private
  886 {
  887    public:
  888    bool AuxRequests = false;
  889 };
  890 pkgAcquire::MethodConfig::MethodConfig() : d(new Private()), Next(0), SingleInstance(false),
  891                        Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
  892                        Removable(false)
  893 {
  894 }
  895                                     /*}}}*/
  896 bool pkgAcquire::MethodConfig::GetAuxRequests() const /*{{{*/
  897 {
  898    return d->AuxRequests;
  899 }
  900                                     /*}}}*/
  901 void pkgAcquire::MethodConfig::SetAuxRequests(bool const value) /*{{{*/
  902 {
  903    d->AuxRequests = value;
  904 }
  905                                     /*}}}*/
  906 // Queue::Queue - Constructor                       /*{{{*/
  907 // ---------------------------------------------------------------------
  908 /* */
  909 pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0),
  910    Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1)
  911 {
  912 }
  913                                     /*}}}*/
  914 // Queue::~Queue - Destructor                       /*{{{*/
  915 // ---------------------------------------------------------------------
  916 /* */
  917 pkgAcquire::Queue::~Queue()
  918 {
  919    Shutdown(true);
  920    
  921    while (Items != 0)
  922    {
  923       QItem *Jnk = Items;
  924       Items = Items->Next;
  925       delete Jnk;
  926    }
  927 }
  928                                     /*}}}*/
  929 // Queue::Enqueue - Queue an item to the queue              /*{{{*/
  930 // ---------------------------------------------------------------------
  931 /* */
  932 bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
  933 {
  934    // MetaKeysMatch checks whether the two items have no non-matching
  935    // meta-keys. If the items are not transaction items, it returns
  936    // true, so other items can still be merged.
  937    auto MetaKeysMatch = [](pkgAcquire::ItemDesc const &A, pkgAcquire::Queue::QItem const *B) {
  938       auto OwnerA = dynamic_cast<pkgAcqTransactionItem*>(A.Owner);
  939       if (OwnerA == nullptr)
  940      return true;
  941 
  942       for (auto const & OwnerBUncast : B->Owners) {
  943      auto OwnerB = dynamic_cast<pkgAcqTransactionItem*>(OwnerBUncast);
  944 
  945      if (OwnerB != nullptr && OwnerA->GetMetaKey() != OwnerB->GetMetaKey())
  946         return false;
  947       }
  948       return true;
  949    };
  950    QItem **OptimalI = &Items;
  951    QItem **I = &Items;
  952    // move to the end of the queue and check for duplicates here
  953    for (; *I != 0; ) {
  954       if (Item.URI == (*I)->URI && MetaKeysMatch(Item, *I))
  955       {
  956      if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
  957         std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
  958      (*I)->Owners.push_back(Item.Owner);
  959      Item.Owner->Status = (*I)->Owner->Status;
  960      return false;
  961       }
  962       // Determine the optimal position to insert: before anything with a
  963       // higher priority.
  964       int priority = (*I)->GetPriority();
  965 
  966       I = &(*I)->Next;
  967       if (priority >= Item.Owner->Priority()) {
  968      OptimalI = I;
  969       }
  970    }
  971 
  972 
  973    // Create a new item
  974    QItem *Itm = new QItem;
  975    *Itm = Item;
  976    Itm->Next = *OptimalI;
  977    *OptimalI = Itm;
  978    
  979    Item.Owner->QueueCounter++;   
  980    if (Items->Next == 0)
  981       Cycle();
  982    return true;
  983 }
  984                                     /*}}}*/
  985 // Queue::Dequeue - Remove an item from the queue           /*{{{*/
  986 // ---------------------------------------------------------------------
  987 /* We return true if we hit something */
  988 bool pkgAcquire::Queue::Dequeue(Item *Owner)
  989 {
  990    if (Owner->Status == pkgAcquire::Item::StatFetching)
  991       return _error->Error("Tried to dequeue a fetching object");
  992 
  993    bool Res = false;
  994 
  995    QItem **I = &Items;
  996    for (; *I != 0;)
  997    {
  998       if (Owner == (*I)->Owner)
  999       {
 1000      QItem *Jnk= *I;
 1001      *I = (*I)->Next;
 1002      Owner->QueueCounter--;
 1003      delete Jnk;
 1004      Res = true;
 1005       }
 1006       else
 1007      I = &(*I)->Next;
 1008    }
 1009 
 1010    return Res;
 1011 }
 1012                                     /*}}}*/
 1013 // Queue::Startup - Start the worker processes              /*{{{*/
 1014 // ---------------------------------------------------------------------
 1015 /* It is possible for this to be called with a pre-existing set of
 1016    workers. */
 1017 bool pkgAcquire::Queue::Startup()
 1018 {
 1019    if (Workers == 0)
 1020    {
 1021       URI U(Name);
 1022       pkgAcquire::MethodConfig * const Cnf = Owner->GetConfig(U.Access);
 1023       if (unlikely(Cnf == nullptr))
 1024      return false;
 1025 
 1026       // now-running twin of the pkgAcquire::Enqueue call
 1027       for (QItem *I = Items; I != 0; )
 1028       {
 1029      auto const INext = I->Next;
 1030      for (auto &&O: I->Owners)
 1031         CheckForBadItemAndFailIt(O, Cnf, Owner->Log);
 1032      // if an item failed, it will be auto-dequeued invalidation our I here
 1033      I = INext;
 1034       }
 1035 
 1036       Workers = new Worker(this,Cnf,Owner->Log);
 1037       Owner->Add(Workers);
 1038       if (Workers->Start() == false)
 1039      return false;
 1040       
 1041       /* When pipelining we commit 10 items. This needs to change when we
 1042          added other source retry to have cycle maintain a pipeline depth
 1043          on its own. */
 1044       if (Cnf->Pipeline == true)
 1045      MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10);
 1046       else
 1047      MaxPipeDepth = 1;
 1048    }
 1049    
 1050    return Cycle();
 1051 }
 1052                                     /*}}}*/
 1053 // Queue::Shutdown - Shutdown the worker processes          /*{{{*/
 1054 // ---------------------------------------------------------------------
 1055 /* If final is true then all workers are eliminated, otherwise only workers
 1056    that do not need cleanup are removed */
 1057 bool pkgAcquire::Queue::Shutdown(bool Final)
 1058 {
 1059    // Delete all of the workers
 1060    pkgAcquire::Worker **Cur = &Workers;
 1061    while (*Cur != 0)
 1062    {
 1063       pkgAcquire::Worker *Jnk = *Cur;
 1064       if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
 1065       {
 1066      *Cur = Jnk->NextQueue;
 1067      Owner->Remove(Jnk);
 1068      delete Jnk;
 1069       }
 1070       else
 1071      Cur = &(*Cur)->NextQueue;      
 1072    }
 1073    
 1074    return true;
 1075 }
 1076                                     /*}}}*/
 1077 // Queue::FindItem - Find a URI in the item list            /*{{{*/
 1078 // ---------------------------------------------------------------------
 1079 /* */
 1080 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
 1081 {
 1082    for (QItem *I = Items; I != 0; I = I->Next)
 1083       if (I->URI == URI && I->Worker == Owner)
 1084      return I;
 1085    return 0;
 1086 }
 1087                                     /*}}}*/
 1088 // Queue::ItemDone - Item has been completed                /*{{{*/
 1089 // ---------------------------------------------------------------------
 1090 /* The worker signals this which causes the item to be removed from the
 1091    queue. If this is the last queue instance then it is removed from the
 1092    main queue too.*/
 1093 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
 1094 {
 1095    PipeDepth--;
 1096    for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
 1097    {
 1098       if ((*O)->Status == pkgAcquire::Item::StatFetching)
 1099      (*O)->Status = pkgAcquire::Item::StatDone;
 1100    }
 1101 
 1102    if (Itm->Owner->QueueCounter <= 1)
 1103       Owner->Dequeue(Itm->Owner);
 1104    else
 1105    {
 1106       Dequeue(Itm->Owner);
 1107       Owner->Bump();
 1108    }
 1109 
 1110    return Cycle();
 1111 }
 1112                                     /*}}}*/
 1113 // Queue::Cycle - Queue new items into the method           /*{{{*/
 1114 // ---------------------------------------------------------------------
 1115 /* This locates a new idle item and sends it to the worker. If pipelining
 1116    is enabled then it keeps the pipe full. */
 1117 bool pkgAcquire::Queue::Cycle()
 1118 {
 1119    if (Items == 0 || Workers == 0)
 1120       return true;
 1121 
 1122    if (PipeDepth < 0)
 1123       return _error->Error("Pipedepth failure");
 1124 
 1125    // Look for a queable item
 1126    QItem *I = Items;
 1127    int ActivePriority = 0;
 1128    while (PipeDepth < static_cast<decltype(PipeDepth)>(MaxPipeDepth))
 1129    {
 1130       for (; I != 0; I = I->Next) {
 1131      if (I->Owner->Status == pkgAcquire::Item::StatFetching)
 1132         ActivePriority = std::max(ActivePriority, I->GetPriority());
 1133      if (I->Owner->Status == pkgAcquire::Item::StatIdle)
 1134         break;
 1135       }
 1136 
 1137       // Nothing to do, queue is idle.
 1138       if (I == 0)
 1139      return true;
 1140 
 1141       // This item has a lower priority than stuff in the pipeline, pretend
 1142       // the queue is idle
 1143       if (I->GetPriority() < ActivePriority)
 1144      return true;
 1145       I->Worker = Workers;
 1146       for (auto const &O: I->Owners)
 1147      O->Status = pkgAcquire::Item::StatFetching;
 1148       PipeDepth++;
 1149       if (Workers->QueueItem(I) == false)
 1150      return false;
 1151    }
 1152 
 1153    return true;
 1154 }
 1155                                     /*}}}*/
 1156 // Queue::Bump - Fetch any pending objects if we are idle       /*{{{*/
 1157 // ---------------------------------------------------------------------
 1158 /* This is called when an item in multiple queues is dequeued */
 1159 void pkgAcquire::Queue::Bump()
 1160 {
 1161    Cycle();
 1162 }
 1163                                     /*}}}*/
 1164 HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const  /*{{{*/
 1165 {
 1166    /* each Item can have multiple owners and each owner might have different
 1167       hashes, even if that is unlikely in practice and if so at least some
 1168       owners will later fail. There is one situation through which is not a
 1169       failure and still needs this handling: Two owners who expect the same
 1170       file, but one owner only knows the SHA1 while the other only knows SHA256. */
 1171    HashStringList superhsl;
 1172    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
 1173    {
 1174       HashStringList const hsl = (*O)->GetExpectedHashes();
 1175       // we merge both lists - if we find disagreement send no hashes
 1176       HashStringList::const_iterator hs = hsl.begin();
 1177       for (; hs != hsl.end(); ++hs)
 1178      if (superhsl.push_back(*hs) == false)
 1179         break;
 1180       if (hs != hsl.end())
 1181       {
 1182      superhsl.clear();
 1183      break;
 1184       }
 1185    }
 1186    return superhsl;
 1187 }
 1188                                     /*}}}*/
 1189 APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const    /*{{{*/
 1190 {
 1191    unsigned long long Maximum = std::numeric_limits<unsigned long long>::max();
 1192    for (auto const &O: Owners)
 1193    {
 1194       if (O->FileSize == 0)
 1195      continue;
 1196       Maximum = std::min(Maximum, O->FileSize);
 1197    }
 1198    if (Maximum == std::numeric_limits<unsigned long long>::max())
 1199       return 0;
 1200    return Maximum;
 1201 }
 1202                                     /*}}}*/
 1203 APT_PURE int pkgAcquire::Queue::QItem::GetPriority() const      /*{{{*/
 1204 {
 1205    int Priority = 0;
 1206    for (auto const &O: Owners)
 1207       Priority = std::max(Priority, O->Priority());
 1208 
 1209    return Priority;
 1210 }
 1211                                     /*}}}*/
 1212 void pkgAcquire::Queue::QItem::SyncDestinationFiles() const     /*{{{*/
 1213 {
 1214    /* ensure that the first owner has the best partial file of all and
 1215       the rest have (potentially dangling) symlinks to it so that
 1216       everything (like progress reporting) finds it easily */
 1217    std::string superfile = Owner->DestFile;
 1218    off_t supersize = 0;
 1219    for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
 1220    {
 1221       if ((*O)->DestFile == superfile)
 1222      continue;
 1223       struct stat file;
 1224       if (lstat((*O)->DestFile.c_str(),&file) == 0)
 1225       {
 1226      if ((file.st_mode & S_IFREG) == 0)
 1227         RemoveFile("SyncDestinationFiles", (*O)->DestFile);
 1228      else if (supersize < file.st_size)
 1229      {
 1230         supersize = file.st_size;
 1231         RemoveFile("SyncDestinationFiles", superfile);
 1232         rename((*O)->DestFile.c_str(), superfile.c_str());
 1233      }
 1234      else
 1235         RemoveFile("SyncDestinationFiles", (*O)->DestFile);
 1236      if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
 1237      {
 1238         ; // not a problem per-se and no real alternative
 1239      }
 1240       }
 1241    }
 1242 }
 1243                                     /*}}}*/
 1244 std::string pkgAcquire::Queue::QItem::Custom600Headers() const      /*{{{*/
 1245 {
 1246    /* The others are relatively easy to merge, but this one?
 1247       Lets not merge and see how far we can run with it…
 1248       Likely, nobody will ever notice as all the items will
 1249       be of the same class and hence generate the same headers. */
 1250    return Owner->Custom600Headers();
 1251 }
 1252                                     /*}}}*/
 1253 
 1254 // AcquireStatus::pkgAcquireStatus - Constructor            /*{{{*/
 1255 // ---------------------------------------------------------------------
 1256 /* */
 1257 pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
 1258 {
 1259    Start();
 1260 }
 1261                                     /*}}}*/
 1262 // AcquireStatus::Pulse - Called periodically               /*{{{*/
 1263 // ---------------------------------------------------------------------
 1264 /* This computes some internal state variables for the derived classes to
 1265    use. It generates the current downloaded bytes and total bytes to download
 1266    as well as the current CPS estimate. */
 1267 static struct timeval GetTimevalFromSteadyClock()
 1268 {
 1269    auto const Time = std::chrono::steady_clock::now().time_since_epoch();
 1270    auto const Time_sec = std::chrono::duration_cast<std::chrono::seconds>(Time);
 1271    auto const Time_usec = std::chrono::duration_cast<std::chrono::microseconds>(Time - Time_sec);
 1272    return { Time_sec.count(), Time_usec.count() };
 1273 }
 1274 bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
 1275 {
 1276    TotalBytes = 0;
 1277    CurrentBytes = 0;
 1278    TotalItems = 0;
 1279    CurrentItems = 0;
 1280 
 1281    // Compute the total number of bytes to fetch
 1282    unsigned int Unknown = 0;
 1283    unsigned int Count = 0;
 1284    bool ExpectAdditionalItems = false;
 1285    for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); 
 1286         I != Owner->ItemsEnd();
 1287     ++I, ++Count)
 1288    {
 1289       TotalItems++;
 1290       if ((*I)->Status == pkgAcquire::Item::StatDone)
 1291      ++CurrentItems;
 1292 
 1293       // do we expect to acquire more files than we know of yet?
 1294       if ((*I)->ExpectedAdditionalItems > 0)
 1295          ExpectAdditionalItems = true;
 1296 
 1297       TotalBytes += (*I)->FileSize;
 1298       if ((*I)->Complete == true)
 1299      CurrentBytes += (*I)->FileSize;
 1300       if ((*I)->FileSize == 0 && (*I)->Complete == false)
 1301      ++Unknown;
 1302    }
 1303 
 1304    // Compute the current completion
 1305    unsigned long long ResumeSize = 0;
 1306    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
 1307     I = Owner->WorkerStep(I))
 1308    {
 1309       if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
 1310       {
 1311      CurrentBytes += I->CurrentItem->CurrentSize;
 1312      ResumeSize += I->CurrentItem->ResumePoint;
 1313 
 1314      // Files with unknown size always have 100% completion
 1315      if (I->CurrentItem->Owner->FileSize == 0 &&
 1316          I->CurrentItem->Owner->Complete == false)
 1317         TotalBytes += I->CurrentItem->CurrentSize;
 1318       }
 1319    }
 1320    
 1321    // Normalize the figures and account for unknown size downloads
 1322    if (TotalBytes <= 0)
 1323       TotalBytes = 1;
 1324    if (Unknown == Count)
 1325       TotalBytes = Unknown;
 1326 
 1327    // Wha?! Is not supposed to happen.
 1328    if (CurrentBytes > TotalBytes)
 1329       CurrentBytes = TotalBytes;
 1330 
 1331    // Compute the CPS
 1332    struct timeval NewTime = GetTimevalFromSteadyClock();
 1333 
 1334    if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) ||
 1335        NewTime.tv_sec - Time.tv_sec > 6)
 1336    {
 1337       std::chrono::duration<double> Delta =
 1338      std::chrono::seconds(NewTime.tv_sec - Time.tv_sec) +
 1339      std::chrono::microseconds(NewTime.tv_usec - Time.tv_usec);
 1340 
 1341       // Compute the CPS value
 1342       if (Delta < std::chrono::milliseconds(10))
 1343      CurrentCPS = 0;
 1344       else
 1345      CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/ Delta.count();
 1346       LastBytes = CurrentBytes - ResumeSize;
 1347       ElapsedTime = llround(Delta.count());
 1348       Time = NewTime;
 1349    }
 1350 
 1351    double const OldPercent = Percent;
 1352    // calculate the percentage, if we have too little data assume 1%
 1353    if (ExpectAdditionalItems)
 1354       Percent = 0;
 1355    else
 1356       // use both files and bytes because bytes can be unreliable
 1357       Percent = (0.8 * (CurrentBytes/double(TotalBytes)*100.0) +
 1358                  0.2 * (CurrentItems/double(TotalItems)*100.0));
 1359 
 1360    // debug
 1361    if (_config->FindB("Debug::acquire::progress", false) == true)
 1362    {
 1363       std::clog
 1364          << "["
 1365          << std::setw(5) << std::setprecision(4) << std::showpoint << Percent 
 1366          << "]"
 1367          << " Bytes: "
 1368          << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
 1369          << " # Files: "
 1370          << CurrentItems << " / " << TotalItems
 1371          << std::endl;
 1372    }
 1373 
 1374    double const DiffPercent = Percent - OldPercent;
 1375    if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
 1376       return true;
 1377 
 1378    int fd = _config->FindI("APT::Status-Fd",-1);
 1379    if(fd > 0)
 1380    {
 1381       ostringstream status;
 1382 
 1383       unsigned long long ETA = 0;
 1384       if(CurrentCPS > 0 && TotalBytes > CurrentBytes)
 1385          ETA = (TotalBytes - CurrentBytes) / CurrentCPS;
 1386 
 1387       std::string msg;
 1388       long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
 1389       // only show the ETA if it makes sense
 1390       if (ETA > 0 && ETA < std::chrono::seconds(std::chrono::hours(24 * 2)).count())
 1391      strprintf(msg, _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
 1392       else
 1393      strprintf(msg, _("Retrieving file %li of %li"), i, TotalItems);
 1394 
 1395       // build the status str
 1396       std::ostringstream str;
 1397       str.imbue(std::locale::classic());
 1398       str.precision(4);
 1399       str << "dlstatus" << ':' << std::fixed << i << ':' << Percent << ':' << msg << '\n';
 1400       auto const dlstatus = str.str();
 1401       FileFd::Write(fd, dlstatus.data(), dlstatus.size());
 1402    }
 1403 
 1404    return true;
 1405 }
 1406                                     /*}}}*/
 1407 // AcquireStatus::Start - Called when the download is started       /*{{{*/
 1408 // ---------------------------------------------------------------------
 1409 /* We just reset the counters */
 1410 void pkgAcquireStatus::Start()
 1411 {
 1412    Time = StartTime = GetTimevalFromSteadyClock();
 1413    LastBytes = 0;
 1414    CurrentCPS = 0;
 1415    CurrentBytes = 0;
 1416    TotalBytes = 0;
 1417    FetchedBytes = 0;
 1418    ElapsedTime = 0;
 1419    TotalItems = 0;
 1420    CurrentItems = 0;
 1421 }
 1422                                     /*}}}*/
 1423 // AcquireStatus::Stop - Finished downloading               /*{{{*/
 1424 // ---------------------------------------------------------------------
 1425 /* This accurately computes the elapsed time and the total overall CPS. */
 1426 void pkgAcquireStatus::Stop()
 1427 {
 1428    // Compute the CPS and elapsed time
 1429    struct timeval NewTime = GetTimevalFromSteadyClock();
 1430 
 1431    std::chrono::duration<double> Delta =
 1432       std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
 1433       std::chrono::microseconds(NewTime.tv_usec - StartTime.tv_usec);
 1434 
 1435    // Compute the CPS value
 1436    if (Delta < std::chrono::milliseconds(10))
 1437       CurrentCPS = 0;
 1438    else
 1439       CurrentCPS = FetchedBytes / Delta.count();
 1440    LastBytes = CurrentBytes;
 1441    ElapsedTime = llround(Delta.count());
 1442 }
 1443                                     /*}}}*/
 1444 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
 1445 // ---------------------------------------------------------------------
 1446 /* This is used to get accurate final transfer rate reporting. */
 1447 void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
 1448 {
 1449    FetchedBytes += Size - Resume;
 1450 }
 1451                                     /*}}}*/
 1452 bool pkgAcquireStatus::ReleaseInfoChanges(metaIndex const * const LastRelease, metaIndex const * const CurrentRelease, std::vector<ReleaseInfoChange> &&Changes)/*{{{*/
 1453 {
 1454    (void) LastRelease;
 1455    (void) CurrentRelease;
 1456    return ReleaseInfoChangesAsGlobalErrors(std::move(Changes));
 1457 }
 1458                                     /*}}}*/
 1459 bool pkgAcquireStatus::ReleaseInfoChangesAsGlobalErrors(std::vector<ReleaseInfoChange> &&Changes)/*{{{*/
 1460 {
 1461    bool AllOkay = true;
 1462    for (auto const &c: Changes)
 1463       if (c.DefaultAction)
 1464      _error->Notice("%s", c.Message.c_str());
 1465       else
 1466       {
 1467      _error->Error("%s", c.Message.c_str());
 1468      AllOkay = false;
 1469       }
 1470    return AllOkay;
 1471 }
 1472                                     /*}}}*/
 1473 
 1474 
 1475 pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
 1476 {
 1477    while (CurItem == 0 && CurQ != 0)
 1478    {
 1479       CurItem = CurQ->Items;
 1480       CurQ = CurQ->Next;
 1481    }
 1482 }
 1483 
 1484 pkgAcquire::UriIterator::~UriIterator() {}
 1485 pkgAcquire::MethodConfig::~MethodConfig() {}
 1486 pkgAcquireStatus::~pkgAcquireStatus() {}