apt  2.2.4
About: Apt (Advanced Package Tool) is a management system for software packages (Debian/Ubuntu). Release series 2.2.
  Fossies Dox: apt-2.2.4.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

acquire.cc
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4 
5  Acquire - File Acquiration
6 
7  The core element for the schedule system is the concept of a named
8  queue. Each queue is unique and each queue has a name derived from the
9  URI. The degree of paralization can be controlled by how the queue
10  name is derived from the URI.
11 
12  ##################################################################### */
13  /*}}}*/
14 // Include Files /*{{{*/
15 #include <config.h>
16 
17 #include <apt-pkg/acquire-item.h>
18 #include <apt-pkg/acquire-worker.h>
19 #include <apt-pkg/acquire.h>
20 #include <apt-pkg/configuration.h>
21 #include <apt-pkg/error.h>
22 #include <apt-pkg/fileutl.h>
23 #include <apt-pkg/strutl.h>
24 
25 #include <algorithm>
26 #include <chrono>
27 #include <iomanip>
28 #include <iostream>
29 #include <memory>
30 #include <numeric>
31 #include <sstream>
32 #include <string>
33 #include <vector>
34 #include <cmath>
35 
36 #include <dirent.h>
37 #include <errno.h>
38 #include <fcntl.h>
39 #include <grp.h>
40 #include <pwd.h>
41 #include <stdio.h>
42 #include <stdlib.h>
43 #include <string.h>
44 #include <sys/select.h>
45 #include <sys/stat.h>
46 #include <sys/time.h>
47 #include <unistd.h>
48 
49 #include <apti18n.h>
50  /*}}}*/
51 
52 using namespace std;
53 
54 std::string pkgAcquire::URIEncode(std::string const &part) /*{{{*/
55 {
56  // The "+" is encoded as a workaround for an S3 bug (LP#1003633 and LP#1086997)
57  return QuoteString(part, _config->Find("Acquire::URIEncode", "+~ ").c_str());
58 }
59  /*}}}*/
60 // Acquire::pkgAcquire - Constructor /*{{{*/
61 // ---------------------------------------------------------------------
62 /* We grab some runtime state from the configuration space */
63 pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0),
64  Debug(_config->FindB("Debug::pkgAcquire",false)),
65  Running(false)
66 {
67  Initialize();
68 }
69 pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0),
70  Configs(0), Log(NULL), ToFetch(0),
71  Debug(_config->FindB("Debug::pkgAcquire",false)),
72  Running(false)
73 {
74  Initialize();
75  SetLog(Progress);
76 }
78 {
79  string const Mode = _config->Find("Acquire::Queue-Mode","host");
80  if (strcasecmp(Mode.c_str(),"host") == 0)
82  if (strcasecmp(Mode.c_str(),"access") == 0)
84 }
85  /*}}}*/
86 // Acquire::GetLock - lock directory and prepare for action /*{{{*/
87 static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode)
88 {
89  if (_config->FindB("Debug::SetupAPTPartialDirectory::AssumeGood", false))
90  return true;
91  std::string const partial = parent + postfix;
92  bool const partialExists = DirectoryExists(partial);
93  if (partialExists == false)
94  {
95  mode_t const old_umask = umask(S_IWGRP | S_IWOTH);
96  bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
97  CreateAPTDirectoryIfNeeded(parent, partial) == false);
98  umask(old_umask);
99  if (creation_fail == true)
100  return false;
101  }
102 
103  std::string const SandboxUser = _config->Find("APT::Sandbox::User");
104  if (getuid() == 0)
105  {
106  if (SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
107  {
108  struct passwd const * const pw = getpwnam(SandboxUser.c_str());
109  struct group const * const gr = getgrnam(ROOT_GROUP);
110  if (pw != NULL && gr != NULL)
111  {
112  // chown the partial dir
113  if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
114  _error->WarningE("SetupAPTPartialDirectory", "chown to %s:%s of directory %s failed", SandboxUser.c_str(), ROOT_GROUP, partial.c_str());
115  }
116  }
117  if (chmod(partial.c_str(), mode) != 0)
118  _error->WarningE("SetupAPTPartialDirectory", "chmod 0%03o of directory %s failed", mode, partial.c_str());
119 
120  }
121  else if (chmod(partial.c_str(), mode) != 0)
122  {
123  // if we haven't created the dir and aren't root, it is kinda expected that chmod doesn't work
124  if (partialExists == false)
125  _error->WarningE("SetupAPTPartialDirectory", "chmod 0%03o of directory %s failed", mode, partial.c_str());
126  }
127 
128  _error->PushToStack();
129  // remove 'old' FAILED files to stop us from collecting them for no reason
130  for (auto const &Failed: GetListOfFilesInDir(partial, "FAILED", false, false))
131  RemoveFile("SetupAPTPartialDirectory", Failed);
132  _error->RevertToStack();
133 
134  return true;
135 }
136 bool pkgAcquire::GetLock(std::string const &Lock)
137 {
138  if (Lock.empty() == true)
139  return false;
140 
141  // check for existence and possibly create auxiliary directories
142  string const listDir = _config->FindDir("Dir::State::lists");
143  string const archivesDir = _config->FindDir("Dir::Cache::Archives");
144 
145  if (Lock == listDir)
146  {
147  if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false)
148  return _error->Errno("Acquire", _("List directory %s is missing."), (listDir + "partial").c_str());
149  }
150  if (Lock == archivesDir)
151  {
152  if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false)
153  return _error->Errno("Acquire", _("Archives directory %s is missing."), (archivesDir + "partial").c_str());
154  }
155  if (Lock == listDir || Lock == archivesDir)
156  {
157  if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
158  {
159  // not being able to create lists/auxfiles isn't critical as we will use a tmpdir then
160  }
161  }
162 
163  if (_config->FindB("Debug::NoLocking", false) == true)
164  return true;
165 
166  // Lock the directory this acquire object will work in
167  if (LockFD != -1)
168  close(LockFD);
169  LockFD = ::GetLock(flCombine(Lock, "lock"));
170  if (LockFD == -1)
171  return _error->Error(_("Unable to lock directory %s"), Lock.c_str());
172 
173  return true;
174 }
175  /*}}}*/
176 // Acquire::~pkgAcquire - Destructor /*{{{*/
177 // ---------------------------------------------------------------------
178 /* Free our memory, clean up the queues (destroy the workers) */
180 {
181  Shutdown();
182 
183  if (LockFD != -1)
184  close(LockFD);
185 
186  while (Configs != 0)
187  {
188  MethodConfig *Jnk = Configs;
189  Configs = Configs->Next;
190  delete Jnk;
191  }
192 }
193  /*}}}*/
194 // Acquire::Shutdown - Clean out the acquire object /*{{{*/
195 // ---------------------------------------------------------------------
196 /* */
198 {
199  while (Items.empty() == false)
200  {
201  if (Items[0]->Status == Item::StatFetching)
202  Items[0]->Status = Item::StatError;
203  delete Items[0];
204  }
205 
206  while (Queues != 0)
207  {
208  Queue *Jnk = Queues;
209  Queues = Queues->Next;
210  delete Jnk;
211  }
212 }
213  /*}}}*/
214 // Acquire::Add - Add a new item /*{{{*/
215 // ---------------------------------------------------------------------
216 /* This puts an item on the acquire list. This list is mainly for tracking
217  item status */
218 void pkgAcquire::Add(Item *Itm)
219 {
220  Items.push_back(Itm);
221 }
222  /*}}}*/
223 // Acquire::Remove - Remove a item /*{{{*/
224 // ---------------------------------------------------------------------
225 /* Remove an item from the acquire list. This is usually not used.. */
226 void pkgAcquire::Remove(Item *Itm)
227 {
228  Dequeue(Itm);
229 
230  for (ItemIterator I = Items.begin(); I != Items.end();)
231  {
232  if (*I == Itm)
233  {
234  Items.erase(I);
235  I = Items.begin();
236  }
237  else
238  ++I;
239  }
240 }
241  /*}}}*/
242 // Acquire::Add - Add a worker /*{{{*/
243 // ---------------------------------------------------------------------
244 /* A list of workers is kept so that the select loop can direct their FD
245  usage. */
246 void pkgAcquire::Add(Worker *Work)
247 {
248  Work->NextAcquire = Workers;
249  Workers = Work;
250 }
251  /*}}}*/
252 // Acquire::Remove - Remove a worker /*{{{*/
253 // ---------------------------------------------------------------------
254 /* A worker has died. This can not be done while the select loop is running
255  as it would require that RunFds could handling a changing list state and
256  it can't.. */
257 void pkgAcquire::Remove(Worker *Work)
258 {
259  if (Running == true)
260  abort();
261 
262  Worker **I = &Workers;
263  for (; *I != 0;)
264  {
265  if (*I == Work)
266  *I = (*I)->NextAcquire;
267  else
268  I = &(*I)->NextAcquire;
269  }
270 }
271  /*}}}*/
272 // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
273 // ---------------------------------------------------------------------
274 /* This is the entry point for an item. An item calls this function when
275  it is constructed which creates a queue (based on the current queue
276  mode) and puts the item in that queue. If the system is running then
277  the queue might be started. */
279  pkgAcquire::MethodConfig const * const Config, pkgAcquireStatus * const Log)
280 {
281  auto SavedDesc = Item->GetItemDesc();
282  if (Item->IsRedirectionLoop(SavedDesc.URI))
283  {
284  std::string const Message = "400 URI Failure"
285  "\nURI: " + SavedDesc.URI +
286  "\nFilename: " + Item->DestFile +
287  "\nFailReason: RedirectionLoop";
288 
289  Item->Status = pkgAcquire::Item::StatError;
290  Item->Failed(Message, Config);
291  if (Log != nullptr)
292  Log->Fail(SavedDesc);
293  return true;
294  }
295 
296  HashStringList const hsl = Item->GetExpectedHashes();
297  if (hsl.usable() == false && Item->HashesRequired() &&
298  _config->Exists("Acquire::ForceHash") == false)
299  {
300  std::string const Message = "400 URI Failure"
301  "\nURI: " + SavedDesc.URI +
302  "\nFilename: " + Item->DestFile +
303  "\nFailReason: WeakHashSums";
304 
305  Item->Status = pkgAcquire::Item::StatAuthError;
306  Item->Failed(Message, Config);
307  if (Log != nullptr)
308  Log->Fail(SavedDesc);
309  return true;
310  }
311  return false;
312 }
313 void pkgAcquire::Enqueue(ItemDesc &Item)
314 {
315  // Determine which queue to put the item in
316  const MethodConfig *Config = nullptr;
317  string Name = QueueName(Item.URI,Config);
318  if (Name.empty() == true)
319  {
320  Item.Owner->Status = pkgAcquire::Item::StatError;
321  return;
322  }
323 
324  /* the check for running avoids that we produce errors
325  in logging before we actually have started, which would
326  be easier to implement but would confuse users/implementations
327  so we check the items skipped here in #Startup */
329  return;
330 
331  // Find the queue structure
332  Queue *I = Queues;
333  for (; I != 0 && I->Name != Name; I = I->Next);
334  if (I == 0)
335  {
336  I = new Queue(Name,this);
337  I->Next = Queues;
338  Queues = I;
339 
340  if (Running == true)
341  I->Startup();
342  }
343 
344  // See if this is a local only URI
345  if (Config->LocalOnly == true && Item.Owner->Complete == false)
346  Item.Owner->Local = true;
347  Item.Owner->Status = Item::StatIdle;
348 
349  // Queue it into the named queue
350  if(I->Enqueue(Item))
351  ToFetch++;
352 
353  // Some trace stuff
354  if (Debug == true)
355  {
356  clog << "Fetching " << Item.URI << endl;
357  clog << " to " << Item.Owner->DestFile << endl;
358  clog << " Queue is: " << Name << endl;
359  }
360 }
361  /*}}}*/
362 // Acquire::Dequeue - Remove an item from all queues /*{{{*/
363 // ---------------------------------------------------------------------
364 /* This is called when an item is finished being fetched. It removes it
365  from all the queues */
366 void pkgAcquire::Dequeue(Item *Itm)
367 {
368  Queue *I = Queues;
369  bool Res = false;
370  if (Debug == true)
371  clog << "Dequeuing " << Itm->DestFile << endl;
372 
373  for (; I != 0; I = I->Next)
374  {
375  if (I->Dequeue(Itm))
376  {
377  Res = true;
378  if (Debug == true)
379  clog << "Dequeued from " << I->Name << endl;
380  }
381  }
382 
383  if (Res == true)
384  ToFetch--;
385 }
386  /*}}}*/
387 // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
388 // ---------------------------------------------------------------------
389 /* The string returned depends on the configuration settings and the
390  method parameters. Given something like http://foo.org/bar it can
391  return http://foo.org or http */
392 string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
393 {
394  constexpr int DEFAULT_HOST_LIMIT = 10;
395  URI U(Uri);
396 
397  // Note that this gets written through the reference to the caller.
398  Config = GetConfig(U.Access);
399  if (Config == nullptr)
400  return {};
401 
402  // Access mode forces all methods to be Single-Instance
403  if (QueueMode == QueueAccess)
404  return U.Access;
405 
406  // Single-Instance methods get exactly one queue per URI
407  if (Config->SingleInstance == true)
408  return U.Access;
409 
410  // Host-less methods like rred, store, …
411  if (U.Host.empty())
412  {
413  int existing = 0;
414  // check how many queues exist already and reuse empty ones
415  auto const AccessSchema = U.Access + ':';
416  for (Queue const *I = Queues; I != 0; I = I->Next)
417  if (APT::String::Startswith(I->Name, AccessSchema))
418  {
419  if (I->Items == nullptr)
420  return I->Name;
421  ++existing;
422  }
423 
424  int const Limit = _config->FindI("Acquire::QueueHost::Limit",
425 #ifdef _SC_NPROCESSORS_ONLN
426  sysconf(_SC_NPROCESSORS_ONLN) * 2
427 #else
428  DEFAULT_HOST_LIMIT
429 #endif
430  );
431 
432  // create a new worker if we don't have too many yet
433  if (Limit <= 0 || existing < Limit)
434  return AccessSchema + std::to_string(existing);
435 
436  // find the worker with the least to do
437  // we already established that there are no empty and we can't spawn new
438  Queue const *selected = nullptr;
439  auto selected_backlog = std::numeric_limits<decltype(HashStringList().FileSize())>::max();
440  for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next)
441  if (APT::String::Startswith(Q->Name, AccessSchema))
442  {
443  decltype(selected_backlog) current_backlog = 0;
444  for (auto const *I = Q->Items; I != nullptr; I = I->Next)
445  {
446  auto const hashes = I->Owner->GetExpectedHashes();
447  if (not hashes.empty())
448  current_backlog += hashes.FileSize();
449  else
450  current_backlog += I->Owner->FileSize;
451  }
452  if (current_backlog < selected_backlog)
453  {
454  selected = Q;
455  selected_backlog = current_backlog;
456  }
457  }
458 
459  if (unlikely(selected == nullptr))
460  return AccessSchema + "0";
461  return selected->Name;
462  }
463  // most methods talking to remotes like http
464  else
465  {
466  auto const FullQueueName = U.Access + ':' + U.Host;
467  // if the queue already exists, re-use it
468  for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next)
469  if (Q->Name == FullQueueName)
470  return FullQueueName;
471 
472  int existing = 0;
473  // check how many queues exist already and reuse empty ones
474  auto const AccessSchema = U.Access + ':';
475  for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next)
476  if (APT::String::Startswith(Q->Name, AccessSchema))
477  ++existing;
478 
479  int const Limit = _config->FindI("Acquire::QueueHost::Limit", DEFAULT_HOST_LIMIT);
480  // if we have too many hosts open use a single generic for the rest
481  if (existing >= Limit)
482  return U.Access;
483 
484  // we can still create new named queues
485  return FullQueueName;
486  }
487 }
488  /*}}}*/
489 // Acquire::GetConfig - Fetch the configuration information /*{{{*/
490 // ---------------------------------------------------------------------
491 /* This locates the configuration structure for an access method. If
492  a config structure cannot be found a Worker will be created to
493  retrieve it */
495 {
496  // Search for an existing config
497  MethodConfig *Conf;
498  for (Conf = Configs; Conf != 0; Conf = Conf->Next)
499  if (Conf->Access == Access)
500  return Conf;
501 
502  // Create the new config class
503  Conf = new MethodConfig;
504  Conf->Access = Access;
505 
506  // Create the worker to fetch the configuration
507  Worker Work(Conf);
508  if (Work.Start() == false)
509  {
510  delete Conf;
511  return nullptr;
512  }
513  Conf->Next = Configs;
514  Configs = Conf;
515 
516  /* if a method uses DownloadLimit, we switch to SingleInstance mode */
517  if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
518  Conf->SingleInstance = true;
519 
520  return Conf;
521 }
522  /*}}}*/
523 // Acquire::SetFds - Deal with readable FDs /*{{{*/
524 // ---------------------------------------------------------------------
525 /* Collect FDs that have activity monitors into the fd sets */
526 void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
527 {
528  for (Worker *I = Workers; I != 0; I = I->NextAcquire)
529  {
530  if (I->InReady == true && I->InFd >= 0)
531  {
532  if (Fd < I->InFd)
533  Fd = I->InFd;
534  FD_SET(I->InFd,RSet);
535  }
536  if (I->OutReady == true && I->OutFd >= 0)
537  {
538  if (Fd < I->OutFd)
539  Fd = I->OutFd;
540  FD_SET(I->OutFd,WSet);
541  }
542  }
543 }
544  /*}}}*/
545 // Acquire::RunFds - Deal with active FDs /*{{{*/
546 // ---------------------------------------------------------------------
547 /* Dispatch active FDs over to the proper workers. It is very important
548  that a worker never be erased while this is running! The queue class
549  should never erase a worker except during shutdown processing. */
550 bool pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
551 {
552  bool Res = true;
553 
554  for (Worker *I = Workers; I != 0; I = I->NextAcquire)
555  {
556  if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
557  Res &= I->InFdReady();
558  if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
559  Res &= I->OutFdReady();
560  }
561 
562  return Res;
563 }
564  /*}}}*/
565 // Acquire::Run - Run the fetch sequence /*{{{*/
566 // ---------------------------------------------------------------------
567 /* This runs the queues. It manages a select loop for all of the
568  Worker tasks. The workers interact with the queues and items to
569  manage the actual fetch. */
570 static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite)
571 {
572  // you would think this is easily to answer with faccessat, right? Wrong!
573  // It e.g. gets groups wrong, so the only thing which works reliable is trying
574  // to open the file we want to open later on…
575  if (unlikely(filename.empty()))
576  return true;
577 
578  if (ReadWrite == false)
579  {
580  errno = 0;
581  // can we read a file? Note that non-existing files are "fine"
582  int const fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC);
583  if (fd == -1 && errno == EACCES)
584  return false;
585  close(fd);
586  return true;
587  }
588  else
589  {
590  // the file might not exist yet and even if it does we will fix permissions,
591  // so important is here just that the directory it is in allows that
592  std::string const dirname = flNotFile(filename);
593  if (unlikely(dirname.empty()))
594  return true;
595 
596  char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
597  std::string const tmpfile_tpl = flCombine(dirname, filetag);
598  std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
599  int const fd = mkstemp(tmpfile.get());
600  if (fd == -1 && errno == EACCES)
601  return false;
602  RemoveFile("IsAccessibleBySandboxUser", tmpfile.get());
603  close(fd);
604  return true;
605  }
606 }
607 static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
608 {
609  if(getuid() != 0)
610  return;
611 
612  std::string const SandboxUser = _config->Find("APT::Sandbox::User");
613  if (SandboxUser.empty() || SandboxUser == "root")
614  return;
615 
616  struct passwd const * const pw = getpwnam(SandboxUser.c_str());
617  if (pw == NULL)
618  {
619  _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str());
620  _config->Set("APT::Sandbox::User", "");
621  return;
622  }
623 
624  gid_t const old_euid = geteuid();
625  gid_t const old_egid = getegid();
626 
627  long const ngroups_max = sysconf(_SC_NGROUPS_MAX);
628  std::unique_ptr<gid_t[]> old_gidlist(new gid_t[ngroups_max]);
629  if (unlikely(old_gidlist == NULL))
630  return;
631  ssize_t old_gidlist_nr;
632  if ((old_gidlist_nr = getgroups(ngroups_max, old_gidlist.get())) < 0)
633  {
634  _error->FatalE("getgroups", "getgroups %lu failed", ngroups_max);
635  old_gidlist[0] = 0;
636  old_gidlist_nr = 1;
637  }
638  if (setgroups(1, &pw->pw_gid))
639  _error->FatalE("setgroups", "setgroups %u failed", pw->pw_gid);
640 
641  if (setegid(pw->pw_gid) != 0)
642  _error->FatalE("setegid", "setegid %u failed", pw->pw_gid);
643  if (seteuid(pw->pw_uid) != 0)
644  _error->FatalE("seteuid", "seteuid %u failed", pw->pw_uid);
645 
646  for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin();
647  I != Fetcher.ItemsEnd(); ++I)
648  {
649  // no need to drop privileges for a complete file
650  if ((*I)->Complete == true || (*I)->Status != pkgAcquire::Item::StatIdle)
651  continue;
652 
653  // if destination file is inaccessible all hope is lost for privilege dropping
654  if (IsAccessibleBySandboxUser((*I)->DestFile, true) == false)
655  {
656  _error->WarningE("pkgAcquire::Run", _("Download is performed unsandboxed as root as file '%s' couldn't be accessed by user '%s'."),
657  (*I)->DestFile.c_str(), SandboxUser.c_str());
658  _config->Set("APT::Sandbox::User", "");
659  break;
660  }
661 
662  // if its the source file (e.g. local sources) we might be lucky
663  // by dropping the dropping only for some methods.
664  URI const source((*I)->DescURI());
665  if (source.Access == "file" || source.Access == "copy")
666  {
667  std::string const conf = "Binary::" + source.Access + "::APT::Sandbox::User";
668  if (_config->Exists(conf) == true)
669  continue;
670 
671  auto const filepath = DeQuoteString(source.Path);
672  if (not IsAccessibleBySandboxUser(filepath, false))
673  {
674  _error->NoticeE("pkgAcquire::Run", _("Download is performed unsandboxed as root as file '%s' couldn't be accessed by user '%s'."),
675  filepath.c_str(), SandboxUser.c_str());
676  _config->CndSet("Binary::file::APT::Sandbox::User", "root");
677  _config->CndSet("Binary::copy::APT::Sandbox::User", "root");
678  }
679  }
680  }
681 
682  if (seteuid(old_euid) != 0)
683  _error->FatalE("seteuid", "seteuid %u failed", old_euid);
684  if (setegid(old_egid) != 0)
685  _error->FatalE("setegid", "setegid %u failed", old_egid);
686  if (setgroups(old_gidlist_nr, old_gidlist.get()))
687  _error->FatalE("setgroups", "setgroups %u failed", 0);
688 }
690 {
691  _error->PushToStack();
693 
694  Running = true;
695 
696  if (Log != 0)
697  Log->Start();
698 
699  for (Queue *I = Queues; I != 0; I = I->Next)
700  I->Startup();
701 
702  bool WasCancelled = false;
703 
704  // Run till all things have been acquired
705  struct timeval tv;
706  tv.tv_sec = 0;
707  tv.tv_usec = PulseIntervall;
708  while (ToFetch > 0)
709  {
710  fd_set RFds;
711  fd_set WFds;
712  int Highest = 0;
713  FD_ZERO(&RFds);
714  FD_ZERO(&WFds);
715  SetFds(Highest,&RFds,&WFds);
716 
717  int Res;
718  do
719  {
720  Res = select(Highest+1,&RFds,&WFds,0,&tv);
721  }
722  while (Res < 0 && errno == EINTR);
723 
724  if (Res < 0)
725  {
726  _error->Errno("select","Select has failed");
727  break;
728  }
729 
730  if(RunFds(&RFds,&WFds) == false)
731  break;
732 
733  // Timeout, notify the log class
734  if (Res == 0 || (Log != 0 && Log->Update == true))
735  {
736  tv.tv_usec = PulseIntervall;
737  for (Worker *I = Workers; I != 0; I = I->NextAcquire)
738  I->Pulse();
739  if (Log != 0 && Log->Pulse(this) == false)
740  {
741  WasCancelled = true;
742  break;
743  }
744  }
745  }
746 
747  if (Log != 0)
748  Log->Stop();
749 
750  // Shut down the acquire bits
751  Running = false;
752  for (Queue *I = Queues; I != 0; I = I->Next)
753  I->Shutdown(false);
754 
755  // Shut down the items
756  for (ItemIterator I = Items.begin(); I != Items.end(); ++I)
757  (*I)->Finished();
758 
759  bool const newError = _error->PendingError();
760  _error->MergeWithStack();
761  if (newError)
762  return Failed;
763  if (WasCancelled)
764  return Cancelled;
765  return Continue;
766 }
767  /*}}}*/
768 // Acquire::Bump - Called when an item is dequeued /*{{{*/
769 // ---------------------------------------------------------------------
770 /* This routine bumps idle queues in hopes that they will be able to fetch
771  the dequeued item */
773 {
774  for (Queue *I = Queues; I != 0; I = I->Next)
775  I->Bump();
776 }
777  /*}}}*/
778 // Acquire::WorkerStep - Step to the next worker /*{{{*/
779 // ---------------------------------------------------------------------
780 /* Not inlined to advoid including acquire-worker.h */
782 {
783  return I->NextAcquire;
784 }
785  /*}}}*/
786 // Acquire::Clean - Cleans a directory /*{{{*/
787 // ---------------------------------------------------------------------
788 /* This is a bit simplistic, it looks at every file in the dir and sees
789  if it is part of the download set. */
790 bool pkgAcquire::Clean(string Dir)
791 {
792  // non-existing directories are by definition clean…
793  if (DirectoryExists(Dir) == false)
794  return true;
795 
796  if(Dir == "/")
797  return _error->Error(_("Clean of %s is not supported"), Dir.c_str());
798 
799  int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
800  if (dirfd == -1)
801  return _error->Errno("open",_("Unable to read %s"),Dir.c_str());
802  DIR * const D = fdopendir(dirfd);
803  if (D == nullptr)
804  return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
805 
806  for (struct dirent *E = readdir(D); E != nullptr; E = readdir(D))
807  {
808  // Skip some entries
809  if (strcmp(E->d_name, "lock") == 0 ||
810  strcmp(E->d_name, "partial") == 0 ||
811  strcmp(E->d_name, "auxfiles") == 0 ||
812  strcmp(E->d_name, "lost+found") == 0 ||
813  strcmp(E->d_name, ".") == 0 ||
814  strcmp(E->d_name, "..") == 0)
815  continue;
816 
817  // Look in the get list and if not found nuke
818  if (std::any_of(Items.cbegin(), Items.cend(),
819  [&E](pkgAcquire::Item const * const I) {
820  return flNotDir(I->DestFile) == E->d_name;
821  }) == false)
822  {
823  RemoveFileAt("pkgAcquire::Clean", dirfd, E->d_name);
824  }
825  }
826  closedir(D);
827  return true;
828 }
829  /*}}}*/
830 // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
831 // ---------------------------------------------------------------------
832 /* This is the total number of bytes needed */
833 APT_PURE unsigned long long pkgAcquire::TotalNeeded()
834 {
835  return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
836  [](unsigned long long const T, Item const * const I) {
837  return T + I->FileSize;
838  });
839 }
840  /*}}}*/
841 // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
842 // ---------------------------------------------------------------------
843 /* This is the number of bytes that is not local */
844 APT_PURE unsigned long long pkgAcquire::FetchNeeded()
845 {
846  return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
847  [](unsigned long long const T, Item const * const I) {
848  if (I->Local == false)
849  return T + I->FileSize;
850  else
851  return T;
852  });
853 }
854  /*}}}*/
855 // Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/
856 // ---------------------------------------------------------------------
857 /* This is the number of bytes that is not local */
859 {
860  return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu,
861  [](unsigned long long const T, Item const * const I) {
862  if (I->Local == false)
863  return T + I->PartialSize;
864  else
865  return T;
866  });
867 }
868  /*}}}*/
869 // Acquire::UriBegin - Start iterator for the uri list /*{{{*/
870 // ---------------------------------------------------------------------
871 /* */
873 {
874  return UriIterator(Queues);
875 }
876  /*}}}*/
877 // Acquire::UriEnd - End iterator for the uri list /*{{{*/
878 // ---------------------------------------------------------------------
879 /* */
881 {
882  return UriIterator(0);
883 }
884  /*}}}*/
885 // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
887 {
888  public:
889  bool AuxRequests = false;
890  bool SendURIEncoded = false;
891 };
892 pkgAcquire::MethodConfig::MethodConfig() : d(new Private()), Next(0), SingleInstance(false),
893  Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
894  Removable(false)
895 {
896 }
897  /*}}}*/
898 bool pkgAcquire::MethodConfig::GetAuxRequests() const /*{{{*/
899 {
900  return d->AuxRequests;
901 }
902  /*}}}*/
903 void pkgAcquire::MethodConfig::SetAuxRequests(bool const value) /*{{{*/
904 {
905  d->AuxRequests = value;
906 }
907  /*}}}*/
908 bool pkgAcquire::MethodConfig::GetSendURIEncoded() const /*{{{*/
909 {
910  return d->SendURIEncoded;
911 }
912  /*}}}*/
913 void pkgAcquire::MethodConfig::SetSendURIEncoded(bool const value) /*{{{*/
914 {
915  d->SendURIEncoded = value;
916 }
917  /*}}}*/
918 
919 // Queue::Queue - Constructor /*{{{*/
920 // ---------------------------------------------------------------------
921 /* */
922 pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0),
923  Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1)
924 {
925 }
926  /*}}}*/
927 // Queue::~Queue - Destructor /*{{{*/
928 // ---------------------------------------------------------------------
929 /* */
930 pkgAcquire::Queue::~Queue()
931 {
932  Shutdown(true);
933 
934  while (Items != 0)
935  {
936  QItem *Jnk = Items;
937  Items = Items->Next;
938  delete Jnk;
939  }
940 }
941  /*}}}*/
942 // Queue::Enqueue - Queue an item to the queue /*{{{*/
943 // ---------------------------------------------------------------------
944 /* */
945 bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
946 {
947  // MetaKeysMatch checks whether the two items have no non-matching
948  // meta-keys. If the items are not transaction items, it returns
949  // true, so other items can still be merged.
950  auto MetaKeysMatch = [](pkgAcquire::ItemDesc const &A, pkgAcquire::Queue::QItem const *B) {
951  auto OwnerA = dynamic_cast<pkgAcqTransactionItem*>(A.Owner);
952  if (OwnerA == nullptr)
953  return true;
954 
955  for (auto const & OwnerBUncast : B->Owners) {
956  auto OwnerB = dynamic_cast<pkgAcqTransactionItem*>(OwnerBUncast);
957 
958  if (OwnerB != nullptr && OwnerA->GetMetaKey() != OwnerB->GetMetaKey())
959  return false;
960  }
961  return true;
962  };
963  QItem **OptimalI = &Items;
964  QItem **I = &Items;
965  // move to the end of the queue and check for duplicates here
966  for (; *I != 0; ) {
967  if (Item.URI == (*I)->URI && MetaKeysMatch(Item, *I))
968  {
969  if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
970  std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
971  (*I)->Owners.push_back(Item.Owner);
972  Item.Owner->Status = (*I)->Owner->Status;
973  return false;
974  }
975  // Determine the optimal position to insert: before anything with a
976  // higher priority.
977  int priority = (*I)->GetPriority();
978 
979  I = &(*I)->Next;
980  if (priority >= Item.Owner->Priority()) {
981  OptimalI = I;
982  }
983  }
984 
985 
986  // Create a new item
987  QItem *Itm = new QItem;
988  *Itm = Item;
989  Itm->Next = *OptimalI;
990  *OptimalI = Itm;
991 
992  Item.Owner->QueueCounter++;
993  if (Items->Next == 0)
994  Cycle();
995  return true;
996 }
997  /*}}}*/
998 // Queue::Dequeue - Remove an item from the queue /*{{{*/
999 // ---------------------------------------------------------------------
1000 /* We return true if we hit something */
1001 bool pkgAcquire::Queue::Dequeue(Item *Owner)
1002 {
1003  if (Owner->Status == pkgAcquire::Item::StatFetching)
1004  return _error->Error("Tried to dequeue a fetching object");
1005 
1006  bool Res = false;
1007 
1008  QItem **I = &Items;
1009  for (; *I != 0;)
1010  {
1011  if (Owner == (*I)->Owner)
1012  {
1013  QItem *Jnk= *I;
1014  *I = (*I)->Next;
1015  Owner->QueueCounter--;
1016  delete Jnk;
1017  Res = true;
1018  }
1019  else
1020  I = &(*I)->Next;
1021  }
1022 
1023  return Res;
1024 }
1025  /*}}}*/
1026 // Queue::Startup - Start the worker processes /*{{{*/
1027 // ---------------------------------------------------------------------
1028 /* It is possible for this to be called with a pre-existing set of
1029  workers. */
1030 bool pkgAcquire::Queue::Startup()
1031 {
1032  if (Workers == 0)
1033  {
1034  URI U(Name);
1035  pkgAcquire::MethodConfig * const Cnf = Owner->GetConfig(U.Access);
1036  if (unlikely(Cnf == nullptr))
1037  return false;
1038 
1039  // now-running twin of the pkgAcquire::Enqueue call
1040  for (QItem *I = Items; I != 0; )
1041  {
1042  auto const INext = I->Next;
1043  for (auto &&O: I->Owners)
1045  // if an item failed, it will be auto-dequeued invalidation our I here
1046  I = INext;
1047  }
1048 
1049  Workers = new Worker(this,Cnf,Owner->Log);
1050  Owner->Add(Workers);
1051  if (Workers->Start() == false)
1052  return false;
1053 
1054  /* When pipelining we commit 10 items. This needs to change when we
1055  added other source retry to have cycle maintain a pipeline depth
1056  on its own. */
1057  if (Cnf->Pipeline == true)
1058  MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10);
1059  else
1060  MaxPipeDepth = 1;
1061  }
1062 
1063  return Cycle();
1064 }
1065  /*}}}*/
1066 // Queue::Shutdown - Shutdown the worker processes /*{{{*/
1067 // ---------------------------------------------------------------------
1068 /* If final is true then all workers are eliminated, otherwise only workers
1069  that do not need cleanup are removed */
1070 bool pkgAcquire::Queue::Shutdown(bool Final)
1071 {
1072  // Delete all of the workers
1073  pkgAcquire::Worker **Cur = &Workers;
1074  while (*Cur != 0)
1075  {
1076  pkgAcquire::Worker *Jnk = *Cur;
1077  if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
1078  {
1079  *Cur = Jnk->NextQueue;
1080  Owner->Remove(Jnk);
1081  delete Jnk;
1082  }
1083  else
1084  Cur = &(*Cur)->NextQueue;
1085  }
1086 
1087  return true;
1088 }
1089  /*}}}*/
1090 // Queue::FindItem - Find a URI in the item list /*{{{*/
1091 // ---------------------------------------------------------------------
1092 /* */
1093 pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
1094 {
1095  if (Owner->Config->GetSendURIEncoded())
1096  {
1097  for (QItem *I = Items; I != nullptr; I = I->Next)
1098  if (I->URI == URI && I->Worker == Owner)
1099  return I;
1100  }
1101  else
1102  {
1103  for (QItem *I = Items; I != nullptr; I = I->Next)
1104  {
1105  if (I->Worker != Owner)
1106  continue;
1107  ::URI tmpuri{I->URI};
1108  tmpuri.Path = DeQuoteString(tmpuri.Path);
1109  if (URI == std::string(tmpuri))
1110  return I;
1111  }
1112  }
1113  return nullptr;
1114 }
1115  /*}}}*/
1116 // Queue::ItemDone - Item has been completed /*{{{*/
1117 // ---------------------------------------------------------------------
1118 /* The worker signals this which causes the item to be removed from the
1119  queue. If this is the last queue instance then it is removed from the
1120  main queue too.*/
1121 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
1122 {
1123  PipeDepth--;
1124  for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
1125  {
1126  if ((*O)->Status == pkgAcquire::Item::StatFetching)
1127  (*O)->Status = pkgAcquire::Item::StatDone;
1128  }
1129 
1130  if (Itm->Owner->QueueCounter <= 1)
1131  Owner->Dequeue(Itm->Owner);
1132  else
1133  {
1134  Dequeue(Itm->Owner);
1135  Owner->Bump();
1136  }
1137 
1138  return Cycle();
1139 }
1140  /*}}}*/
1141 // Queue::Cycle - Queue new items into the method /*{{{*/
1142 // ---------------------------------------------------------------------
1143 /* This locates a new idle item and sends it to the worker. If pipelining
1144  is enabled then it keeps the pipe full. */
1145 bool pkgAcquire::Queue::Cycle()
1146 {
1147  if (Items == 0 || Workers == 0)
1148  return true;
1149 
1150  if (PipeDepth < 0)
1151  return _error->Error("Pipedepth failure");
1152 
1153  // Look for a queable item
1154  QItem *I = Items;
1155  int ActivePriority = 0;
1156  while (PipeDepth < static_cast<decltype(PipeDepth)>(MaxPipeDepth))
1157  {
1158  for (; I != 0; I = I->Next) {
1159  if (I->Owner->Status == pkgAcquire::Item::StatFetching)
1160  ActivePriority = std::max(ActivePriority, I->GetPriority());
1161  if (I->Owner->Status == pkgAcquire::Item::StatIdle)
1162  break;
1163  }
1164 
1165  // Nothing to do, queue is idle.
1166  if (I == 0)
1167  return true;
1168 
1169  // This item has a lower priority than stuff in the pipeline, pretend
1170  // the queue is idle
1171  if (I->GetPriority() < ActivePriority)
1172  return true;
1173  I->Worker = Workers;
1174  for (auto const &O: I->Owners)
1175  O->Status = pkgAcquire::Item::StatFetching;
1176  PipeDepth++;
1177  if (Workers->QueueItem(I) == false)
1178  return false;
1179  }
1180 
1181  return true;
1182 }
1183  /*}}}*/
1184 // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
1185 // ---------------------------------------------------------------------
1186 /* This is called when an item in multiple queues is dequeued */
1187 void pkgAcquire::Queue::Bump()
1188 {
1189  Cycle();
1190 }
1191  /*}}}*/
1192 HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const /*{{{*/
1193 {
1194  /* each Item can have multiple owners and each owner might have different
1195  hashes, even if that is unlikely in practice and if so at least some
1196  owners will later fail. There is one situation through which is not a
1197  failure and still needs this handling: Two owners who expect the same
1198  file, but one owner only knows the SHA1 while the other only knows SHA256. */
1199  HashStringList superhsl;
1200  for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1201  {
1202  HashStringList const hsl = (*O)->GetExpectedHashes();
1203  // we merge both lists - if we find disagreement send no hashes
1205  for (; hs != hsl.end(); ++hs)
1206  if (superhsl.push_back(*hs) == false)
1207  break;
1208  if (hs != hsl.end())
1209  {
1210  superhsl.clear();
1211  break;
1212  }
1213  }
1214  return superhsl;
1215 }
1216  /*}}}*/
1217 APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const /*{{{*/
1218 {
1219  unsigned long long Maximum = std::numeric_limits<unsigned long long>::max();
1220  for (auto const &O: Owners)
1221  {
1222  if (O->FileSize == 0)
1223  continue;
1224  Maximum = std::min(Maximum, O->FileSize);
1225  }
1226  if (Maximum == std::numeric_limits<unsigned long long>::max())
1227  return 0;
1228  return Maximum;
1229 }
1230  /*}}}*/
1231 APT_PURE int pkgAcquire::Queue::QItem::GetPriority() const /*{{{*/
1232 {
1233  int Priority = 0;
1234  for (auto const &O: Owners)
1235  Priority = std::max(Priority, O->Priority());
1236 
1237  return Priority;
1238 }
1239  /*}}}*/
1240 void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/
1241 {
1242  /* ensure that the first owner has the best partial file of all and
1243  the rest have (potentially dangling) symlinks to it so that
1244  everything (like progress reporting) finds it easily */
1245  std::string superfile = Owner->DestFile;
1246  off_t supersize = 0;
1247  for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
1248  {
1249  if ((*O)->DestFile == superfile)
1250  continue;
1251  struct stat file;
1252  if (lstat((*O)->DestFile.c_str(),&file) == 0)
1253  {
1254  if ((file.st_mode & S_IFREG) == 0)
1255  RemoveFile("SyncDestinationFiles", (*O)->DestFile);
1256  else if (supersize < file.st_size)
1257  {
1258  supersize = file.st_size;
1259  RemoveFile("SyncDestinationFiles", superfile);
1260  rename((*O)->DestFile.c_str(), superfile.c_str());
1261  }
1262  else
1263  RemoveFile("SyncDestinationFiles", (*O)->DestFile);
1264  if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
1265  {
1266  ; // not a problem per-se and no real alternative
1267  }
1268  }
1269  }
1270 }
1271  /*}}}*/
1272 std::string pkgAcquire::Queue::QItem::Custom600Headers() const /*{{{*/
1273 {
1274  /* The others are relatively easy to merge, but this one?
1275  Lets not merge and see how far we can run with it…
1276  Likely, nobody will ever notice as all the items will
1277  be of the same class and hence generate the same headers. */
1278  return Owner->Custom600Headers();
1279 }
1280  /*}}}*/
1281 
1282 // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
1283 // ---------------------------------------------------------------------
1284 /* */
1285 pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
1286 {
1287  Start();
1288 }
1289  /*}}}*/
1290 // AcquireStatus::Pulse - Called periodically /*{{{*/
1291 // ---------------------------------------------------------------------
1292 /* This computes some internal state variables for the derived classes to
1293  use. It generates the current downloaded bytes and total bytes to download
1294  as well as the current CPS estimate. */
1295 static struct timeval GetTimevalFromSteadyClock()
1296 {
1297  auto const Time = std::chrono::steady_clock::now().time_since_epoch();
1298  auto const Time_sec = std::chrono::duration_cast<std::chrono::seconds>(Time);
1299  auto const Time_usec = std::chrono::duration_cast<std::chrono::microseconds>(Time - Time_sec);
1300  return { Time_sec.count(), Time_usec.count() };
1301 }
1303 {
1304  TotalBytes = 0;
1305  CurrentBytes = 0;
1306  TotalItems = 0;
1307  CurrentItems = 0;
1308 
1309  // Compute the total number of bytes to fetch
1310  unsigned int Unknown = 0;
1311  unsigned int Count = 0;
1312  bool ExpectAdditionalItems = false;
1313  for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin();
1314  I != Owner->ItemsEnd();
1315  ++I, ++Count)
1316  {
1317  TotalItems++;
1318  if ((*I)->Status == pkgAcquire::Item::StatDone)
1319  ++CurrentItems;
1320 
1321  // do we expect to acquire more files than we know of yet?
1322  if ((*I)->ExpectedAdditionalItems > 0)
1323  ExpectAdditionalItems = true;
1324 
1325  TotalBytes += (*I)->FileSize;
1326  if ((*I)->Complete == true)
1327  CurrentBytes += (*I)->FileSize;
1328  if ((*I)->FileSize == 0 && (*I)->Complete == false)
1329  ++Unknown;
1330  }
1331 
1332  // Compute the current completion
1333  unsigned long long ResumeSize = 0;
1334  for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
1335  I = Owner->WorkerStep(I))
1336  {
1337  if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
1338  {
1339  CurrentBytes += I->CurrentItem->CurrentSize;
1340  ResumeSize += I->CurrentItem->ResumePoint;
1341 
1342  // Files with unknown size always have 100% completion
1343  if (I->CurrentItem->Owner->FileSize == 0 &&
1344  I->CurrentItem->Owner->Complete == false)
1345  TotalBytes += I->CurrentItem->CurrentSize;
1346  }
1347  }
1348 
1349  // Normalize the figures and account for unknown size downloads
1350  if (TotalBytes <= 0)
1351  TotalBytes = 1;
1352  if (Unknown == Count)
1353  TotalBytes = Unknown;
1354 
1355  // Wha?! Is not supposed to happen.
1356  if (CurrentBytes > TotalBytes)
1358 
1359  // Compute the CPS
1360  struct timeval NewTime = GetTimevalFromSteadyClock();
1361 
1362  if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) ||
1363  NewTime.tv_sec - Time.tv_sec > 6)
1364  {
1365  std::chrono::duration<double> Delta =
1366  std::chrono::seconds(NewTime.tv_sec - Time.tv_sec) +
1367  std::chrono::microseconds(NewTime.tv_usec - Time.tv_usec);
1368 
1369  // Compute the CPS value
1370  if (Delta < std::chrono::milliseconds(10))
1371  CurrentCPS = 0;
1372  else
1373  CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/ Delta.count();
1374  LastBytes = CurrentBytes - ResumeSize;
1375  ElapsedTime = llround(Delta.count());
1376  Time = NewTime;
1377  }
1378 
1379  double const OldPercent = Percent;
1380  // calculate the percentage, if we have too little data assume 1%
1381  if (ExpectAdditionalItems)
1382  Percent = 0;
1383  else
1384  // use both files and bytes because bytes can be unreliable
1385  Percent = (0.8 * (CurrentBytes/double(TotalBytes)*100.0) +
1386  0.2 * (CurrentItems/double(TotalItems)*100.0));
1387 
1388  // debug
1389  if (_config->FindB("Debug::acquire::progress", false) == true)
1390  {
1391  std::clog
1392  << "["
1393  << std::setw(5) << std::setprecision(4) << std::showpoint << Percent
1394  << "]"
1395  << " Bytes: "
1396  << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
1397  << " # Files: "
1398  << CurrentItems << " / " << TotalItems
1399  << std::endl;
1400  }
1401 
1402  double const DiffPercent = Percent - OldPercent;
1403  if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
1404  return true;
1405 
1406  int fd = _config->FindI("APT::Status-Fd",-1);
1407  if(fd > 0)
1408  {
1409  unsigned long long ETA = 0;
1410  if(CurrentCPS > 0 && TotalBytes > CurrentBytes)
1411  ETA = (TotalBytes - CurrentBytes) / CurrentCPS;
1412 
1413  std::string msg;
1414  long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
1415  // only show the ETA if it makes sense
1416  if (ETA > 0 && ETA < std::chrono::seconds(std::chrono::hours(24 * 2)).count())
1417  strprintf(msg, _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
1418  else
1419  strprintf(msg, _("Retrieving file %li of %li"), i, TotalItems);
1420 
1421  // build the status str
1422  std::ostringstream str;
1423  str.imbue(std::locale::classic());
1424  str.precision(4);
1425  str << "dlstatus" << ':' << std::fixed << i << ':' << Percent << ':' << msg << '\n';
1426  auto const dlstatus = str.str();
1427  FileFd::Write(fd, dlstatus.data(), dlstatus.size());
1428  }
1429 
1430  return true;
1431 }
1432  /*}}}*/
1433 // AcquireStatus::Start - Called when the download is started /*{{{*/
1434 // ---------------------------------------------------------------------
1435 /* We just reset the counters */
1437 {
1439  LastBytes = 0;
1440  CurrentCPS = 0;
1441  CurrentBytes = 0;
1442  TotalBytes = 0;
1443  FetchedBytes = 0;
1444  ElapsedTime = 0;
1445  TotalItems = 0;
1446  CurrentItems = 0;
1447 }
1448  /*}}}*/
1449 // AcquireStatus::Stop - Finished downloading /*{{{*/
1450 // ---------------------------------------------------------------------
1451 /* This accurately computes the elapsed time and the total overall CPS. */
1453 {
1454  // Compute the CPS and elapsed time
1455  struct timeval NewTime = GetTimevalFromSteadyClock();
1456 
1457  std::chrono::duration<double> Delta =
1458  std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
1459  std::chrono::microseconds(NewTime.tv_usec - StartTime.tv_usec);
1460 
1461  // Compute the CPS value
1462  if (Delta < std::chrono::milliseconds(10))
1463  CurrentCPS = 0;
1464  else
1465  CurrentCPS = FetchedBytes / Delta.count();
1467  ElapsedTime = llround(Delta.count());
1468 }
1469  /*}}}*/
1470 // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
1471 // ---------------------------------------------------------------------
1472 /* This is used to get accurate final transfer rate reporting. */
1473 void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
1474 {
1475  FetchedBytes += Size - Resume;
1476 }
1477  /*}}}*/
1478 bool pkgAcquireStatus::ReleaseInfoChanges(metaIndex const * const LastRelease, metaIndex const * const CurrentRelease, std::vector<ReleaseInfoChange> &&Changes)/*{{{*/
1479 {
1480  (void) LastRelease;
1481  (void) CurrentRelease;
1482  return ReleaseInfoChangesAsGlobalErrors(std::move(Changes));
1483 }
1484  /*}}}*/
1485 bool pkgAcquireStatus::ReleaseInfoChangesAsGlobalErrors(std::vector<ReleaseInfoChange> &&Changes)/*{{{*/
1486 {
1487  bool AllOkay = true;
1488  for (auto const &c: Changes)
1489  if (c.DefaultAction)
1490  _error->Notice("%s", c.Message.c_str());
1491  else
1492  {
1493  _error->Error("%s", c.Message.c_str());
1494  AllOkay = false;
1495  }
1496  return AllOkay;
1497 }
1498  /*}}}*/
1499 
1500 
1501 pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
1502 {
1503  while (CurItem == 0 && CurQ != 0)
1504  {
1505  CurItem = CurQ->Items;
1506  CurQ = CurQ->Next;
1507  }
1508 }
1509 
1510 pkgAcquire::UriIterator::~UriIterator() {}
1511 pkgAcquire::MethodConfig::~MethodConfig() { delete d; }
strprintf(m, msg, repo.c_str())
static bool std::string const metaIndex const *const pkgAcqMetaClearSig *const pkgAcquire::Item *const I
return false
static char const *const msg
static bool CheckForBadItemAndFailIt(pkgAcquire::Item *const Item, pkgAcquire::MethodConfig const *const Config, pkgAcquireStatus *const Log)
Definition: acquire.cc:278
static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher)
Definition: acquire.cc:607
static struct timeval GetTimevalFromSteadyClock()
Definition: acquire.cc:1295
static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite)
Definition: acquire.cc:570
static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode)
Definition: acquire.cc:87
bool Write(const void *From, unsigned long long Size)
Definition: fileutl.cc:2819
bool usable() const
Definition: hashes.cc:178
const_iterator begin() const
Definition: hashes.h:133
std::vector< HashString >::const_iterator const_iterator
Definition: hashes.h:130
bool push_back(const HashString &hashString)
Definition: hashes.cc:232
void clear()
Definition: hashes.h:139
const_iterator end() const
Definition: hashes.h:136
Definition: strutl.h:193
std::string Access
Definition: strutl.h:198
std::string Path
Definition: strutl.h:202
std::string Host
Definition: strutl.h:201
std::string URI
Definition: metaindex.h:39
baseclass for the indexes files to manage them all together
Definition: acquire-item.h:366
A monitor object for downloads controlled by the pkgAcquire class. {{{.
Definition: acquire.h:696
virtual void Start()
Invoked when the Acquire process starts running.
Definition: acquire.cc:1436
virtual void Stop()
Invoked when the Acquire process stops running.
Definition: acquire.cc:1452
unsigned long long ElapsedTime
The amount of time that has elapsed since the download started.
Definition: acquire.h:738
virtual bool Pulse(pkgAcquire *Owner)
Periodically invoked while the Acquire process is underway.
Definition: acquire.cc:1302
struct timeval StartTime
The time at which the download started.
Definition: acquire.h:706
double Percent
The estimated percentage of the download (0-100)
Definition: acquire.h:752
virtual ~pkgAcquireStatus()
Definition: acquire.cc:1512
virtual void Fail(pkgAcquire::ItemDesc &)
Invoked when the process of fetching an item encounters a fatal error.
Definition: acquire.h:840
pkgAcquireStatus()
Initialize all counters to 0 and the time to the current time.
Definition: acquire.cc:1285
unsigned long long FetchedBytes
The total number of bytes accounted for by items that were successfully fetched.
Definition: acquire.h:733
bool Update
If true, the download scheduler should call Pulse() at the next available opportunity.
Definition: acquire.h:759
struct timeval Time
The last time at which this monitor object was updated.
Definition: acquire.h:703
unsigned long long CurrentBytes
The number of bytes fetched as of the most recent call to pkgAcquireStatus::Pulse,...
Definition: acquire.h:721
unsigned long CurrentItems
The number of items that have been successfully downloaded.
Definition: acquire.h:748
static APT_HIDDEN bool ReleaseInfoChangesAsGlobalErrors(std::vector< ReleaseInfoChange > &&Changes)
Definition: acquire.cc:1485
unsigned long long TotalBytes
The total number of bytes that need to be fetched.
Definition: acquire.h:728
unsigned long long CurrentCPS
The current rate of download as of the most recent call to pkgAcquireStatus::Pulse,...
Definition: acquire.h:716
unsigned long long LastBytes
The number of bytes fetched as of the previous call to pkgAcquireStatus::Pulse, including local items...
Definition: acquire.h:711
virtual void Fetched(unsigned long long Size, unsigned long long ResumePoint)
Invoked when a local or remote file has been completely fetched.
Definition: acquire.cc:1473
unsigned long TotalItems
The total number of items that need to be fetched.
Definition: acquire.h:745
virtual bool ReleaseInfoChanges(metaIndex const *const LastRelease, metaIndex const *const CurrentRelease, std::vector< ReleaseInfoChange > &&Changes)
ask the user for confirmation of changes to infos about a repository
Definition: acquire.cc:1478
Represents the process by which a pkgAcquire object should retrieve a file or a collection of files.
Definition: acquire-item.h:59
std::string Access
The access method to be used by this worker.
UriIterator UriBegin()
Get the head of the list of enqueued item URIs.
Definition: acquire.cc:872
ItemIterator ItemsEnd()
Get the end iterator of the list of items.
Definition: acquire.h:305
unsigned long long FetchNeeded()
Definition: acquire.cc:844
friend class Queue
int APT_HIDDEN Priority()
The priority of the item, used for queuing.
unsigned long long TotalNeeded()
Definition: acquire.cc:833
Worker * WorkersBegin()
Get the first Worker object.
Definition: acquire.h:291
Queue * Next
The next queue in the pkgAcquire object's list of queues.
Definition: acquire.h:403
unsigned int QueueCounter
The number of fetch queues into which this item has been inserted.
Definition: acquire-item.h:138
MethodConfig()
Set up the default method parameters.
int InFd
A file descriptor connected to the standard output of the subprocess.
MethodConfig * Config
The configuration of this method. On startup, the target of this pointer is filled in with basic data...
bool Cycle()
Send idle items to the worker process.
UriIterator UriEnd()
Get the end iterator of the list of enqueued item URIs.
Definition: acquire.cc:880
virtual ~pkgAcquire()
Destroy this pkgAcquire object.
Definition: acquire.cc:179
virtual std::string Custom600Headers() const
Custom headers to be sent to the fetch process.
std::string QueueName(std::string URI, MethodConfig const *&Config)
Determine the fetch method and queue of a URI.
Definition: acquire.cc:392
void Dequeue()
Remove this item from its owner's queue.
pkgAcquireStatus * Log
The download progress indicator to which progress messages should be sent.
virtual void SetFds(int &Fd, fd_set *RSet, fd_set *WSet)
Build up the set of file descriptors upon which select() should block.
Definition: acquire.cc:526
std::vector< Item * > Items
A list of items to download.
Definition: acquire.h:121
void Add(Item *Item)
Add the given item to the list of items.
Definition: acquire.cc:218
std::vector< Item * >::const_iterator ItemCIterator
Definition: acquire.h:112
unsigned long long FileSize
The size of the object to fetch.
Definition: acquire-item.h:96
signed long PipeDepth
The number of entries in this queue that are currently being downloaded.
Definition: acquire.h:489
Private *const d
dpointer placeholder (for later in case we need it)
Definition: acquire-item.h:357
int LockFD
FD of the Lock file we acquire in Setup (if any)
Definition: acquire.h:96
APT_HIDDEN void Initialize()
Definition: acquire.cc:77
bool GetLock(std::string const &Lock)
acquire lock and perform directory setup
Definition: acquire.cc:136
Queue * Queues
The head of the list of active queues.
Definition: acquire.h:128
void SetLog(pkgAcquireStatus *Progress)
Definition: acquire.h:344
friend class Item
Definition: acquire.h:106
bool Clean(std::string Dir)
Definition: acquire.cc:790
MethodConfig * Configs
The head of the list of acquire method configurations.
Definition: acquire.h:147
std::string DestFile
The name of the file into which the retrieved object will be written.
Definition: acquire-item.h:153
static APT_HIDDEN std::string URIEncode(std::string const &part)
Definition: acquire.cc:54
unsigned long ToFetch
The number of files which are to be fetched.
Definition: acquire.h:153
void Bump()
Check for idle queues with ready-to-fetch items.
Definition: acquire.cc:772
pkgAcquire *const Owner
The acquire object with which this item is associated.
Definition: acquire-item.h:306
Worker(Queue *OwnerQ, MethodConfig *Config, pkgAcquireStatus *Log)
Create a new Worker to download files.
void Shutdown()
Remove all items from this download process, terminate all download workers, and empty all queues.
Definition: acquire.cc:197
friend class pkgAcquire
RunResult
Provides information on how a download terminated.
Definition: acquire.h:257
@ Failed
Some files failed to download.
Definition: acquire.h:262
@ Continue
All files were fetched successfully.
Definition: acquire.h:259
@ Cancelled
The download was cancelled by the user (i.e., Log's pkgAcquireStatus::Pulse() method returned false).
Definition: acquire.h:267
std::vector< Item * >::iterator ItemIterator
Definition: acquire.h:111
MethodConfig * GetConfig(std::string Access)
Retrieve information about a fetch method by name.
Definition: acquire.cc:494
UriIterator(pkgAcquire::Queue *Q)
Create a new UriIterator.
std::string Name
The name of this queue.
Definition: acquire.h:464
Worker * Workers
The head of the list of active workers.
Definition: acquire.h:135
void Enqueue(ItemDesc &Item)
Insert the given fetch request into the appropriate queue.
Definition: acquire.cc:313
enum pkgAcquire::QueueStrategy QueueMode
enum pkgAcquire::ItemState Status
bool Running
If true, a download is currently in progress.
Definition: acquire.h:171
void Remove(Item *Item)
Remove the given item from the list of items.
Definition: acquire.cc:226
Worker * WorkerStep(Worker *I) APT_PURE
Advance to the next Worker object.
Definition: acquire.cc:781
@ QueueHost
Generate one queue for each protocol/host combination; downloads from multiple hosts can proceed in p...
Definition: acquire.h:162
@ QueueAccess
Generate a single queue for each protocol; serialize downloads from multiple hosts.
Definition: acquire.h:166
RunResult Run(int PulseInterval=500000)
Download all the items that have been Add()ed to this download process.
Definition: acquire.cc:689
int OutFd
A file descriptor connected to the standard input of the subprocess.
ItemIterator ItemsBegin()
Get the head of the list of items.
Definition: acquire.h:301
unsigned long long PartialPresent()
Definition: acquire.cc:858
virtual bool RunFds(fd_set *RSet, fd_set *WSet)
Definition: acquire.cc:550
unsigned long MaxPipeDepth
The maximum number of entries that this queue will attempt to download at once.
Definition: acquire.h:494
DIR
Configuration * _config
string flNotFile(string File)
Definition: fileutl.cc:676
bool DirectoryExists(string const &Path)
Definition: fileutl.cc:348
bool CreateAPTDirectoryIfNeeded(string const &Parent, string const &Path)
Ensure the existence of the given Path.
Definition: fileutl.cc:400
string flCombine(string Dir, string File)
Definition: fileutl.cc:740
bool RemoveFile(char const *const Function, std::string const &FileName)
Definition: fileutl.cc:198
bool RemoveFileAt(char const *const Function, int const dirfd, std::string const &FileName)
Definition: fileutl.cc:183
std::vector< string > GetListOfFilesInDir(string const &Dir, string const &Ext, bool const &SortList, bool const &AllowNoExt)
Definition: fileutl.cc:421
#define APT_PURE
Definition: macros.h:56
bool Startswith(const std::string &s, const std::string &start)
Definition: strutl.cc:84
std::vector< Item * >::const_iterator owner_iterator
Definition: acquire.h:433
string SizeToStr(double Size)
Definition: strutl.cc:437
string TimeToStr(unsigned long Sec)
Definition: strutl.cc:473
string DeQuoteString(const string &Str)
Definition: strutl.cc:404
string QuoteString(const string &Str, const char *Bad)
Definition: strutl.cc:384