apt  2.2.4
About: Apt (Advanced Package Tool) is a management system for software packages (Debian/Ubuntu). Release series 2.2.
  Fossies Dox: apt-2.2.4.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

acquire-item.cc
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4 
5  Acquire Item - Item to acquire
6 
7  Each item can download to exactly one file at a time. This means you
8  cannot create an item that fetches two uri's to two files at the same
9  time. The pkgAcqIndex class creates a second class upon instantiation
10  to fetch the other index files because of this.
11 
12  ##################################################################### */
13  /*}}}*/
14 // Include Files /*{{{*/
15 #include <config.h>
16 
17 #include <apt-pkg/acquire-item.h>
18 #include <apt-pkg/acquire-worker.h>
19 #include <apt-pkg/acquire.h>
21 #include <apt-pkg/configuration.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/fileutl.h>
24 #include <apt-pkg/gpgv.h>
25 #include <apt-pkg/hashes.h>
26 #include <apt-pkg/indexfile.h>
27 #include <apt-pkg/metaindex.h>
28 #include <apt-pkg/pkgcache.h>
29 #include <apt-pkg/pkgrecords.h>
30 #include <apt-pkg/sourcelist.h>
31 #include <apt-pkg/strutl.h>
32 #include <apt-pkg/tagfile.h>
33 
34 #include <algorithm>
35 #include <ctime>
36 #include <chrono>
37 #include <iostream>
38 #include <memory>
39 #include <numeric>
40 #include <random>
41 #include <sstream>
42 #include <string>
43 #include <unordered_set>
44 #include <vector>
45 #include <errno.h>
46 #include <stddef.h>
47 #include <stdio.h>
48 #include <stdlib.h>
49 #include <string.h>
50 #include <sys/stat.h>
51 #include <unistd.h>
52 
53 #include <apti18n.h>
54  /*}}}*/
55 
56 using namespace std;
57 
58 static std::string GetPartialFileName(std::string const &file) /*{{{*/
59 {
60  std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/";
61  DestFile += file;
62  return DestFile;
63 }
64  /*}}}*/
65 static std::string GetPartialFileNameFromURI(std::string const &uri) /*{{{*/
66 {
67  return GetPartialFileName(URItoFileName(uri));
68 }
69  /*}}}*/
70 static std::string GetFinalFileNameFromURI(std::string const &uri) /*{{{*/
71 {
72  return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
73 }
74  /*}}}*/
75 static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/
76 {
77  if (Target.KeepCompressed == false)
78  return file;
79 
80  std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
81  if (KeepCompressedAs.empty() == false)
82  {
83  std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
84  if (ext != "uncompressed")
85  file.append(".").append(ext);
86  }
87  return file;
88 }
89  /*}}}*/
90 static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
91 {
92  // rred expects the patch as $FinalFile.ed.$patchname.gz
93  return Final + ".ed." + Patch + ".gz";
94 }
95  /*}}}*/
96 static std::string GetDiffsPatchFileName(std::string const &Final) /*{{{*/
97 {
98  // rred expects the patch as $FinalFile.ed
99  return Final + ".ed";
100 }
101  /*}}}*/
102 static std::string GetExistingFilename(std::string const &File) /*{{{*/
103 {
104  if (RealFileExists(File))
105  return File;
106  for (auto const &type : APT::Configuration::getCompressorExtensions())
107  {
108  std::string const Final = File + type;
109  if (RealFileExists(Final))
110  return Final;
111  }
112  return "";
113 }
114  /*}}}*/
115 static std::string GetDiffIndexFileName(std::string const &Name) /*{{{*/
116 {
117  return Name + ".diff/Index";
118 }
119  /*}}}*/
120 static std::string GetDiffIndexURI(IndexTarget const &Target) /*{{{*/
121 {
122  return Target.URI + ".diff/Index";
123 }
124  /*}}}*/
125 
126 static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string const &FailCode, std::string const &Details)/*{{{*/
127 {
128  // we only act if a mirror was used at all
129  if(I.UsedMirror.empty())
130  return;
131 #if 0
132  std::cerr << "\nReportMirrorFailure: "
133  << UsedMirror
134  << " Uri: " << DescURI()
135  << " FailCode: "
136  << FailCode << std::endl;
137 #endif
138  string const report = _config->Find("Methods::Mirror::ProblemReporting",
139  LIBEXEC_DIR "/apt-report-mirror-failure");
140  if(!FileExists(report))
141  return;
142 
143  std::vector<char const*> const Args = {
144  report.c_str(),
145  I.UsedMirror.c_str(),
146  I.DescURI().c_str(),
147  FailCode.c_str(),
148  Details.c_str(),
149  NULL
150  };
151 
152  pid_t pid = ExecFork();
153  if(pid < 0)
154  {
155  _error->Error("ReportMirrorFailure Fork failed");
156  return;
157  }
158  else if(pid == 0)
159  {
160  execvp(Args[0], (char**)Args.data());
161  std::cerr << "Could not exec " << Args[0] << std::endl;
162  _exit(100);
163  }
164  if(!ExecWait(pid, "report-mirror-failure"))
165  _error->Warning("Couldn't report problem to '%s'", report.c_str());
166 }
167  /*}}}*/
168 
169 static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/
170 {
171  std::string m;
172  strprintf(m, msg, repo.c_str());
173  if (isError)
174  {
175  _error->Error("%s", m.c_str());
176  _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default."));
177  }
178  else
179  {
180  _error->Warning("%s", m.c_str());
181  _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use."));
182  }
183  _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details."));
184  return false;
185 }
186  /*}}}*/
187 // AllowInsecureRepositories /*{{{*/
189 static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type)
190 {
191  if (_config->FindB("Acquire::AllowInsecureRepositories"))
192  return true;
193 
195  return true;
196 
197  switch (type)
198  {
199  case InsecureType::UNSIGNED: break;
200  case InsecureType::NORELEASE: break;
201  case InsecureType::WEAK:
202  if (_config->FindB("Acquire::AllowWeakRepositories"))
203  return true;
205  return true;
206  break;
207  }
208  return false;
209 }
210 static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo,
212 {
213  // we skip weak downgrades as its unlikely that a repository gets really weaker –
214  // its more realistic that apt got pickier in a newer version
215  if (msg != InsecureType::WEAK)
216  {
217  std::string const FinalInRelease = TransactionManager->GetFinalFilename();
218  std::string const FinalReleasegpg = FinalInRelease.substr(0, FinalInRelease.length() - strlen("InRelease")) + "Release.gpg";
219  if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
220  {
221  char const * msgstr = nullptr;
222  switch (msg)
223  {
224  case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is no longer signed."); break;
225  case InsecureType::NORELEASE: msgstr = _("The repository '%s' no longer has a Release file."); break;
226  case InsecureType::WEAK: /* unreachable */ break;
227  }
228  if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") ||
230  {
231  // meh, the users wants to take risks (we still mark the packages
232  // from this repository as unauthenticated)
233  _error->Warning(msgstr, repo.c_str());
234  _error->Warning(_("This is normally not allowed, but the option "
235  "Acquire::AllowDowngradeToInsecureRepositories was "
236  "given to override it."));
237  } else {
239  TransactionManager->AbortTransaction();
240  I->Status = pkgAcquire::Item::StatError;
241  return false;
242  }
243  }
244  }
245 
247  return true;
248 
249  char const * msgstr = nullptr;
250  switch (msg)
251  {
252  case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is not signed."); break;
253  case InsecureType::NORELEASE: msgstr = _("The repository '%s' does not have a Release file."); break;
254  case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break;
255  }
256 
258  {
260  return true;
261  }
262 
264  TransactionManager->AbortTransaction();
265  I->Status = pkgAcquire::Item::StatError;
266  return false;
267 }
268  /*}}}*/
269 static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/
270 {
271  if (Parser == NULL)
272  return HashStringList();
273  metaIndex::checkSum * const R = Parser->Lookup(MetaKey);
274  if (R == NULL)
275  return HashStringList();
276  return R->Hashes;
277 }
278  /*}}}*/
279 
281 {
282 public:
284  {
285  std::string URI;
286  std::unordered_map<std::string, std::string> changefields;
287  AlternateURI(std::string &&u, decltype(changefields) &&cf) : URI(u), changefields(cf) {}
288  };
289  std::list<AlternateURI> AlternativeURIs;
290  std::vector<std::string> BadAlternativeSites;
291  std::vector<std::string> PastRedirections;
292  std::unordered_map<std::string, std::string> CustomFields;
293 
295  {
296  }
297 };
298  /*}}}*/
299 
300 // all ::HashesRequired and ::GetExpectedHashes implementations /*{{{*/
301 /* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
302  It is best to implement it as broadly as possible, while ::HashesRequired defaults
303  to true and should be as restrictive as possible for false cases. Note that if
304  a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't
305  ::HashesRequired is called to evaluate if its okay to have no hashes. */
307 {
308  /* signed repositories obviously have a parser and good hashes.
309  unsigned repositories, too, as even if we can't trust them for security,
310  we can at least trust them for integrity of the download itself.
311  Only repositories without a Release file can (obviously) not have
312  hashes – and they are very uncommon and strongly discouraged */
314  return false;
316  {
317  /* If we allow weak hashes, we check that we have some (weak) and then
318  declare hashes not needed. That will tip us in the right direction
319  as if hashes exist, they will be used, even if not required */
320  auto const hsl = GetExpectedHashes();
321  if (hsl.usable())
322  return true;
323  if (hsl.empty() == false)
324  return false;
325  }
326  return true;
327 }
329 {
331 }
332 
334 {
335  // Release and co have no hashes 'by design'.
336  return false;
337 }
339 {
340  return HashStringList();
341 }
342 
344 {
345  /* We can't check hashes of rred result as we don't know what the
346  hash of the file will be. We just know the hash of the patch(es),
347  the hash of the file they will apply on and the hash of the resulting
348  file. */
349  if (State == StateFetchDiff)
350  return true;
351  return false;
352 }
354 {
355  if (State == StateFetchDiff)
356  return available_patches[0].download_hashes;
357  return HashStringList();
358 }
359 
361 {
362  /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
363  we can check the rred result after all patches are applied as
364  we know the expected result rather than potentially apply more patches */
365  if (State == StateFetchDiff)
366  return true;
367  return State == StateApplyDiff;
368 }
370 {
371  if (State == StateFetchDiff)
372  return patch.download_hashes;
373  else if (State == StateApplyDiff)
375  return HashStringList();
376 }
377 
379 {
380  return LocalSource == false;
381 }
383 {
384  // figured out while parsing the records
385  return ExpectedHashes;
386 }
387 
389 {
390  // supplied as parameter at creation time, so the caller decides
391  return ExpectedHashes.usable();
392 }
394 {
395  return ExpectedHashes;
396 }
397  /*}}}*/
398 // Acquire::Item::QueueURI and specialisations from child classes /*{{{*/
399 bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item)
400 {
401  Owner->Enqueue(Item);
402  return true;
403 }
404 /* The idea here is that an item isn't queued if it exists on disk and the
405  transition manager was a hit as this means that the files it contains
406  the checksums for can't be updated either (or they are and we are asking
407  for a hashsum mismatch to happen which helps nobody) */
408 bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
409 {
411  {
412  if (_config->FindB("Debug::Acquire::Transaction", false))
413  std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl;
414  return false;
415  }
416  std::string const FinalFile = GetFinalFilename();
417  if (TransactionManager->IMSHit == true && FileExists(FinalFile) == true)
418  {
419  PartialFile = DestFile = FinalFile;
420  Status = StatDone;
421  return false;
422  }
423  // this ensures we rewrite only once and only the first step
424  auto const OldBaseURI = Target.Option(IndexTarget::BASE_URI);
425  if (OldBaseURI.empty() || APT::String::Startswith(Item.URI, OldBaseURI) == false)
426  return pkgAcquire::Item::QueueURI(Item);
427  // the given URI is our last resort
428  PushAlternativeURI(std::string(Item.URI), {}, false);
429  // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too
430  std::string SameMirrorURI;
431  if (TransactionManager->BaseURI.empty() == false && TransactionManager->UsedMirror.empty() == false &&
433  {
434  auto ExtraPath = Item.URI.substr(OldBaseURI.length());
435  auto newURI = flCombine(TransactionManager->BaseURI, std::move(ExtraPath));
436  if (IsGoodAlternativeURI(newURI))
437  {
438  SameMirrorURI = std::move(newURI);
439  PushAlternativeURI(std::string(SameMirrorURI), {}, false);
440  }
441  }
442  // add URI and by-hash based on it
443  if (AcquireByHash())
444  {
445  // if we use the mirror transport, ask it for by-hash uris
446  // we need to stick to the same mirror only for non-unique filenames
447  auto const sameMirrorException = [&]() {
448  if (Item.URI.find("mirror") == std::string::npos)
449  return false;
450  ::URI uri(Item.URI);
451  return uri.Access == "mirror" || APT::String::Startswith(uri.Access, "mirror+") ||
452  APT::String::Endswith(uri.Access, "+mirror") || uri.Access.find("+mirror+") != std::string::npos;
453  }();
454  if (sameMirrorException)
455  SameMirrorURI.clear();
456  // now add the actual by-hash uris
457  auto const Expected = GetExpectedHashes();
458  auto const TargetHash = Expected.find(nullptr);
459  auto const PushByHashURI = [&](std::string U) {
460  if (unlikely(TargetHash == nullptr))
461  return false;
462  auto const trailing_slash = U.find_last_of("/");
463  if (unlikely(trailing_slash == std::string::npos))
464  return false;
465  auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
466  U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix));
467  PushAlternativeURI(std::move(U), {}, false);
468  return true;
469  };
470  PushByHashURI(Item.URI);
471  if (SameMirrorURI.empty() == false && PushByHashURI(SameMirrorURI) == false)
472  SameMirrorURI.clear();
473  }
474  // the last URI added is the first one tried
475  if (unlikely(PopAlternativeURI(Item.URI) == false))
476  return false;
477  if (SameMirrorURI.empty() == false)
478  {
479  UsedMirror = TransactionManager->UsedMirror;
480  if (Item.Description.find(" ") != string::npos)
481  Item.Description.replace(0, Item.Description.find(" "), UsedMirror);
482  }
483  return pkgAcquire::Item::QueueURI(Item);
484 }
485 /* The transition manager InRelease itself (or its older sisters-in-law
486  Release & Release.gpg) is always queued as this allows us to rerun gpgv
487  on it to verify that we aren't stalled with old files */
488 bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
489 {
490  return pkgAcquire::Item::QueueURI(Item);
491 }
492 /* the Diff/Index needs to queue also the up-to-date complete index file
493  to ensure that the list cleaner isn't eating it */
494 bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
495 {
496  if (pkgAcqTransactionItem::QueueURI(Item) == true)
497  return true;
498  QueueOnIMSHit();
499  return false;
500 }
501  /*}}}*/
502 // Acquire::Item::GetFinalFilename and specialisations for child classes /*{{{*/
503 std::string pkgAcquire::Item::GetFinalFilename() const
504 {
505  // Beware: Desc.URI is modified by redirections
506  return GetFinalFileNameFromURI(Desc.URI);
507 }
509 {
510  std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
511  // we don't want recompress, so lets keep whatever we got
512  if (CurrentCompressionExtension == "uncompressed")
513  return FinalFile;
514  return FinalFile + "." + CurrentCompressionExtension;
515 }
516 std::string pkgAcqIndex::GetFinalFilename() const
517 {
518  std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
519  return GetKeepCompressedFileName(FinalFile, Target);
520 }
522 {
524 }
526 {
528 }
530 {
532 }
534 {
535  return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
536 }
537  /*}}}*/
538 // pkgAcqTransactionItem::GetMetaKey and specialisations for child classes /*{{{*/
540 {
541  return Target.MetaKey;
542 }
543 std::string pkgAcqIndex::GetMetaKey() const
544 {
546  return Target.MetaKey;
548 }
549 std::string pkgAcqDiffIndex::GetMetaKey() const
550 {
551  auto const metakey = GetDiffIndexFileName(Target.MetaKey);
552  if (CurrentCompressionExtension == "uncompressed")
553  return metakey;
554  return metakey + "." + CurrentCompressionExtension;
555 }
556  /*}}}*/
557 //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
559 {
560  bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
561  switch(state)
562  {
563  case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
564  case TransactionAbort:
565  if(Debug == true)
566  std::clog << " Cancel: " << DestFile << std::endl;
567  if (Status == pkgAcquire::Item::StatIdle)
568  {
569  Status = pkgAcquire::Item::StatDone;
570  Dequeue();
571  }
572  break;
573  case TransactionCommit:
574  if(PartialFile.empty() == false)
575  {
576  bool sameFile = (PartialFile == DestFile);
577  // we use symlinks on IMS-Hit to avoid copies
578  if (RealFileExists(DestFile))
579  {
580  struct stat Buf;
581  if (lstat(PartialFile.c_str(), &Buf) != -1)
582  {
583  if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
584  {
585  char partial[Buf.st_size + 1];
586  ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
587  if (sp == -1)
588  _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
589  else
590  {
591  partial[sp] = '\0';
592  sameFile = (DestFile == partial);
593  }
594  }
595  }
596  else
597  _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
598  }
599  if (sameFile == false)
600  {
601  // ensure that even without lists-cleanup all compressions are nuked
602  std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
603  if (FileExists(FinalFile))
604  {
605  if(Debug == true)
606  std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl;
607  if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false)
608  return false;
609  }
610  for (auto const &ext: APT::Configuration::getCompressorExtensions())
611  {
612  auto const Final = FinalFile + ext;
613  if (FileExists(Final))
614  {
615  if(Debug == true)
616  std::clog << "rm " << Final << " # " << DescURI() << std::endl;
617  if (RemoveFile("TransactionStates-Cleanup", Final) == false)
618  return false;
619  }
620  }
621  if(Debug == true)
622  std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
623  if (Rename(PartialFile, DestFile) == false)
624  return false;
625  }
626  else if(Debug == true)
627  std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl;
628 
629  } else {
630  if(Debug == true)
631  std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
632  if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
633  return false;
634  }
635  break;
636  }
637  return true;
638 }
640 {
641  // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
642  if (TransactionManager->IMSHit == false)
644  return true;
645 }
647 {
648  if (pkgAcqTransactionItem::TransactionState(state) == false)
649  return false;
650 
651  switch (state)
652  {
653  case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break;
654  case TransactionAbort:
656  {
657  // keep the compressed file, but drop the decompressed
658  EraseFileName.clear();
660  RemoveFile("TransactionAbort", PartialFile);
661  }
662  break;
663  case TransactionCommit:
664  if (EraseFileName.empty() == false)
665  RemoveFile("AcqIndex::TransactionCommit", EraseFileName);
666  break;
667  }
668  return true;
669 }
671 {
672  if (pkgAcqTransactionItem::TransactionState(state) == false)
673  return false;
674 
675  switch (state)
676  {
677  case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
678  case TransactionCommit:
679  break;
680  case TransactionAbort:
681  std::string const Partial = GetPartialFileNameFromURI(Target.URI);
682  RemoveFile("TransactionAbort", Partial);
683  break;
684  }
685 
686  return true;
687 }
688  /*}}}*/
689 // pkgAcqTransactionItem::AcquireByHash and specialisations for child classes /*{{{*/
691 {
692  if (TransactionManager->MetaIndexParser == nullptr)
693  return false;
694  auto const useByHashConf = Target.Option(IndexTarget::BY_HASH);
695  if (useByHashConf == "force")
696  return true;
697  return StringToBool(useByHashConf) == true && TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
698 }
699 // pdiff patches have a unique name already, no need for by-hash
701 {
702  return false;
703 }
705 {
706  return false;
707 }
708  /*}}}*/
709 
711 /* The sole purpose of this class is having an item which does nothing to
712  reach its done state to prevent cleanup deleting the mentioned file.
713  Handy in cases in which we know we have the file already, like IMS-Hits. */
714 {
716  public:
717  virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
719 
720  NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) :
721  pkgAcquire::Item(Owner), Target(Target)
722  {
723  Status = StatDone;
724  DestFile = GetFinalFileNameFromURI(Target.URI);
725  }
726  NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) :
727  pkgAcquire::Item(Owner), Target(Target)
728  {
729  Status = StatDone;
730  DestFile = FinalFile;
731  }
732 };
733  /*}}}*/
735 /* This class ensures that a file which was configured but isn't downloaded
736  for various reasons isn't kept in an old version in the lists directory.
737  In a way its the reverse of NoActionItem as it helps with removing files
738  even if the lists-cleanup is deactivated. */
739 {
740  public:
741  virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
743 
744  CleanupItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) :
746  {
747  Status = StatDone;
748  DestFile = GetFinalFileNameFromURI(Target.URI);
749  }
751  {
752  switch (state)
753  {
754  case TransactionStarted:
755  break;
756  case TransactionAbort:
757  break;
758  case TransactionCommit:
759  if (_config->FindB("Debug::Acquire::Transaction", false) == true)
760  std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
761  if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
762  return false;
763  break;
764  }
765  return true;
766  }
767 };
768  /*}}}*/
769 
770 // Acquire::Item::Item - Constructor /*{{{*/
771 pkgAcquire::Item::Item(pkgAcquire * const owner) :
772  FileSize(0), PartialSize(0), ID(0), Complete(false), Local(false),
773  QueueCounter(0), ExpectedAdditionalItems(0), Retries(_config->FindI("Acquire::Retries", 0)), Owner(owner), d(new Private())
774 {
775  Owner->Add(this);
776  Status = StatIdle;
777 }
778  /*}}}*/
779 // Acquire::Item::~Item - Destructor /*{{{*/
780 pkgAcquire::Item::~Item()
781 {
782  Owner->Remove(this);
783  delete d;
784 }
785  /*}}}*/
786 std::string pkgAcquire::Item::Custom600Headers() const /*{{{*/
787 {
788  std::ostringstream header;
789  for (auto const &f : d->CustomFields)
790  if (f.second.empty() == false)
791  header << '\n'
792  << f.first << ": " << f.second;
793  return header.str();
794 }
795  /*}}}*/
796 std::unordered_map<std::string, std::string> &pkgAcquire::Item::ModifyCustomFields() /*{{{*/
797 {
798  return d->CustomFields;
799 }
800  /*}}}*/
801 bool pkgAcquire::Item::PopAlternativeURI(std::string &NewURI) /*{{{*/
802 {
803  if (d->AlternativeURIs.empty())
804  return false;
805  auto const AltUri = d->AlternativeURIs.front();
806  d->AlternativeURIs.pop_front();
807  NewURI = AltUri.URI;
808  auto &CustomFields = ModifyCustomFields();
809  for (auto const &f : AltUri.changefields)
810  {
811  if (f.second.empty())
812  CustomFields.erase(f.first);
813  else
814  CustomFields[f.first] = f.second;
815  }
816  return true;
817 }
818  /*}}}*/
819 bool pkgAcquire::Item::IsGoodAlternativeURI(std::string const &AltUri) const/*{{{*/
820 {
821  return std::find(d->PastRedirections.cbegin(), d->PastRedirections.cend(), AltUri) == d->PastRedirections.cend() &&
822  std::find(d->BadAlternativeSites.cbegin(), d->BadAlternativeSites.cend(), URI::SiteOnly(AltUri)) == d->BadAlternativeSites.cend();
823 }
824  /*}}}*/
825 void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back) /*{{{*/
826 {
827  if (IsGoodAlternativeURI(NewURI) == false)
828  return;
829  if (at_the_back)
830  d->AlternativeURIs.emplace_back(std::move(NewURI), std::move(fields));
831  else
832  d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields));
833 }
834  /*}}}*/
835 void pkgAcquire::Item::RemoveAlternativeSite(std::string &&OldSite) /*{{{*/
836 {
837  d->AlternativeURIs.erase(std::remove_if(d->AlternativeURIs.begin(), d->AlternativeURIs.end(),
838  [&](decltype(*d->AlternativeURIs.cbegin()) AltUri) {
839  return URI::SiteOnly(AltUri.URI) == OldSite;
840  }),
841  d->AlternativeURIs.end());
842  d->BadAlternativeSites.push_back(std::move(OldSite));
843 }
844  /*}}}*/
845 std::string pkgAcquire::Item::ShortDesc() const /*{{{*/
846 {
847  return DescURI();
848 }
849  /*}}}*/
850 void pkgAcquire::Item::Finished() /*{{{*/
851 {
852 }
853  /*}}}*/
854 APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const /*{{{*/
855 {
856  return Owner;
857 }
858  /*}}}*/
859 APT_PURE pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc() /*{{{*/
860 {
861  return Desc;
862 }
863  /*}}}*/
864 APT_PURE bool pkgAcquire::Item::IsTrusted() const /*{{{*/
865 {
866  return false;
867 }
868  /*}}}*/
869 // Acquire::Item::Failed - Item failed to download /*{{{*/
870 // ---------------------------------------------------------------------
871 /* We return to an idle state if there are still other queues that could
872  fetch this object */
873 static void formatHashsum(std::ostream &out, HashString const &hs)
874 {
875  auto const type = hs.HashType();
876  if (type == "Checksum-FileSize")
877  out << " - Filesize";
878  else
879  out << " - " << type;
880  out << ':' << hs.HashValue();
881  if (hs.usable() == false)
882  out << " [weak]";
883  out << std::endl;
884 }
885 void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
886 {
887  if (QueueCounter <= 1)
888  {
889  /* This indicates that the file is not available right now but might
890  be sometime later. If we do a retry cycle then this should be
891  retried [CDROMs] */
892  if (Cnf != NULL && Cnf->LocalOnly == true &&
893  StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
894  {
895  Status = StatIdle;
896  Dequeue();
897  return;
898  }
899 
900  switch (Status)
901  {
902  case StatIdle:
903  case StatFetching:
904  case StatDone:
905  Status = StatError;
906  break;
907  case StatAuthError:
908  case StatError:
909  case StatTransientNetworkError:
910  break;
911  }
912  Complete = false;
913  Dequeue();
914  }
915 
916  FailMessage(Message);
917 
918  if (QueueCounter > 1)
919  Status = StatIdle;
920 }
921 void pkgAcquire::Item::FailMessage(string const &Message)
922 {
923  string const FailReason = LookupTag(Message, "FailReason");
924  enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, REDIRECTION_LOOP, OTHER } failreason = OTHER;
925  if ( FailReason == "MaximumSizeExceeded")
926  failreason = MAXIMUM_SIZE_EXCEEDED;
927  else if ( FailReason == "WeakHashSums")
928  failreason = WEAK_HASHSUMS;
929  else if (FailReason == "RedirectionLoop")
930  failreason = REDIRECTION_LOOP;
931  else if (Status == StatAuthError)
932  failreason = HASHSUM_MISMATCH;
933 
934  if(ErrorText.empty())
935  {
936  std::ostringstream out;
937  switch (failreason)
938  {
939  case HASHSUM_MISMATCH:
940  out << _("Hash Sum mismatch") << std::endl;
941  break;
942  case WEAK_HASHSUMS:
943  out << _("Insufficient information available to perform this download securely") << std::endl;
944  break;
945  case REDIRECTION_LOOP:
946  out << "Redirection loop encountered" << std::endl;
947  break;
948  case MAXIMUM_SIZE_EXCEEDED:
949  out << LookupTag(Message, "Message") << std::endl;
950  break;
951  case OTHER:
952  out << LookupTag(Message, "Message");
953  break;
954  }
955 
956  if (Status == StatAuthError)
957  {
958  auto const ExpectedHashes = GetExpectedHashes();
959  if (ExpectedHashes.empty() == false)
960  {
961  out << "Hashes of expected file:" << std::endl;
962  for (auto const &hs: ExpectedHashes)
963  formatHashsum(out, hs);
964  }
965  if (failreason == HASHSUM_MISMATCH)
966  {
967  out << "Hashes of received file:" << std::endl;
968  for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
969  {
970  std::string const tagname = std::string(*type) + "-Hash";
971  std::string const hashsum = LookupTag(Message, tagname.c_str());
972  if (hashsum.empty() == false)
973  formatHashsum(out, HashString(*type, hashsum));
974  }
975  }
976  auto const lastmod = LookupTag(Message, "Last-Modified", "");
977  if (lastmod.empty() == false)
978  out << "Last modification reported: " << lastmod << std::endl;
979  }
980  ErrorText = out.str();
981  }
982 
983  switch (failreason)
984  {
985  case MAXIMUM_SIZE_EXCEEDED: RenameOnError(MaximumSizeExceeded); break;
986  case HASHSUM_MISMATCH: RenameOnError(HashSumMismatch); break;
987  case WEAK_HASHSUMS: break;
988  case REDIRECTION_LOOP: break;
989  case OTHER: break;
990  }
991 
992  if (FailReason.empty() == false)
993  ReportMirrorFailureToCentral(*this, FailReason, ErrorText);
994  else
995  ReportMirrorFailureToCentral(*this, ErrorText, ErrorText);
996 }
997  /*}}}*/
998 // Acquire::Item::Start - Item has begun to download /*{{{*/
999 // ---------------------------------------------------------------------
1000 /* Stash status and the file size. Note that setting Complete means
1001  sub-phases of the acquire process such as decompresion are operating */
1002 void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size)
1003 {
1004  Status = StatFetching;
1005  ErrorText.clear();
1006  if (FileSize == 0 && Complete == false)
1007  FileSize = Size;
1008 }
1009  /*}}}*/
1010 // Acquire::Item::VerifyDone - check if Item was downloaded OK /*{{{*/
1011 /* Note that hash-verification is 'hardcoded' in acquire-worker and has
1012  * already passed if this method is called. */
1013 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
1014  pkgAcquire::MethodConfig const * const /*Cnf*/)
1015 {
1016  std::string const FileName = LookupTag(Message,"Filename");
1017  if (FileName.empty() == true)
1018  {
1019  Status = StatError;
1020  ErrorText = "Method gave a blank filename";
1021  return false;
1022  }
1023 
1024  return true;
1025 }
1026  /*}}}*/
1027 // Acquire::Item::Done - Item downloaded OK /*{{{*/
1028 void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes,
1029  pkgAcquire::MethodConfig const * const /*Cnf*/)
1030 {
1031  // We just downloaded something..
1032  if (FileSize == 0)
1033  {
1034  unsigned long long const downloadedSize = Hashes.FileSize();
1035  if (downloadedSize != 0)
1036  {
1037  FileSize = downloadedSize;
1038  }
1039  }
1040  Status = StatDone;
1041  ErrorText.clear();
1042  Dequeue();
1043 }
1044  /*}}}*/
1045 // Acquire::Item::Rename - Rename a file /*{{{*/
1046 // ---------------------------------------------------------------------
1047 /* This helper function is used by a lot of item methods as their final
1048  step */
1049 bool pkgAcquire::Item::Rename(string const &From,string const &To)
1050 {
1051  if (From == To || rename(From.c_str(),To.c_str()) == 0)
1052  return true;
1053 
1054  std::string S;
1055  strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
1056  From.c_str(),To.c_str());
1057  Status = StatError;
1058  if (ErrorText.empty())
1059  ErrorText = S;
1060  else
1061  ErrorText = ErrorText + ": " + S;
1062  return false;
1063 }
1064  /*}}}*/
1065 void pkgAcquire::Item::Dequeue() /*{{{*/
1066 {
1067  d->AlternativeURIs.clear();
1068  Owner->Dequeue(this);
1069 }
1070  /*}}}*/
1071 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
1072 {
1073  if (RealFileExists(DestFile))
1074  Rename(DestFile, DestFile + ".FAILED");
1075 
1076  std::string errtext;
1077  switch (error)
1078  {
1079  case HashSumMismatch:
1080  errtext = _("Hash Sum mismatch");
1081  break;
1082  case SizeMismatch:
1083  errtext = _("Size mismatch");
1084  Status = StatAuthError;
1085  break;
1086  case InvalidFormat:
1087  errtext = _("Invalid file format");
1088  Status = StatError;
1089  // do not report as usually its not the mirrors fault, but Portal/Proxy
1090  break;
1091  case SignatureError:
1092  errtext = _("Signature error");
1093  Status = StatError;
1094  break;
1095  case NotClearsigned:
1096  strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT");
1097  Status = StatAuthError;
1098  break;
1099  case MaximumSizeExceeded:
1100  // the method is expected to report a good error for this
1101  break;
1102  case PDiffError:
1103  // no handling here, done by callers
1104  break;
1105  }
1106  if (ErrorText.empty())
1107  ErrorText = errtext;
1108  return false;
1109 }
1110  /*}}}*/
1111 void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/
1112 {
1113  ActiveSubprocess = subprocess;
1114 }
1115  /*}}}*/
1116 std::string pkgAcquire::Item::HashSum() const /*{{{*/
1117 {
1118  HashStringList const hashes = GetExpectedHashes();
1119  HashString const * const hs = hashes.find(NULL);
1120  return hs != NULL ? hs->toStr() : "";
1121 }
1122  /*}}}*/
1123 bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/
1124 {
1125  // store can fail due to permission errors and the item will "loop" then
1126  if (APT::String::Startswith(NewURI, "store:"))
1127  return false;
1128  if (d->PastRedirections.empty())
1129  {
1130  d->PastRedirections.push_back(NewURI);
1131  return false;
1132  }
1133  auto const LastURI = std::prev(d->PastRedirections.end());
1134  // redirections to the same file are a way of restarting/resheduling,
1135  // individual methods will have to make sure that they aren't looping this way
1136  if (*LastURI == NewURI)
1137  return false;
1138  if (std::find(d->PastRedirections.begin(), LastURI, NewURI) != LastURI)
1139  return true;
1140  d->PastRedirections.push_back(NewURI);
1141  return false;
1142 }
1143  /*}}}*/
1144 int pkgAcquire::Item::Priority() /*{{{*/
1145 {
1146  // Stage 0: Files requested by methods
1147  // - they will usually not end up here, but if they do we make sure
1148  // to get them as soon as possible as they are probably blocking
1149  // the processing of files by the requesting method
1150  if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr)
1151  return 5000;
1152  // Stage 1: Meta indices and diff indices
1153  // - those need to be fetched first to have progress reporting working
1154  // for the rest
1155  if (dynamic_cast<pkgAcqMetaSig*>(this) != nullptr
1156  || dynamic_cast<pkgAcqMetaBase*>(this) != nullptr
1157  || dynamic_cast<pkgAcqDiffIndex*>(this) != nullptr)
1158  return 1000;
1159  // Stage 2: Diff files
1160  // - fetch before complete indexes so we can apply the diffs while fetching
1161  // larger files.
1162  if (dynamic_cast<pkgAcqIndexDiffs*>(this) != nullptr ||
1163  dynamic_cast<pkgAcqIndexMergeDiffs*>(this) != nullptr)
1164  return 800;
1165 
1166  // Stage 3: The rest - complete index files and other stuff
1167  return 500;
1168 }
1169  /*}}}*/
1170 
1172  pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) :
1173  pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager)
1174 {
1175  if (TransactionManager != this)
1176  TransactionManager->Add(this);
1177  ModifyCustomFields() = {
1178  {"Target-Site", Target.Option(IndexTarget::SITE)},
1179  {"Target-Repo-URI", Target.Option(IndexTarget::REPO_URI)},
1180  {"Target-Base-URI", Target.Option(IndexTarget::BASE_URI)},
1181  {"Target-Component", Target.Option(IndexTarget::COMPONENT)},
1182  {"Target-Release", Target.Option(IndexTarget::RELEASE)},
1183  {"Target-Architecture", Target.Option(IndexTarget::ARCHITECTURE)},
1184  {"Target-Language", Target.Option(IndexTarget::LANGUAGE)},
1185  {"Target-Type", "index"},
1186  };
1187 }
1188  /*}}}*/
1190 {
1191 }
1192  /*}}}*/
1193 HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const /*{{{*/
1194 {
1195  return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey);
1196 }
1197  /*}}}*/
1198 
1199 static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/
1200 {
1201  if (TransactionManager->IMSHit == true)
1202  return;
1203  if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
1204  {
1205  TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
1206  if (TransactionManager->LastMetaIndexParser != NULL)
1207  {
1208  _error->PushToStack();
1209  if (RealFileExists(FinalInRelease))
1210  TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
1211  else
1212  TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
1213  // its unlikely to happen, but if what we have is bad ignore it
1214  if (_error->PendingError())
1215  {
1216  delete TransactionManager->LastMetaIndexParser;
1217  TransactionManager->LastMetaIndexParser = NULL;
1218  }
1219  _error->RevertToStack();
1220  }
1221  }
1222 }
1223  /*}}}*/
1224 
1225 // AcqMetaBase - Constructor /*{{{*/
1228  IndexTarget const &DataTarget)
1229 : pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL),
1230  AuthPass(false), IMSHit(false), State(TransactionStarted)
1231 {
1232 }
1233  /*}}}*/
1234 // AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/
1236 {
1237  Transaction.push_back(I);
1238 }
1239  /*}}}*/
1240 // AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/
1242 {
1243  if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1244  std::clog << "AbortTransaction: " << TransactionManager << std::endl;
1245 
1246  switch (TransactionManager->State)
1247  {
1248  case TransactionStarted: break;
1249  case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return;
1250  case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now committed", TransactionManager->Target.URI.c_str()); return;
1251  }
1253  TransactionManager->ExpectedAdditionalItems = 0;
1254 
1255  // ensure the toplevel is in error state too
1256  for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
1257  I != Transaction.end(); ++I)
1258  {
1259  (*I)->ExpectedAdditionalItems = 0;
1260  if ((*I)->Status != pkgAcquire::Item::StatFetching)
1261  (*I)->Dequeue();
1263  }
1264  Transaction.clear();
1265 }
1266  /*}}}*/
1267 // AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/
1269 {
1270  for (std::vector<pkgAcqTransactionItem*>::const_iterator I = Transaction.begin();
1271  I != Transaction.end(); ++I)
1272  {
1273  switch((*I)->Status) {
1274  case StatDone: break;
1275  case StatIdle: break;
1276  case StatAuthError: return true;
1277  case StatError: return true;
1278  case StatTransientNetworkError: return true;
1279  case StatFetching: break;
1280  }
1281  }
1282  return false;
1283 }
1284  /*}}}*/
1285 // AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/
1287 {
1288  if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1289  std::clog << "CommitTransaction: " << this << std::endl;
1290 
1291  switch (TransactionManager->State)
1292  {
1293  case TransactionStarted: break;
1294  case TransactionAbort: _error->Fatal("Transaction %s was already committed and is now aborted", TransactionManager->Target.URI.c_str()); return;
1295  case TransactionCommit: _error->Fatal("Transaction %s was already committed and is again committed", TransactionManager->Target.URI.c_str()); return;
1296  }
1298 
1299  // move new files into place *and* remove files that are not
1300  // part of the transaction but are still on disk
1301  for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
1302  I != Transaction.end(); ++I)
1303  {
1304  (*I)->TransactionState(TransactionCommit);
1305  }
1306  Transaction.clear();
1307 }
1308  /*}}}*/
1309 // AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/
1311  const std::string &From,
1312  const std::string &To)
1313 {
1314  I->PartialFile = From;
1315  I->DestFile = To;
1316 }
1317  /*}}}*/
1318 // AcqMetaBase::TransactionStageRemoval - Stage a file for removal /*{{{*/
1320  const std::string &FinalFile)
1321 {
1322  I->PartialFile = "";
1323  I->DestFile = FinalFile;
1324 }
1325  /*}}}*/
1326 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/
1327 /* This method is called from ::Failed handlers. If it returns true,
1328  no fallback to other files or modi is performed */
1329 bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
1330 {
1331  string const Final = I->GetFinalFilename();
1332  std::string const GPGError = LookupTag(Message, "Message");
1333  if (FileExists(Final))
1334  {
1335  I->Status = StatTransientNetworkError;
1336  _error->Warning(_("An error occurred during the signature verification. "
1337  "The repository is not updated and the previous index files will be used. "
1338  "GPG error: %s: %s"),
1339  Desc.Description.c_str(),
1340  GPGError.c_str());
1341  RunScripts("APT::Update::Auth-Failure");
1342  return true;
1343  } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1344  /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1345  _error->Error(_("GPG error: %s: %s"),
1346  Desc.Description.c_str(),
1347  GPGError.c_str());
1348  I->Status = StatAuthError;
1349  return true;
1350  } else {
1351  _error->Warning(_("GPG error: %s: %s"),
1352  Desc.Description.c_str(),
1353  GPGError.c_str());
1354  }
1355  // gpgv method failed
1356  ReportMirrorFailureToCentral(*this, "GPGFailure", GPGError);
1357  return false;
1358 }
1359  /*}}}*/
1360 // AcqMetaBase::Custom600Headers - Get header for AcqMetaBase /*{{{*/
1361 // ---------------------------------------------------------------------
1363 {
1364  std::string Header = pkgAcqTransactionItem::Custom600Headers();
1365  Header.append("\nIndex-File: true");
1366  std::string MaximumSize;
1367  strprintf(MaximumSize, "\nMaximum-Size: %i",
1368  _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
1369  Header += MaximumSize;
1370 
1371  string const FinalFile = GetFinalFilename();
1372  struct stat Buf;
1373  if (stat(FinalFile.c_str(),&Buf) == 0)
1374  Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
1375 
1376  return Header;
1377 }
1378  /*}}}*/
1379 // AcqMetaBase::QueueForSignatureVerify /*{{{*/
1380 void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature)
1381 {
1382  AuthPass = true;
1383  I->Desc.URI = "gpgv:" + pkgAcquire::URIEncode(Signature);
1384  I->DestFile = File;
1385  QueueURI(I->Desc);
1386  I->SetActiveSubprocess("gpgv");
1387 }
1388  /*}}}*/
1389 // AcqMetaBase::CheckDownloadDone /*{{{*/
1390 bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const
1391 {
1392  // We have just finished downloading a Release file (it is not
1393  // verified yet)
1394 
1395  // Save the final base URI we got this Release file from
1396  if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
1397  {
1398  auto InReleasePath = Target.Option(IndexTarget::INRELEASE_PATH);
1399  if (InReleasePath.empty())
1400  InReleasePath = "InRelease";
1401 
1402  if (APT::String::Endswith(I->Desc.URI, InReleasePath))
1403  {
1404  TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - InReleasePath.length());
1405  TransactionManager->UsedMirror = I->UsedMirror;
1406  }
1407  else if (APT::String::Endswith(I->Desc.URI, "Release"))
1408  {
1409  TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release"));
1410  TransactionManager->UsedMirror = I->UsedMirror;
1411  }
1412  }
1413 
1414  std::string const FileName = LookupTag(Message,"Filename");
1415  if (FileName != I->DestFile && RealFileExists(I->DestFile) == false)
1416  {
1417  I->Local = true;
1418  I->Desc.URI = "copy:" + pkgAcquire::URIEncode(FileName);
1419  I->QueueURI(I->Desc);
1420  return false;
1421  }
1422 
1423  // make sure to verify against the right file on I-M-S hit
1424  bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
1425  if (IMSHit == false && Hashes.usable())
1426  {
1427  // detect IMS-Hits servers haven't detected by Hash comparison
1428  std::string const FinalFile = I->GetFinalFilename();
1429  if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
1430  {
1431  IMSHit = true;
1432  RemoveFile("CheckDownloadDone", I->DestFile);
1433  }
1434  }
1435 
1436  if(IMSHit == true)
1437  {
1438  // for simplicity, the transaction manager is always InRelease
1439  // even if it doesn't exist.
1440  TransactionManager->IMSHit = true;
1441  I->PartialFile = I->DestFile = I->GetFinalFilename();
1442  }
1443 
1444  // set Item to complete as the remaining work is all local (verify etc)
1445  I->Complete = true;
1446 
1447  return true;
1448 }
1449  /*}}}*/
1450 bool pkgAcqMetaBase::CheckAuthDone(string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/
1451 {
1452  /* If we work with a recent version of our gpgv method, we expect that it tells us
1453  which key(s) have signed the file so stuff like CVE-2018-0501 is harder in the future */
1454  if (Cnf->Version != "1.0" && LookupTag(Message, "Signed-By").empty())
1455  {
1456  std::string errmsg;
1457  strprintf(errmsg, "Internal Error: Signature on %s seems good, but expected details are missing! (%s)", Target.URI.c_str(), "Signed-By");
1458  if (ErrorText.empty())
1459  ErrorText = errmsg;
1460  Status = StatAuthError;
1461  return _error->Error("%s", errmsg.c_str());
1462  }
1463 
1464  // At this point, the gpgv method has succeeded, so there is a
1465  // valid signature from a key in the trusted keyring. We
1466  // perform additional verification of its contents, and use them
1467  // to verify the indexes we are about to download
1468  if (_config->FindB("Debug::pkgAcquire::Auth", false))
1469  std::cerr << "Signature verification succeeded: " << DestFile << std::endl;
1470 
1471  if (TransactionManager->IMSHit == false)
1472  {
1473  // open the last (In)Release if we have it
1474  std::string const FinalFile = GetFinalFilename();
1475  std::string FinalRelease;
1476  std::string FinalInRelease;
1477  if (APT::String::Endswith(FinalFile, "InRelease"))
1478  {
1479  FinalInRelease = FinalFile;
1480  FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
1481  }
1482  else
1483  {
1484  FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
1485  FinalRelease = FinalFile;
1486  }
1487  LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1488  }
1489 
1490  bool const GoodAuth = TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText);
1491  if (GoodAuth == false && AllowInsecureRepositories(InsecureType::WEAK, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == false)
1492  {
1493  Status = StatAuthError;
1494  return false;
1495  }
1496 
1497  if (!VerifyVendor(Message))
1498  {
1499  Status = StatAuthError;
1500  return false;
1501  }
1502 
1503  // Download further indexes with verification
1504  TransactionManager->QueueIndexes(GoodAuth);
1505 
1506  return GoodAuth;
1507 }
1508  /*}}}*/
1509 void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/
1510 {
1511  // at this point the real Items are loaded in the fetcher
1512  ExpectedAdditionalItems = 0;
1513 
1514  std::unordered_set<std::string> targetsSeen, componentsSeen;
1515  bool const hasReleaseFile = TransactionManager->MetaIndexParser != NULL;
1516  bool hasHashes = true;
1517  auto IndexTargets = TransactionManager->MetaIndexParser->GetIndexTargets();
1518  if (hasReleaseFile && verify == false)
1519  hasHashes = std::any_of(IndexTargets.begin(), IndexTargets.end(),
1520  [&](IndexTarget const &Target) { return TransactionManager->MetaIndexParser->Exists(Target.MetaKey); });
1521  if (_config->FindB("Acquire::IndexTargets::Randomized", true) && likely(IndexTargets.empty() == false))
1522  {
1523  /* For fallback handling and to have some reasonable progress information
1524  we can't randomize everything, but at least the order in the same type
1525  can be as we shouldn't be telling the mirrors (and everyone else watching)
1526  which is native/foreign arch, specific order of preference of translations, … */
1527  auto range_start = IndexTargets.begin();
1528  auto seed = (std::chrono::high_resolution_clock::now().time_since_epoch() / std::chrono::nanoseconds(1)) ^ getpid();
1529  std::default_random_engine g(seed);
1530  do {
1531  auto const type = range_start->Option(IndexTarget::CREATED_BY);
1532  auto const range_end = std::find_if_not(range_start, IndexTargets.end(),
1533  [&type](IndexTarget const &T) { return type == T.Option(IndexTarget::CREATED_BY); });
1534  std::shuffle(range_start, range_end, g);
1535  range_start = range_end;
1536  } while (range_start != IndexTargets.end());
1537  }
1538  /* Collect all components for which files exist to prevent apt from warning users
1539  about "hidden" components for which not all files exist like main/debian-installer
1540  and Translation files */
1541  if (hasReleaseFile == true)
1542  for (auto const &Target : IndexTargets)
1544  {
1545  auto component = Target.Option(IndexTarget::COMPONENT);
1546  if (component.empty() == false)
1547  componentsSeen.emplace(std::move(component));
1548  }
1549 
1550  for (auto&& Target: IndexTargets)
1551  {
1552  // if we have seen a target which is created-by a target this one here is declared a
1553  // fallback to, we skip acquiring the fallback (but we make sure we clean up)
1554  if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
1555  {
1556  targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1557  new CleanupItem(Owner, TransactionManager, Target);
1558  continue;
1559  }
1560  // all is an implementation detail. Users shouldn't use this as arch
1561  // We need this support trickery here as e.g. Debian has binary-all files already,
1562  // but arch:all packages are still in the arch:any files, so we would waste precious
1563  // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all
1564  // in the set of supported architectures, so we can filter based on this property rather
1565  // than invent an entirely new flag we would need to carry for all of eternity.
1566  if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
1567  {
1569  {
1570  new CleanupItem(Owner, TransactionManager, Target);
1571  continue;
1572  }
1573  }
1574 
1575  bool trypdiff = Target.OptionBool(IndexTarget::PDIFFS);
1576  if (hasReleaseFile == true)
1577  {
1579  {
1580  auto const component = Target.Option(IndexTarget::COMPONENT);
1581  if (component.empty() == false &&
1582  componentsSeen.find(component) == std::end(componentsSeen) &&
1584  {
1585  new CleanupItem(Owner, TransactionManager, Target);
1586  _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' doesn't have the component '%s' (component misspelt in sources.list?)"),
1587  Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), component.c_str());
1588  continue;
1589 
1590  }
1591 
1592  // optional targets that we do not have in the Release file are skipped
1593  if (hasHashes == true && Target.IsOptional)
1594  {
1595  new CleanupItem(Owner, TransactionManager, Target);
1596  continue;
1597  }
1598 
1599  std::string const &arch = Target.Option(IndexTarget::ARCHITECTURE);
1600  if (arch.empty() == false)
1601  {
1603  {
1604  new CleanupItem(Owner, TransactionManager, Target);
1605  _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"),
1606  Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str());
1607  continue;
1608  }
1609  // if the architecture is officially supported but currently no packages for it available,
1610  // ignore silently as this is pretty much the same as just shipping an empty file.
1611  // if we don't know which architectures are supported, we do NOT ignore it to notify user about this
1612  if (hasHashes == true && TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false)
1613  {
1614  new CleanupItem(Owner, TransactionManager, Target);
1615  continue;
1616  }
1617  }
1618 
1619  if (hasHashes == true)
1620  {
1621  new CleanupItem(Owner, TransactionManager, Target);
1622  _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' does not seem to provide it (sources.list entry misspelt?)"),
1623  Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
1624  continue;
1625  }
1626  else
1627  {
1628  new pkgAcqIndex(Owner, TransactionManager, Target);
1629  continue;
1630  }
1631  }
1632  else if (verify)
1633  {
1634  auto const hashes = GetExpectedHashesFor(Target.MetaKey);
1635  if (hashes.empty() == false)
1636  {
1637  if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false)
1638  {
1639  new CleanupItem(Owner, TransactionManager, Target);
1640  _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
1641  Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
1642  continue;
1643  }
1644  // empty files are skipped as acquiring the very small compressed files is a waste of time
1645  else if (hashes.FileSize() == 0)
1646  {
1647  new CleanupItem(Owner, TransactionManager, Target);
1648  targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1649  continue;
1650  }
1651  }
1652  }
1653 
1654  // autoselect the compression method
1655  std::vector<std::string> types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
1656  types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) {
1657  if (t == "uncompressed")
1658  return TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false;
1659  std::string const MetaKey = Target.MetaKey + "." + t;
1660  return TransactionManager->MetaIndexParser->Exists(MetaKey) == false;
1661  }), types.end());
1662  if (types.empty() == false)
1663  {
1664  std::ostringstream os;
1665  std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "));
1666  os << *types.rbegin();
1667  Target.Options["COMPRESSIONTYPES"] = os.str();
1668  }
1669  else
1670  Target.Options["COMPRESSIONTYPES"].clear();
1671 
1672  std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
1673  if (filename.empty() == false)
1674  {
1675  // if the Release file is a hit and we have an index it must be the current one
1676  if (TransactionManager->IMSHit == true)
1677  ;
1678  else if (TransactionManager->LastMetaIndexParser != NULL)
1679  {
1680  // see if the file changed since the last Release file
1681  // we use the uncompressed files as we might compress differently compared to the server,
1682  // so the hashes might not match, even if they contain the same data.
1685  if (newFile != oldFile)
1686  filename.clear();
1687  }
1688  else
1689  filename.clear();
1690  }
1691  else
1692  trypdiff = false; // no file to patch
1693 
1694  if (filename.empty() == false)
1695  {
1696  new NoActionItem(Owner, Target, filename);
1697  std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
1698  if (FileExists(idxfilename))
1699  new NoActionItem(Owner, Target, idxfilename);
1700  targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1701  continue;
1702  }
1703 
1704  // check if we have patches available
1706  }
1707  else
1708  {
1709  // if we have no file to patch, no point in trying
1710  trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false);
1711  }
1712 
1713  // no point in patching from local sources
1714  if (trypdiff)
1715  {
1716  std::string const proto = Target.URI.substr(0, strlen("file:/"));
1717  if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:")
1718  trypdiff = false;
1719  }
1720 
1721  // Queue the Index file (Packages, Sources, Translation-$foo, …)
1722  targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1723  if (trypdiff)
1725  else
1726  new pkgAcqIndex(Owner, TransactionManager, Target);
1727  }
1728 }
1729  /*}}}*/
1730 bool pkgAcqMetaBase::VerifyVendor(string const &) /*{{{*/
1731 {
1733  {
1734  time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil();
1735  if (invalid_since > 0)
1736  {
1737  std::string errmsg;
1738  strprintf(errmsg,
1739  // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1740  // the time since then the file is invalid - formatted in the same way as in
1741  // the download progress display (e.g. 7d 3h 42min 1s)
1742  _("Release file for %s is expired (invalid since %s). "
1743  "Updates for this repository will not be applied."),
1744  Target.URI.c_str(), TimeToStr(invalid_since).c_str());
1745  if (ErrorText.empty())
1746  ErrorText = errmsg;
1747  return _error->Error("%s", errmsg.c_str());
1748  }
1749  }
1750 
1752  {
1753  time_t const invalid_for = TransactionManager->MetaIndexParser->GetNotBefore() - time(nullptr);
1754  if (invalid_for > 0)
1755  {
1756  std::string errmsg;
1757  strprintf(errmsg,
1758  // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1759  // the time until the file will be valid - formatted in the same way as in
1760  // the download progress display (e.g. 7d 3h 42min 1s)
1761  _("Release file for %s is not valid yet (invalid for another %s). "
1762  "Updates for this repository will not be applied."),
1763  Target.URI.c_str(), TimeToStr(invalid_for).c_str());
1764  if (ErrorText.empty())
1765  ErrorText = errmsg;
1766  return _error->Error("%s", errmsg.c_str());
1767  }
1768  }
1769 
1770  /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
1771  as a prevention of downgrading us to older (still valid) files */
1774  {
1775  TransactionManager->IMSHit = true;
1776  RemoveFile("VerifyVendor", DestFile);
1777  PartialFile = DestFile = GetFinalFilename();
1778  // load the 'old' file in the 'new' one instead of flipping pointers as
1779  // the new one isn't owned by us, while the old one is so cleanup would be confused.
1783  }
1784 
1785  if (_config->FindB("Debug::pkgAcquire::Auth", false))
1786  {
1787  std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl;
1788  std::cerr << "Got Suite: " << TransactionManager->MetaIndexParser->GetSuite() << std::endl;
1789  std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl;
1790  }
1791 
1792  // One day that might become fatal…
1793  auto const ExpectedDist = TransactionManager->MetaIndexParser->GetExpectedDist();
1794  auto const NowCodename = TransactionManager->MetaIndexParser->GetCodename();
1795  if (TransactionManager->MetaIndexParser->CheckDist(ExpectedDist) == false)
1796  _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1797  Desc.Description.c_str(), ExpectedDist.c_str(), NowCodename.c_str());
1798 
1799  // changed info potentially breaks user config like pinning
1800  if (TransactionManager->LastMetaIndexParser != nullptr)
1801  {
1802  std::vector<pkgAcquireStatus::ReleaseInfoChange> Changes;
1803  auto const AllowInfoChange = _config->FindB("Acquire::AllowReleaseInfoChange", false);
1804  auto const quietInfoChange = _config->FindB("quiet::ReleaseInfoChange", false);
1805  struct {
1806  char const * const Type;
1807  bool const Allowed;
1808  decltype(&metaIndex::GetOrigin) const Getter;
1809  } checkers[] = {
1810  { "Origin", AllowInfoChange, &metaIndex::GetOrigin },
1811  { "Label", AllowInfoChange, &metaIndex::GetLabel },
1812  { "Version", true, &metaIndex::GetVersion }, // numbers change all the time, that is okay
1813  { "Suite", true, &metaIndex::GetSuite },
1814  { "Codename", AllowInfoChange, &metaIndex::GetCodename },
1815  { nullptr, false, nullptr }
1816  };
1817  auto const CheckReleaseInfo = [&](char const * const Type, bool const AllowChange, decltype(checkers[0].Getter) const Getter) {
1818  std::string const Last = (TransactionManager->LastMetaIndexParser->*Getter)();
1819  std::string const Now = (TransactionManager->MetaIndexParser->*Getter)();
1820  if (Last == Now)
1821  return;
1822  auto const Allow = _config->FindB(std::string("Acquire::AllowReleaseInfoChange::").append(Type), AllowChange);
1823  if (Allow == true && _config->FindB(std::string("quiet::ReleaseInfoChange::").append(Type), quietInfoChange) == true)
1824  return;
1825  std::string msg;
1826  strprintf(msg, _("Repository '%s' changed its '%s' value from '%s' to '%s'"),
1827  Desc.Description.c_str(), Type, Last.c_str(), Now.c_str());
1828  Changes.push_back({Type, std::move(Last), std::move(Now), std::move(msg), Allow});
1829  };
1830  for (short i = 0; checkers[i].Type != nullptr; ++i)
1831  CheckReleaseInfo(checkers[i].Type, checkers[i].Allowed, checkers[i].Getter);
1832 
1833  {
1835  auto const Now = TransactionManager->MetaIndexParser->GetDefaultPin();
1836  if (Last != Now)
1837  {
1838  auto const Allow = _config->FindB("Acquire::AllowReleaseInfoChange::DefaultPin", AllowInfoChange);
1839  if (Allow == false || _config->FindB("quiet::ReleaseInfoChange::DefaultPin", quietInfoChange) == false)
1840  {
1841  std::string msg;
1842  strprintf(msg, _("Repository '%s' changed its default priority for %s from %hi to %hi."),
1843  Desc.Description.c_str(), "apt_preferences(5)", Last, Now);
1844  Changes.push_back({"DefaultPin", std::to_string(Last), std::to_string(Now), std::move(msg), Allow});
1845  }
1846  }
1847  }
1848  if (Changes.empty() == false)
1849  {
1850  auto const notes = TransactionManager->MetaIndexParser->GetReleaseNotes();
1851  if (notes.empty() == false)
1852  {
1853  std::string msg;
1854  // TRANSLATOR: the "this" refers to changes in the repository like a new release or owner change
1855  strprintf(msg, _("More information about this can be found online in the Release notes at: %s"), notes.c_str());
1856  Changes.push_back({"Release-Notes", "", std::move(notes), std::move(msg), true});
1857  }
1858  if (std::any_of(Changes.begin(),Changes.end(),[](pkgAcquireStatus::ReleaseInfoChange const &c) { return c.DefaultAction == false; }))
1859  {
1860  std::string msg;
1861  // TRANSLATOR: %s is the name of the manpage in question, e.g. apt-secure(8)
1862  strprintf(msg, _("This must be accepted explicitly before updates for "
1863  "this repository can be applied. See %s manpage for details."), "apt-secure(8)");
1864  Changes.push_back({"Confirmation", "", "", std::move(msg), true});
1865  }
1866 
1867  }
1868  if (Owner->Log == nullptr)
1869  return pkgAcquireStatus::ReleaseInfoChangesAsGlobalErrors(std::move(Changes));
1870  return Owner->Log->ReleaseInfoChanges(TransactionManager->LastMetaIndexParser, TransactionManager->MetaIndexParser, std::move(Changes));
1871  }
1872  return true;
1873 }
1874  /*}}}*/
1876 {
1877 }
1878 
1880  IndexTarget const &ClearsignedTarget,
1881  IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
1882  metaIndex * const MetaIndexParser) :
1883  pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),
1884  d(NULL), DetachedDataTarget(DetachedDataTarget),
1885  MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
1886 {
1887  // index targets + (worst case:) Release/Release.gpg
1888  ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max();
1889  TransactionManager->Add(this);
1890 }
1891  /*}}}*/
1893 {
1894  if (LastMetaIndexParser != NULL)
1895  delete LastMetaIndexParser;
1896 }
1897  /*}}}*/
1898 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1900 {
1901  string Header = pkgAcqMetaBase::Custom600Headers();
1902  Header += "\nFail-Ignore: true";
1903  std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
1904  if (key.empty() == false)
1905  Header += "\nSigned-By: " + key;
1906 
1907  return Header;
1908 }
1909  /*}}}*/
1911 {
1912  if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1913  std::clog << "Finished: " << DestFile <<std::endl;
1917 }
1918  /*}}}*/
1919 bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/
1920  pkgAcquire::MethodConfig const * const Cnf)
1921 {
1922  if (Item::VerifyDone(Message, Cnf) == false)
1923  return false;
1924 
1925  if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
1926  return RenameOnError(NotClearsigned);
1927 
1928  return true;
1929 }
1930  /*}}}*/
1931 // pkgAcqMetaClearSig::Done - We got a file /*{{{*/
1932 void pkgAcqMetaClearSig::Done(std::string const &Message,
1933  HashStringList const &Hashes,
1934  pkgAcquire::MethodConfig const * const Cnf)
1935 {
1936  Item::Done(Message, Hashes, Cnf);
1937 
1938  if(AuthPass == false)
1939  {
1940  if(CheckDownloadDone(this, Message, Hashes) == true)
1941  QueueForSignatureVerify(this, DestFile, DestFile);
1942  return;
1943  }
1944  else if (CheckAuthDone(Message, Cnf) == true)
1945  {
1946  if (TransactionManager->IMSHit == false)
1948  else if (RealFileExists(GetFinalFilename()) == false)
1949  {
1950  // We got an InRelease file IMSHit, but we haven't one, which means
1951  // we had a valid Release/Release.gpg combo stepping in, which we have
1952  // to 'acquire' now to ensure list cleanup isn't removing them
1953  new NoActionItem(Owner, DetachedDataTarget);
1954  new NoActionItem(Owner, DetachedSigTarget);
1955  }
1956  }
1957  else if (Status != StatAuthError)
1958  {
1959  string const FinalFile = GetFinalFileNameFromURI(DetachedDataTarget.URI);
1960  string const OldFile = GetFinalFilename();
1961  if (TransactionManager->IMSHit == false)
1962  TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
1963  else if (RealFileExists(OldFile) == false)
1964  new NoActionItem(Owner, DetachedDataTarget);
1965  else
1966  TransactionManager->TransactionStageCopy(this, OldFile, FinalFile);
1967  }
1968 }
1969  /*}}}*/
1970 void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/
1971 {
1972  Item::Failed(Message, Cnf);
1973 
1974  if (AuthPass == false)
1975  {
1976  if (Status == StatTransientNetworkError)
1977  {
1979  return;
1980  }
1981  auto const failreason = LookupTag(Message, "FailReason");
1982  auto const httperror = "HttpError";
1983  if (Status == StatAuthError ||
1984  Target.Option(IndexTarget::INRELEASE_PATH).empty() == false || /* do not fallback if InRelease was requested */
1985  (strncmp(failreason.c_str(), httperror, strlen(httperror)) == 0 &&
1986  failreason != "HttpError404"))
1987  {
1988  // if we expected a ClearTextSignature (InRelease) but got a network
1989  // error or got a file, but it wasn't valid, we end up here (see VerifyDone).
1990  // As these is usually called by web-portals we do not try Release/Release.gpg
1991  // as this is going to fail anyway and instead abort our try (LP#346386)
1992  _error->PushToStack();
1993  _error->Error(_("Failed to fetch %s %s"), Target.URI.c_str(), ErrorText.c_str());
1994  if (Target.Option(IndexTarget::INRELEASE_PATH).empty() == true && AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1995  _error->RevertToStack();
1996  else
1997  return;
1998  }
1999 
2000  // Queue the 'old' InRelease file for removal if we try Release.gpg
2001  // as otherwise the file will stay around and gives a false-auth
2002  // impression (CVE-2012-0214)
2004  Status = StatDone;
2005 
2007  }
2008  else
2009  {
2010  if(CheckStopAuthentication(this, Message))
2011  return;
2012 
2013  if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
2014  {
2015  Status = StatDone;
2016 
2017  /* InRelease files become Release files, otherwise
2018  * they would be considered as trusted later on */
2019  string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI);
2020  string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI);
2021  string const FinalInRelease = GetFinalFilename();
2022  Rename(DestFile, PartialRelease);
2023  TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
2024  LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
2025 
2026  // we parse the indexes here because at this point the user wanted
2027  // a repository that may potentially harm him
2028  if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
2029  /* expired Release files are still a problem you need extra force for */;
2030  else
2032  }
2033  }
2034 }
2035  /*}}}*/
2036 
2039  IndexTarget const &DataTarget,
2040  IndexTarget const &DetachedSigTarget) :
2041  pkgAcqMetaBase(Owner, TransactionManager, DataTarget), d(NULL),
2042  DetachedSigTarget(DetachedSigTarget)
2043 {
2044  if(_config->FindB("Debug::Acquire::Transaction", false) == true)
2045  std::clog << "New pkgAcqMetaIndex with TransactionManager "
2046  << this->TransactionManager << std::endl;
2047 
2048  DestFile = GetPartialFileNameFromURI(DataTarget.URI);
2049 
2050  // Create the item
2051  Desc.Description = DataTarget.Description;
2052  Desc.Owner = this;
2053  Desc.ShortDesc = DataTarget.ShortDesc;
2054 
2055  // Rewrite the description URI if INRELEASE_PATH was specified so
2056  // we download the specified file instead.
2057  auto InReleasePath = DataTarget.Option(IndexTarget::INRELEASE_PATH);
2058  if (InReleasePath.empty() == false && APT::String::Endswith(DataTarget.URI, "/InRelease"))
2059  {
2060  Desc.URI = DataTarget.URI.substr(0, DataTarget.URI.size() - strlen("InRelease")) + InReleasePath;
2061  }
2062  else
2063  {
2064  Desc.URI = DataTarget.URI;
2065  }
2066 
2067  QueueURI(Desc);
2068 }
2069  /*}}}*/
2070 void pkgAcqMetaIndex::Done(string const &Message, /*{{{*/
2071  HashStringList const &Hashes,
2072  pkgAcquire::MethodConfig const * const Cfg)
2073 {
2074  Item::Done(Message,Hashes,Cfg);
2075 
2076  if(CheckDownloadDone(this, Message, Hashes))
2077  {
2078  // we have a Release file, now download the Signature, all further
2079  // verify/queue for additional downloads will be done in the
2080  // pkgAcqMetaSig::Done() code
2082  }
2083 }
2084  /*}}}*/
2085 // pkgAcqMetaIndex::Failed - no Release file present /*{{{*/
2086 void pkgAcqMetaIndex::Failed(string const &Message,
2087  pkgAcquire::MethodConfig const * const Cnf)
2088 {
2089  pkgAcquire::Item::Failed(Message, Cnf);
2090  Status = StatDone;
2091 
2092  // No Release file was present so fall
2093  // back to queueing Packages files without verification
2094  // only allow going further if the user explicitly wants it
2095  if(AllowInsecureRepositories(InsecureType::NORELEASE, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
2096  {
2097  // ensure old Release files are removed
2099 
2100  // queue without any kind of hashsum support
2102  }
2103 }
2104  /*}}}*/
2105 std::string pkgAcqMetaIndex::DescURI() const /*{{{*/
2106 {
2107  return Target.URI;
2108 }
2109  /*}}}*/
2111 
2112 // AcqMetaSig::AcqMetaSig - Constructor /*{{{*/
2115  IndexTarget const &Target,
2116  pkgAcqMetaIndex * const MetaIndex) :
2117  pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex)
2118 {
2119  DestFile = GetPartialFileNameFromURI(Target.URI);
2120 
2121  // remove any partial downloaded sig-file in partial/.
2122  // it may confuse proxies and is too small to warrant a
2123  // partial download anyway
2124  RemoveFile("pkgAcqMetaSig", DestFile);
2125 
2126  // set the TransactionManager
2127  if(_config->FindB("Debug::Acquire::Transaction", false) == true)
2128  std::clog << "New pkgAcqMetaSig with TransactionManager "
2129  << TransactionManager << std::endl;
2130 
2131  // Create the item
2132  Desc.Description = Target.Description;
2133  Desc.Owner = this;
2134  Desc.ShortDesc = Target.ShortDesc;
2135  Desc.URI = Target.URI;
2136 
2137  // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
2138  // so we skip the download step and go instantly to verification
2140  {
2141  Complete = true;
2142  Status = StatDone;
2143  PartialFile = DestFile = GetFinalFilename();
2144  MetaIndexFileSignature = DestFile;
2145  MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
2146  }
2147  else
2148  QueueURI(Desc);
2149 }
2150  /*}}}*/
2152 {
2153 }
2154  /*}}}*/
2155 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
2157 {
2158  std::string Header = pkgAcqTransactionItem::Custom600Headers();
2159  std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
2160  if (key.empty() == false)
2161  Header += "\nSigned-By: " + key;
2162  return Header;
2163 }
2164  /*}}}*/
2165 // AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/
2166 void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes,
2167  pkgAcquire::MethodConfig const * const Cfg)
2168 {
2169  if (MetaIndexFileSignature.empty() == false)
2170  {
2171  DestFile = MetaIndexFileSignature;
2172  MetaIndexFileSignature.clear();
2173  }
2174  Item::Done(Message, Hashes, Cfg);
2175 
2176  if(MetaIndex->AuthPass == false)
2177  {
2178  if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
2179  {
2180  // destfile will be modified to point to MetaIndexFile for the
2181  // gpgv method, so we need to save it here
2182  MetaIndexFileSignature = DestFile;
2183  MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
2184  }
2185  return;
2186  }
2187  else if (MetaIndex->CheckAuthDone(Message, Cfg) == true)
2188  {
2189  auto const Releasegpg = GetFinalFilename();
2190  auto const Release = MetaIndex->GetFinalFilename();
2191  // if this is an IMS-Hit on Release ensure we also have the Release.gpg file stored
2192  // (previously an unknown pubkey) – but only if the Release file exists locally (unlikely
2193  // event of InRelease removed from the mirror causing fallback but still an IMS-Hit)
2194  if (TransactionManager->IMSHit == false ||
2195  (FileExists(Releasegpg) == false && FileExists(Release) == true))
2196  {
2197  TransactionManager->TransactionStageCopy(this, DestFile, Releasegpg);
2199  }
2200  }
2201  else if (MetaIndex->Status != StatAuthError)
2202  {
2203  std::string const FinalFile = MetaIndex->GetFinalFilename();
2204  if (TransactionManager->IMSHit == false)
2206  else
2207  TransactionManager->TransactionStageCopy(MetaIndex, FinalFile, FinalFile);
2208  }
2209 }
2210  /*}}}*/
2211 void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2212 {
2213  Item::Failed(Message,Cnf);
2214 
2215  // check if we need to fail at this point
2216  if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
2217  return;
2218 
2219  // ensures that a Release.gpg file in the lists/ is removed by the transaction
2220  if (not MetaIndexFileSignature.empty())
2221  {
2222  DestFile = MetaIndexFileSignature;
2223  MetaIndexFileSignature.clear();
2224  }
2226 
2227  // only allow going further if the user explicitly wants it
2229  {
2230  string const FinalRelease = MetaIndex->GetFinalFilename();
2231  string const FinalInRelease = TransactionManager->GetFinalFilename();
2232  LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
2233 
2234  // we parse the indexes here because at this point the user wanted
2235  // a repository that may potentially harm him
2236  bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText);
2237  if (MetaIndex->VerifyVendor(Message) == false)
2238  /* expired Release files are still a problem you need extra force for */;
2239  else
2240  TransactionManager->QueueIndexes(GoodLoad);
2241 
2242  TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalRelease);
2243  }
2244  else if (TransactionManager->IMSHit == false)
2245  Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
2246 
2247  // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
2248  if (Cnf->LocalOnly == true ||
2249  StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
2250  {
2251  // Ignore this
2252  Status = StatDone;
2253  }
2254 }
2255  /*}}}*/
2256 
2257 
2258 // AcqBaseIndex - Constructor /*{{{*/
2261  IndexTarget const &Target)
2262 : pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL)
2263 {
2264 }
2265  /*}}}*/
2266 void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2267 {
2268  pkgAcquire::Item::Failed(Message, Cnf);
2269  if (Status != StatAuthError)
2270  return;
2271 
2272  ErrorText.append("Release file created at: ");
2273  auto const timespec = TransactionManager->MetaIndexParser->GetDate();
2274  if (timespec == 0)
2275  ErrorText.append("<unknown>");
2276  else
2277  ErrorText.append(TimeRFC1123(timespec, true));
2278  ErrorText.append("\n");
2279 }
2280  /*}}}*/
2282 
2283 // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/
2284 // ---------------------------------------------------------------------
2285 /* Get the DiffIndex file first and see if there are patches available
2286  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
2287  * patches. If anything goes wrong in that process, it will fall back to
2288  * the original packages file
2289  */
2292  IndexTarget const &Target)
2293  : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL)
2294 {
2295  // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
2296  ExpectedAdditionalItems = 40;
2297  Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2298 
2299  CompressionExtensions.clear();
2300  {
2301  std::vector<std::string> types = APT::Configuration::getCompressionTypes();
2302  if (types.empty() == false)
2303  {
2304  std::ostringstream os;
2305  std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) {
2306  if (type == "uncompressed")
2307  return true;
2308  return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type);
2309  });
2310  os << *types.rbegin();
2311  CompressionExtensions = os.str();
2312  }
2313  }
2315 
2316  if(Debug)
2317  std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
2318 }
2319  /*}}}*/
2321 {
2322  // list cleanup needs to know that this file as well as the already
2323  // present index is ours, so we create an empty diff to save it for us
2325 }
2326  /*}}}*/
2327 static bool RemoveFileForBootstrapLinking(std::string &ErrorText, std::string const &For, std::string const &Boot)/*{{{*/
2328 {
2329  if (FileExists(Boot) && RemoveFile("Bootstrap-linking", Boot) == false)
2330  {
2331  strprintf(ErrorText, "Bootstrap for patching %s by removing stale %s failed!", For.c_str(), Boot.c_str());
2332  return false;
2333  }
2334  return true;
2335 }
2336  /*}}}*/
2337 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
2338 {
2339  available_patches.clear();
2340  ExpectedAdditionalItems = 0;
2341  // failing here is fine: our caller will take care of trying to
2342  // get the complete file if patching fails
2343  if(Debug)
2344  std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
2345  << std::endl;
2346 
2347  FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension);
2348  pkgTagFile TF(&Fd);
2349  if (Fd.IsOpen() == false || Fd.Failed())
2350  return false;
2351 
2352  pkgTagSection Tags;
2353  if(unlikely(TF.Step(Tags) == false))
2354  return false;
2355 
2356  HashStringList ServerHashes;
2357  unsigned long long ServerSize = 0;
2358 
2359  auto const &posix = std::locale::classic();
2360  for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
2361  {
2362  std::string tagname = *type;
2363  tagname.append("-Current");
2364  std::string const tmp = Tags.FindS(tagname.c_str());
2365  if (tmp.empty() == true)
2366  continue;
2367 
2368  string hash;
2369  unsigned long long size;
2370  std::stringstream ss(tmp);
2371  ss.imbue(posix);
2372  ss >> hash >> size;
2373  if (unlikely(hash.empty() == true))
2374  continue;
2375  if (unlikely(ServerSize != 0 && ServerSize != size))
2376  continue;
2377  ServerHashes.push_back(HashString(*type, hash));
2378  ServerSize = size;
2379  }
2380 
2381  if (ServerHashes.usable() == false)
2382  {
2383  ErrorText = "Did not find a good hashsum in the index";
2384  return false;
2385  }
2386 
2387  std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI);
2388  HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
2389  if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes)
2390  {
2391  ErrorText = "Index has different hashes than parser (probably older)";
2392  return false;
2393  }
2394 
2395  HashStringList LocalHashes;
2396  // try avoiding calculating the hash here as this is costly
2399  if (LocalHashes.usable() == false)
2400  {
2401  FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto);
2402  Hashes LocalHashesCalc(ServerHashes);
2403  LocalHashesCalc.AddFD(fd);
2404  LocalHashes = LocalHashesCalc.GetHashStringList();
2405  }
2406 
2407  if (ServerHashes == LocalHashes)
2408  {
2409  available_patches.clear();
2410  return true;
2411  }
2412 
2413  if(Debug)
2414  std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
2415  << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
2416 
2417  // historically, older hashes have more info than newer ones, so start
2418  // collecting with older ones first to avoid implementing complicated
2419  // information merging techniques… a failure is after all always
2420  // recoverable with a complete file and hashes aren't changed that often.
2421  std::vector<char const *> types;
2422  for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
2423  types.push_back(*type);
2424 
2425  // parse all of (provided) history
2426  bool firstAcceptedHashes = true;
2427  for (auto type = types.crbegin(); type != types.crend(); ++type)
2428  {
2429  if (LocalHashes.find(*type) == NULL)
2430  continue;
2431 
2432  std::string tagname = *type;
2433  tagname.append("-History");
2434  std::string const tmp = Tags.FindS(tagname.c_str());
2435  if (tmp.empty() == true)
2436  continue;
2437 
2438  string hash, filename;
2439  unsigned long long size;
2440  std::stringstream ss(tmp);
2441  ss.imbue(posix);
2442 
2443  while (ss >> hash >> size >> filename)
2444  {
2445  if (unlikely(hash.empty() == true || filename.empty() == true))
2446  continue;
2447 
2448  // see if we have a record for this file already
2449  std::vector<DiffInfo>::iterator cur = available_patches.begin();
2450  for (; cur != available_patches.end(); ++cur)
2451  {
2452  if (cur->file != filename)
2453  continue;
2454  cur->result_hashes.push_back(HashString(*type, hash));
2455  break;
2456  }
2457  if (cur != available_patches.end())
2458  continue;
2459  if (firstAcceptedHashes == true)
2460  {
2461  DiffInfo next;
2462  next.file = filename;
2463  next.result_hashes.push_back(HashString(*type, hash));
2464  next.result_hashes.FileSize(size);
2465  available_patches.push_back(next);
2466  }
2467  else
2468  {
2469  if (Debug == true)
2470  std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2471  << " wasn't in the list for the first parsed hash! (history)" << std::endl;
2472  break;
2473  }
2474  }
2475  firstAcceptedHashes = false;
2476  }
2477 
2478  if (unlikely(available_patches.empty() == true))
2479  {
2480  ErrorText = "Couldn't find any patches for the patch series";
2481  return false;
2482  }
2483 
2484  for (auto type = types.crbegin(); type != types.crend(); ++type)
2485  {
2486  if (LocalHashes.find(*type) == NULL)
2487  continue;
2488 
2489  std::string tagname = *type;
2490  tagname.append("-Patches");
2491  std::string const tmp = Tags.FindS(tagname.c_str());
2492  if (tmp.empty() == true)
2493  continue;
2494 
2495  string hash, filename;
2496  unsigned long long size;
2497  std::stringstream ss(tmp);
2498  ss.imbue(posix);
2499 
2500  while (ss >> hash >> size >> filename)
2501  {
2502  if (unlikely(hash.empty() == true || filename.empty() == true))
2503  continue;
2504 
2505  // see if we have a record for this file already
2506  std::vector<DiffInfo>::iterator cur = available_patches.begin();
2507  for (; cur != available_patches.end(); ++cur)
2508  {
2509  if (cur->file != filename)
2510  continue;
2511  if (cur->patch_hashes.empty())
2512  cur->patch_hashes.FileSize(size);
2513  cur->patch_hashes.push_back(HashString(*type, hash));
2514  break;
2515  }
2516  if (cur != available_patches.end())
2517  continue;
2518  if (Debug == true)
2519  std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2520  << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
2521  break;
2522  }
2523  }
2524 
2525  for (auto type = types.crbegin(); type != types.crend(); ++type)
2526  {
2527  std::string tagname = *type;
2528  tagname.append("-Download");
2529  std::string const tmp = Tags.FindS(tagname.c_str());
2530  if (tmp.empty() == true)
2531  continue;
2532 
2533  string hash, filename;
2534  unsigned long long size;
2535  std::stringstream ss(tmp);
2536  ss.imbue(posix);
2537 
2538  // FIXME: all of pdiff supports only .gz compressed patches
2539  while (ss >> hash >> size >> filename)
2540  {
2541  if (unlikely(hash.empty() == true || filename.empty() == true))
2542  continue;
2543  if (unlikely(APT::String::Endswith(filename, ".gz") == false))
2544  continue;
2545  filename.erase(filename.length() - 3);
2546 
2547  // see if we have a record for this file already
2548  std::vector<DiffInfo>::iterator cur = available_patches.begin();
2549  for (; cur != available_patches.end(); ++cur)
2550  {
2551  if (cur->file != filename)
2552  continue;
2553  if (cur->download_hashes.empty())
2554  cur->download_hashes.FileSize(size);
2555  cur->download_hashes.push_back(HashString(*type, hash));
2556  break;
2557  }
2558  if (cur != available_patches.end())
2559  continue;
2560  if (Debug == true)
2561  std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2562  << " wasn't in the list for the first parsed hash! (download)" << std::endl;
2563  break;
2564  }
2565  }
2566 
2567  {
2568  auto const foundStart = std::find_if(available_patches.rbegin(), available_patches.rend(),
2569  [&](auto const &cur) { return LocalHashes == cur.result_hashes; });
2570  if (foundStart == available_patches.rend() || unlikely(available_patches.empty()))
2571  {
2572  ErrorText = "Couldn't find the start of the patch series";
2573  return false;
2574  }
2575  available_patches.erase(available_patches.begin(), std::prev(foundStart.base()));
2576 
2577  auto const patch = std::find_if(available_patches.cbegin(), available_patches.cend(), [](auto const &patch) {
2578  return not patch.result_hashes.usable() ||
2579  not patch.patch_hashes.usable() ||
2580  not patch.download_hashes.usable();
2581  });
2582  if (patch != available_patches.cend())
2583  {
2584  strprintf(ErrorText, "Provides no usable hashes for %s", patch->file.c_str());
2585  return false;
2586  }
2587  }
2588 
2589  // patching with too many files is rather slow compared to a fast download
2590  unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
2591  if (fileLimit != 0 && fileLimit < available_patches.size())
2592  {
2593  strprintf(ErrorText, "Need %lu diffs, but limit is %lu", available_patches.size(), fileLimit);
2594  return false;
2595  }
2596 
2597  /* decide if we should download patches one by one or in one go:
2598  The first is good if the server merges patches, but many don't so client
2599  based merging can be attempt in which case the second is better.
2600  "bad things" will happen if patches are merged on the server,
2601  but client side merging is attempt as well */
2602  pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
2603  if (pdiff_merge == true)
2604  {
2605  // reprepro and dak add this flag if they merge patches on the server
2606  std::string const precedence = Tags.FindS("X-Patch-Precedence");
2607  pdiff_merge = (precedence != "merged");
2608  }
2609 
2610  // calculate the size of all patches we have to get
2611  unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
2612  if (sizeLimitPercent > 0)
2613  {
2614  unsigned long long downloadSize = 0;
2615  if (pdiff_merge)
2616  downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu,
2617  [](unsigned long long const T, DiffInfo const &I) {
2618  return T + I.download_hashes.FileSize();
2619  });
2620  // if server-side merging, assume we will need only the first patch
2621  else if (not available_patches.empty())
2622  downloadSize = available_patches.front().download_hashes.FileSize();
2623  if (downloadSize != 0)
2624  {
2625  unsigned long long downloadSizeIdx = 0;
2626  auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
2627  for (auto const &t : types)
2628  {
2629  std::string MetaKey = Target.MetaKey;
2630  if (t != "uncompressed")
2631  MetaKey += '.' + t;
2632  HashStringList const hsl = GetExpectedHashesFor(MetaKey);
2633  if (unlikely(hsl.usable() == false))
2634  continue;
2635  downloadSizeIdx = hsl.FileSize();
2636  break;
2637  }
2638  unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
2639  if ((sizeLimit/100) < downloadSize)
2640  {
2641  strprintf(ErrorText, "Need %llu compressed bytes, but limit is %llu and original is %llu", downloadSize, (sizeLimit/100), downloadSizeIdx);
2642  return false;
2643  }
2644  }
2645  }
2646 
2647  // clean the plate
2648  {
2649  std::string const Final = GetExistingFilename(CurrentPackagesFile);
2650  if (unlikely(Final.empty())) // because we wouldn't be called in such a case
2651  return false;
2652  std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);
2653  std::string const PatchedFile = GetKeepCompressedFileName(PartialFile + "-patched", Target);
2654  if (not RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile) ||
2655  not RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile))
2656  return false;
2657  {
2658  auto const exts = APT::Configuration::getCompressorExtensions();
2659  if (not std::all_of(exts.cbegin(), exts.cend(), [&](auto const &ext) {
2660  return RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile + ext) &&
2661  RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile + ext);
2662  }))
2663  return false;
2664  }
2665  std::string const Ext = Final.substr(CurrentPackagesFile.length());
2666  std::string const Partial = PartialFile + Ext;
2667  if (symlink(Final.c_str(), Partial.c_str()) != 0)
2668  {
2669  strprintf(ErrorText, "Bootstrap for patching by linking %s to %s failed!", Final.c_str(), Partial.c_str());
2670  return false;
2671  }
2672  }
2673 
2674  return true;
2675 }
2676  /*}}}*/
2677 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2678 {
2679  if (CommonFailed(GetDiffIndexURI(Target), Message, Cnf))
2680  return;
2681 
2682  RenameOnError(PDiffError);
2683  Status = StatDone;
2684  ExpectedAdditionalItems = 0;
2685 
2686  if(Debug)
2687  std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
2688  << "Falling back to normal index file acquire" << std::endl;
2689 
2690  new pkgAcqIndex(Owner, TransactionManager, Target);
2691 }
2692  /*}}}*/
2693 bool pkgAcqDiffIndex::VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const)/*{{{*/
2694 {
2695  string const FinalFile = GetFinalFilename();
2696  if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
2697  DestFile = FinalFile;
2698 
2699  if (ParseDiffIndex(DestFile))
2700  return true;
2701 
2702  Status = StatError;
2703  if (ErrorText.empty())
2704  ErrorText = "Couldn't parse pdiff index";
2705  return false;
2706 }
2707  /*}}}*/
2708 void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes, /*{{{*/
2709  pkgAcquire::MethodConfig const * const Cnf)
2710 {
2711  if(Debug)
2712  std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
2713 
2714  Item::Done(Message, Hashes, Cnf);
2715 
2716  if (available_patches.empty())
2717  {
2718  // we have the same sha1 as the server so we are done here
2719  if(Debug)
2720  std::clog << "pkgAcqDiffIndex: Package file is up-to-date" << std::endl;
2721  QueueOnIMSHit();
2722  }
2723  else
2724  {
2725  if (pdiff_merge == false)
2727  else
2728  {
2729  diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
2730  for(size_t i = 0; i < available_patches.size(); ++i)
2731  (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
2732  Target,
2733  available_patches[i],
2734  diffs);
2735  }
2736  }
2737 
2739 
2740  Complete = true;
2741  Status = StatDone;
2742  Dequeue();
2743 
2744  return;
2745 }
2746  /*}}}*/
2748 {
2749  if (diffs != NULL)
2750  delete diffs;
2751 }
2752 
2753 // AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/
2754 // ---------------------------------------------------------------------
2755 /* The package diff is added to the queue. one object is constructed
2756  * for each diff and the index
2757  */
2760  IndexTarget const &Target,
2761  vector<DiffInfo> const &diffs)
2762  : pkgAcqBaseIndex(Owner, TransactionManager, Target),
2763  available_patches(diffs)
2764 {
2766 
2767  Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2768 
2769  Desc.Owner = this;
2770  Desc.ShortDesc = Target.ShortDesc;
2771 
2772  if(available_patches.empty() == true)
2773  {
2774  // we are done (yeah!), check hashes against the final file
2776  Finish(true);
2777  }
2778  else
2779  {
2781  QueueNextDiff();
2782  }
2783 }
2784  /*}}}*/
2785 void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2786 {
2787  pkgAcqBaseIndex::Failed(Message,Cnf);
2788  Status = StatDone;
2789 
2791  if(Debug)
2792  std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
2793  << "Falling back to normal index file acquire " << std::endl;
2794  RenameOnError(PDiffError);
2795  std::string const patchname = GetDiffsPatchFileName(DestFile);
2796  if (RealFileExists(patchname))
2797  Rename(patchname, patchname + ".FAILED");
2798  std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2799  if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
2800  Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
2801  new pkgAcqIndex(Owner, TransactionManager, Target);
2802  Finish();
2803 }
2804  /*}}}*/
2805 // Finish - helper that cleans the item out of the fetcher queue /*{{{*/
2806 void pkgAcqIndexDiffs::Finish(bool allDone)
2807 {
2808  if(Debug)
2809  std::clog << "pkgAcqIndexDiffs::Finish(): "
2810  << allDone << " "
2811  << Desc.URI << std::endl;
2812 
2813  // we restore the original name, this is required, otherwise
2814  // the file will be cleaned
2815  if(allDone)
2816  {
2817  std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target);
2818  TransactionManager->TransactionStageCopy(this, DestFile, Final);
2819 
2820  // this is for the "real" finish
2821  Complete = true;
2822  Status = StatDone;
2823  Dequeue();
2824  if(Debug)
2825  std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
2826  return;
2827  }
2828  else
2829  DestFile.clear();
2830 
2831  if(Debug)
2832  std::clog << "Finishing: " << Desc.URI << std::endl;
2833  Complete = false;
2834  Status = StatDone;
2835  Dequeue();
2836  return;
2837 }
2838  /*}}}*/
2840 {
2841  // calc sha1 of the just patched file
2843  if(unlikely(PartialFile.empty()))
2844  {
2845  Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL);
2846  return false;
2847  }
2848 
2850  Hashes LocalHashesCalc;
2851  LocalHashesCalc.AddFD(fd);
2852  HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
2853 
2854  if(Debug)
2855  std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
2856 
2857  HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
2858  if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false))
2859  {
2860  Failed("Local/Expected hashes are not usable for " + PartialFile, NULL);
2861  return false;
2862  }
2863 
2864  // final file reached before all patches are applied
2865  if(LocalHashes == TargetFileHashes)
2866  {
2867  Finish(true);
2868  return true;
2869  }
2870 
2871  // remove all patches until the next matching patch is found
2872  // this requires the Index file to be ordered
2873  available_patches.erase(available_patches.begin(),
2874  std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) {
2875  return I.result_hashes == LocalHashes;
2876  }));
2877 
2878  // error checking and falling back if no patch was found
2879  if(available_patches.empty() == true)
2880  {
2881  Failed("No patches left to reach target for " + PartialFile, NULL);
2882  return false;
2883  }
2884 
2885  // queue the right diff
2886  auto const BaseFileURI = Target.URI + ".diff/" + pkgAcquire::URIEncode(available_patches[0].file);
2887  Desc.URI = BaseFileURI + ".gz";
2888  Desc.Description = Target.Description + " " + available_patches[0].file + string(".pdiff");
2890 
2891  if(Debug)
2892  std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
2893 
2894  QueueURI(Desc);
2895 
2896  return true;
2897 }
2898  /*}}}*/
2899 void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/
2900  pkgAcquire::MethodConfig const * const Cnf)
2901 {
2902  if (Debug)
2903  std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
2904 
2905  Item::Done(Message, Hashes, Cnf);
2906 
2907  std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
2908  std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
2909  std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile);
2910  std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
2911 
2912  switch (State)
2913  {
2914  // success in downloading a diff, enter ApplyDiff state
2915  case StateFetchDiff:
2916  Rename(DestFile, PatchFile);
2917  DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
2918  if(Debug)
2919  std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
2921  Local = true;
2922  Desc.URI = "rred:" + pkgAcquire::URIEncode(UnpatchedFile);
2923  QueueURI(Desc);
2924  SetActiveSubprocess("rred");
2925  return;
2926  // success in download/apply a diff, queue next (if needed)
2927  case StateApplyDiff:
2928  // remove the just applied patch and base file
2929  available_patches.erase(available_patches.begin());
2930  RemoveFile("pkgAcqIndexDiffs::Done", PatchFile);
2931  RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile);
2932  if(Debug)
2933  std::clog << "Moving patched file in place: " << std::endl
2934  << DestFile << " -> " << PatchedFile << std::endl;
2935  Rename(DestFile, PatchedFile);
2936 
2937  // see if there is more to download
2938  if(available_patches.empty() == false)
2939  {
2941  Finish();
2942  } else {
2943  DestFile = PatchedFile;
2944  Finish(true);
2945  }
2946  return;
2947  }
2948 }
2949  /*}}}*/
2950 std::string pkgAcqIndexDiffs::Custom600Headers() const /*{{{*/
2951 {
2952  if(State != StateApplyDiff)
2953  return pkgAcqBaseIndex::Custom600Headers();
2954  std::ostringstream patchhashes;
2955  for (auto && hs : available_patches[0].result_hashes)
2956  patchhashes << "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
2957  for (auto && hs : available_patches[0].patch_hashes)
2958  patchhashes << "\nPatch-0-" << hs.HashType() << "-Hash: " << hs.HashValue();
2959  patchhashes << pkgAcqBaseIndex::Custom600Headers();
2960  return patchhashes.str();
2961 }
2962  /*}}}*/
2964 
2965 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/
2968  IndexTarget const &Target,
2969  DiffInfo const &patch,
2970  std::vector<pkgAcqIndexMergeDiffs *> const *const allPatches)
2971  : pkgAcqBaseIndex(Owner, TransactionManager, Target),
2972  patch(patch), allPatches(allPatches), State(StateFetchDiff)
2973 {
2974  Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2975 
2976  Desc.Owner = this;
2977  Desc.ShortDesc = Target.ShortDesc;
2978  Desc.URI = Target.URI + ".diff/" + pkgAcquire::URIEncode(patch.file) + ".gz";
2979  Desc.Description = Target.Description + " " + patch.file + ".pdiff";
2980  DestFile = GetPartialFileNameFromURI(Desc.URI);
2981 
2982  if(Debug)
2983  std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
2984 
2985  QueueURI(Desc);
2986 }
2987  /*}}}*/
2988 void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2989 {
2990  if(Debug)
2991  std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
2992 
2993  pkgAcqBaseIndex::Failed(Message,Cnf);
2994  Status = StatDone;
2995 
2996  // check if we are the first to fail, otherwise we are done here
2997  State = StateDoneDiff;
2998  for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2999  I != allPatches->end(); ++I)
3000  if ((*I)->State == StateErrorDiff)
3001  {
3003  return;
3004  }
3005 
3006  // first failure means we should fallback
3008  if (Debug)
3009  std::clog << "Falling back to normal index file acquire" << std::endl;
3010  RenameOnError(PDiffError);
3011  std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
3012  if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
3013  Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
3014  DestFile.clear();
3015  new pkgAcqIndex(Owner, TransactionManager, Target);
3016 }
3017  /*}}}*/
3018 void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/
3019  pkgAcquire::MethodConfig const * const Cnf)
3020 {
3021  if(Debug)
3022  std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
3023 
3024  Item::Done(Message, Hashes, Cnf);
3025 
3026  if (std::any_of(allPatches->begin(), allPatches->end(),
3027  [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; }))
3028  {
3029  if(Debug)
3030  std::clog << "Another patch failed already, no point in processing this one." << std::endl;
3032  return;
3033  }
3034 
3035  std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
3036  std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
3037  if (UnpatchedFile.empty())
3038  {
3039  _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str());
3041  return;
3042  }
3043  std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
3044 
3045  switch (State)
3046  {
3047  case StateFetchDiff:
3048  // check if this is the last completed diff
3049  State = StateDoneDiff;
3050  for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
3051  I != allPatches->end(); ++I)
3052  if ((*I)->State != StateDoneDiff)
3053  {
3054  if(Debug)
3055  std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
3056  return;
3057  }
3058  for (auto * diff : *allPatches)
3059  Rename(diff->DestFile, GetMergeDiffsPatchFileName(UnpatchedFile, diff->patch.file));
3060  // this is the last completed diff, so we are ready to apply now
3061  DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
3062  if(Debug)
3063  std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
3065  Local = true;
3066  Desc.URI = "rred:" + pkgAcquire::URIEncode(UnpatchedFile);
3067  QueueURI(Desc);
3068  SetActiveSubprocess("rred");
3069  return;
3070  case StateApplyDiff:
3071  // success in download & apply all diffs, finialize and clean up
3072  if(Debug)
3073  std::clog << "Queue patched file in place: " << std::endl
3074  << DestFile << " -> " << PatchedFile << std::endl;
3075 
3076  // queue for copy by the transaction manager
3078 
3079  // ensure the ed's are gone regardless of list-cleanup
3080  for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
3081  I != allPatches->end(); ++I)
3082  RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file));
3083  RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile);
3084 
3085  // all set and done
3086  Complete = true;
3087  if(Debug)
3088  std::clog << "allDone: " << DestFile << "\n" << std::endl;
3089  return;
3090  case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", patch.file.c_str()); break;
3091  case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", patch.file.c_str()); break;
3092  }
3093 }
3094  /*}}}*/
3095 std::string pkgAcqIndexMergeDiffs::Custom600Headers() const /*{{{*/
3096 {
3097  if(State != StateApplyDiff)
3098  return pkgAcqBaseIndex::Custom600Headers();
3099  std::ostringstream patchhashes;
3100  unsigned int seen_patches = 0;
3101  for (auto && hs : (*allPatches)[0]->patch.result_hashes)
3102  patchhashes << "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
3103  for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
3104  I != allPatches->end(); ++I)
3105  {
3106  HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
3107  for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
3108  patchhashes << "\nPatch-" << std::to_string(seen_patches) << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
3109  ++seen_patches;
3110  }
3111  patchhashes << pkgAcqBaseIndex::Custom600Headers();
3112  return patchhashes.str();
3113 }
3114  /*}}}*/
3116 
3117 // AcqIndex::AcqIndex - Constructor /*{{{*/
3120  IndexTarget const &Target, bool const Derived)
3121  : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
3122  CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
3123 {
3124  if (Derived)
3125  return;
3127 
3128  if(_config->FindB("Debug::Acquire::Transaction", false) == true)
3129  std::clog << "New pkgIndex with TransactionManager "
3130  << TransactionManager << std::endl;
3131 }
3132  /*}}}*/
3133 // AcqIndex::Init - deferred Constructor /*{{{*/
3134 void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
3135  string const &ShortDesc)
3136 {
3138 
3139  DestFile = GetPartialFileNameFromURI(URI);
3140  size_t const nextExt = CompressionExtensions.find(' ');
3141  if (nextExt == std::string::npos)
3142  {
3144  CompressionExtensions.clear();
3145  }
3146  else
3147  {
3149  CompressionExtensions = CompressionExtensions.substr(nextExt+1);
3150  }
3151 
3152  if (CurrentCompressionExtension == "uncompressed")
3153  {
3154  Desc.URI = URI;
3155  }
3156  else if (unlikely(CurrentCompressionExtension.empty()))
3157  return;
3158  else
3159  {
3160  Desc.URI = URI + '.' + CurrentCompressionExtension;
3161  DestFile = DestFile + '.' + CurrentCompressionExtension;
3162  }
3163 
3164  // store file size of the download to ensure the fetcher gives
3165  // accurate progress reporting
3166  FileSize = GetExpectedHashes().FileSize();
3167 
3168  Desc.Description = URIDesc;
3169  Desc.Owner = this;
3170  Desc.ShortDesc = ShortDesc;
3171 
3172  QueueURI(Desc);
3173 }
3174  /*}}}*/
3175 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
3176 // ---------------------------------------------------------------------
3177 /* The only header we use is the last-modified header. */
3179 {
3180  std::string msg = pkgAcqBaseIndex::Custom600Headers();
3181  msg.append("\nIndex-File: true");
3182 
3184  {
3185  std::string const Final = GetFinalFilename();
3186 
3187  struct stat Buf;
3188  if (stat(Final.c_str(),&Buf) == 0)
3189  msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
3190  }
3191 
3192  if(Target.IsOptional)
3193  msg += "\nFail-Ignore: true";
3194 
3195  return msg;
3196 }
3197  /*}}}*/
3198 // AcqIndex::Failed - getting the indexfile failed /*{{{*/
3199 bool pkgAcqIndex::CommonFailed(std::string const &TargetURI,
3200  std::string const &Message, pkgAcquire::MethodConfig const *const Cnf)
3201 {
3202  pkgAcqBaseIndex::Failed(Message,Cnf);
3203  // authorisation matches will not be fixed by other compression types
3204  if (Status != StatAuthError)
3205  {
3206  if (CompressionExtensions.empty() == false)
3207  {
3208  Init(TargetURI, Desc.Description, Desc.ShortDesc);
3209  Status = StatIdle;
3210  return true;
3211  }
3212  }
3213  return false;
3214 }
3215 void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
3216 {
3217  if (CommonFailed(Target.URI, Message, Cnf))
3218  return;
3219 
3220  if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
3221  Status = StatDone;
3222  else
3224 }
3225  /*}}}*/
3226 // AcqIndex::Done - Finished a fetch /*{{{*/
3227 // ---------------------------------------------------------------------
3228 /* This goes through a number of states.. On the initial fetch the
3229  method could possibly return an alternate filename which points
3230  to the uncompressed version of the file. If this is so the file
3231  is copied into the partial directory. In all other cases the file
3232  is decompressed with a compressed uri. */
3233 void pkgAcqIndex::Done(string const &Message,
3234  HashStringList const &Hashes,
3235  pkgAcquire::MethodConfig const * const Cfg)
3236 {
3237  Item::Done(Message,Hashes,Cfg);
3238 
3239  switch(Stage)
3240  {
3241  case STAGE_DOWNLOAD:
3242  StageDownloadDone(Message);
3243  break;
3246  break;
3247  }
3248 }
3249  /*}}}*/
3250 // AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/
3251 void pkgAcqIndex::StageDownloadDone(string const &Message)
3252 {
3253  Local = true;
3254  Complete = true;
3255 
3256  std::string const AltFilename = LookupTag(Message,"Alt-Filename");
3257  std::string Filename = LookupTag(Message,"Filename");
3258 
3259  // we need to verify the file against the current Release file again
3260  // on if-modfied-since hit to avoid a stale attack against us
3261  if (StringToBool(LookupTag(Message, "IMS-Hit"), false))
3262  {
3264  EraseFileName = DestFile = flCombine(flNotFile(DestFile), flNotDir(Filename));
3265  if (symlink(Filename.c_str(), DestFile.c_str()) != 0)
3266  _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
3268  Desc.URI = "store:" + pkgAcquire::URIEncode(DestFile);
3269  QueueURI(Desc);
3270  SetActiveSubprocess(::URI(Desc.URI).Access);
3271  return;
3272  }
3273  // methods like file:// give us an alternative (uncompressed) file
3274  else if (Target.KeepCompressed == false && AltFilename.empty() == false)
3275  {
3276  Filename = AltFilename;
3277  EraseFileName.clear();
3278  }
3279  // Methods like e.g. "file:" will give us a (compressed) FileName that is
3280  // not the "DestFile" we set, in this case we uncompress from the local file
3281  else if (Filename != DestFile && RealFileExists(DestFile) == false)
3282  {
3283  // symlinking ensures that the filename can be used for compression detection
3284  // that is e.g. needed for by-hash which has no extension over file
3285  if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
3286  _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
3287  else
3288  {
3289  EraseFileName = DestFile;
3290  Filename = DestFile;
3291  }
3292  }
3293 
3296  if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
3297  Desc.URI = "copy:" + pkgAcquire::URIEncode(Filename);
3298  else
3299  Desc.URI = "store:" + pkgAcquire::URIEncode(Filename);
3300  if (DestFile == Filename)
3301  {
3302  if (CurrentCompressionExtension == "uncompressed")
3303  return StageDecompressDone();
3304  DestFile = "/dev/null";
3305  }
3306 
3307  if (EraseFileName.empty() && Filename != AltFilename)
3308  EraseFileName = Filename;
3309 
3310  // queue uri for the next stage
3311  QueueURI(Desc);
3312  SetActiveSubprocess(::URI(Desc.URI).Access);
3313 }
3314  /*}}}*/
3315 // AcqIndex::StageDecompressDone - Final verification /*{{{*/
3317 {
3318  if (DestFile == "/dev/null")
3320 
3321  // Done, queue for rename on transaction finished
3323 }
3324  /*}}}*/
3326 
3327 // AcqArchive::AcqArchive - Constructor /*{{{*/
3328 // ---------------------------------------------------------------------
3329 /* This just sets up the initial fetch environment and queues the first
3330  possibilitiy */
3332  pkgRecords *const Recs, pkgCache::VerIterator const &Version,
3333  string &StoreFilename) : Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
3334  StoreFilename(StoreFilename),
3335  Trusted(false)
3336 {
3337  if (Version.Arch() == 0)
3338  {
3339  _error->Error(_("I wasn't able to locate a file for the %s package. "
3340  "This might mean you need to manually fix this package. "
3341  "(due to missing arch)"),
3342  Version.ParentPkg().FullName().c_str());
3343  return;
3344  }
3345 
3346  // check if we have one trusted source for the package. if so, switch
3347  // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
3348  bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
3349  bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
3350  bool seenUntrusted = false;
3351  for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
3352  {
3353  pkgIndexFile *Index;
3354  if (Sources->FindIndex(i.File(),Index) == false)
3355  continue;
3356 
3357  if (debugAuth == true)
3358  std::cerr << "Checking index: " << Index->Describe()
3359  << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
3360 
3361  if (Index->IsTrusted() == true)
3362  {
3363  Trusted = true;
3364  if (allowUnauth == false)
3365  break;
3366  }
3367  else
3368  seenUntrusted = true;
3369  }
3370 
3371  // "allow-unauthenticated" restores apts old fetching behaviour
3372  // that means that e.g. unauthenticated file:// uris are higher
3373  // priority than authenticated http:// uris
3374  if (allowUnauth == true && seenUntrusted == true)
3375  Trusted = false;
3376 
3377  StoreFilename.clear();
3378  for (auto Vf = Version.FileList(); Vf.end() == false; ++Vf)
3379  {
3380  auto const PkgF = Vf.File();
3381  if (unlikely(PkgF.end()))
3382  continue;
3383  if (PkgF.Flagged(pkgCache::Flag::NotSource))
3384  continue;
3385  pkgIndexFile *Index;
3386  if (Sources->FindIndex(PkgF, Index) == false)
3387  continue;
3388  if (Trusted && Index->IsTrusted() == false)
3389  continue;
3390 
3391  pkgRecords::Parser &Parse = Recs->Lookup(Vf);
3392  // collect the hashes from the indexes
3393  auto hsl = Parse.Hashes();
3394  if (ExpectedHashes.empty())
3395  ExpectedHashes = hsl;
3396  else
3397  {
3398  // bad things will likely happen, but the user might be "lucky" still
3399  // if the sources provide the same hashtypes (so that they aren't mixed)
3400  for (auto const &hs : hsl)
3401  if (ExpectedHashes.push_back(hs) == false)
3402  {
3403  _error->Warning("Sources disagree on hashes for supposely identical version '%s' of '%s'.",
3404  Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
3405  break;
3406  }
3407  }
3408  // only allow local volatile sources to have no hashes
3409  if (PkgF.Flagged(pkgCache::Flag::LocalSource))
3410  LocalSource = true;
3411  else if (hsl.empty())
3412  continue;
3413 
3414  std::string poolfilename = Parse.FileName();
3415  if (poolfilename.empty())
3416  continue;
3417 
3418  std::remove_reference<decltype(ModifyCustomFields())>::type fields;
3419  {
3420  auto const debIndex = dynamic_cast<pkgDebianIndexTargetFile const *const>(Index);
3421  if (debIndex != nullptr)
3422  {
3423  auto const IT = debIndex->GetIndexTarget();
3424  fields.emplace("Target-Repo-URI", IT.Option(IndexTarget::REPO_URI));
3425  fields.emplace("Target-Release", IT.Option(IndexTarget::RELEASE));
3426  fields.emplace("Target-Site", IT.Option(IndexTarget::SITE));
3427  }
3428  }
3429  fields.emplace("Target-Base-URI", Index->ArchiveURI(""));
3430  if (PkgF->Component != 0)
3431  fields.emplace("Target-Component", PkgF.Component());
3432  auto const RelF = PkgF.ReleaseFile();
3433  if (RelF.end() == false)
3434  {
3435  if (RelF->Codename != 0)
3436  fields.emplace("Target-Codename", RelF.Codename());
3437  if (RelF->Archive != 0)
3438  fields.emplace("Target-Suite", RelF.Archive());
3439  }
3440  fields.emplace("Target-Architecture", Version.Arch());
3441  fields.emplace("Target-Type", flExtension(poolfilename));
3442 
3443  if (StoreFilename.empty())
3444  {
3445  /* We pick a filename based on the information we have for the version,
3446  but we don't know what extension such a file should have, so we look
3447  at the filenames used online and assume that they are the same for
3448  all repositories containing this file */
3449  StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' +
3450  QuoteString(Version.VerStr(), "_:") + '_' +
3451  QuoteString(Version.Arch(), "_:.") +
3452  "." + flExtension(poolfilename);
3453 
3454  Desc.URI = Index->ArchiveURI(poolfilename);
3455  Desc.Description = Index->ArchiveInfo(Version);
3456  Desc.Owner = this;
3457  Desc.ShortDesc = Version.ParentPkg().FullName(true);
3458  auto &customfields = ModifyCustomFields();
3459  for (auto const &f : fields)
3460  customfields[f.first] = f.second;
3461  FileSize = Version->Size;
3462  }
3463  else
3464  PushAlternativeURI(Index->ArchiveURI(poolfilename), std::move(fields), true);
3465  }
3466  if (StoreFilename.empty())
3467  {
3468  _error->Error(_("Can't find a source to download version '%s' of '%s'"),
3469  Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
3470  return;
3471  }
3472  if (FileSize == 0 && not _config->FindB("Acquire::AllowUnsizedPackages", false))
3473  {
3474  _error->Warning("Repository is broken: %s (= %s) has no Size information",
3475  Version.ParentPkg().FullName(false).c_str(),
3476  Version.VerStr());
3477  }
3478 
3479  // Check if we already downloaded the file
3480  struct stat Buf;
3481  auto FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
3482  if (stat(FinalFile.c_str(), &Buf) == 0)
3483  {
3484  // Make sure the size matches
3485  if ((unsigned long long)Buf.st_size == Version->Size)
3486  {
3487  Complete = true;
3488  Local = true;
3489  Status = StatDone;
3490  StoreFilename = DestFile = FinalFile;
3491  return;
3492  }
3493 
3494  /* Hmm, we have a file and its size does not match, this shouldn't
3495  happen.. */
3496  RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
3497  }
3498 
3499  // Check the destination file
3500  DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
3501  if (stat(DestFile.c_str(), &Buf) == 0)
3502  {
3503  // Hmm, the partial file is too big, erase it
3504  if ((unsigned long long)Buf.st_size > Version->Size)
3505  RemoveFile("pkgAcqArchive::QueueNext", DestFile);
3506  else
3507  PartialSize = Buf.st_size;
3508  }
3509 
3510  // Disables download of archives - useful if no real installation follows,
3511  // e.g. if we are just interested in proposed installation order
3512  if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
3513  {
3514  Complete = true;
3515  Local = true;
3516  Status = StatDone;
3517  StoreFilename = DestFile = FinalFile;
3518  return;
3519  }
3520 
3521  // Create the item
3522  Local = false;
3523  QueueURI(Desc);
3524 }
3525  /*}}}*/
3527 {
3528  return false;
3529 }
3530  /*}}}*/
3531 // AcqArchive::Done - Finished fetching /*{{{*/
3532 // ---------------------------------------------------------------------
3533 /* */
3534 void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
3535  pkgAcquire::MethodConfig const * const Cfg)
3536 {
3537  Item::Done(Message, Hashes, Cfg);
3538 
3539  // Grab the output filename
3540  std::string const FileName = LookupTag(Message,"Filename");
3541  if (DestFile != FileName && RealFileExists(DestFile) == false)
3542  {
3543  StoreFilename = DestFile = FileName;
3544  Local = true;
3545  Complete = true;
3546  return;
3547  }
3548 
3549  // Done, move it into position
3550  string const FinalFile = GetFinalFilename();
3551  Rename(DestFile,FinalFile);
3552  StoreFilename = DestFile = FinalFile;
3553  Complete = true;
3554 }
3555  /*}}}*/
3556 // AcqArchive::Failed - Failure handler /*{{{*/
3557 // ---------------------------------------------------------------------
3558 /* Here we try other sources */
3559 void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
3560 {
3561  Item::Failed(Message,Cnf);
3562 }
3563  /*}}}*/
3565 {
3566  return Trusted;
3567 }
3568  /*}}}*/
3570 {
3571  if (Status == pkgAcquire::Item::StatDone &&
3572  Complete == true)
3573  return;
3574  StoreFilename = string();
3575 }
3576  /*}}}*/
3577 std::string pkgAcqArchive::DescURI() const /*{{{*/
3578 {
3579  return Desc.URI;
3580 }
3581  /*}}}*/
3582 std::string pkgAcqArchive::ShortDesc() const /*{{{*/
3583 {
3584  return Desc.ShortDesc;
3585 }
3586  /*}}}*/
3588 
3589 // AcqChangelog::pkgAcqChangelog - Constructors /*{{{*/
3591 {
3592  public:
3593  std::string FinalFile;
3594 };
3596  std::string const &DestDir, std::string const &DestFilename) :
3597  pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr())
3598 {
3599  Desc.URI = URI(Ver);
3600  Init(DestDir, DestFilename);
3601 }
3602 // some parameters are char* here as they come likely from char* interfaces – which can also return NULL
3604  char const * const Component, char const * const SrcName, char const * const SrcVersion,
3605  const string &DestDir, const string &DestFilename) :
3606  pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3607 {
3608  Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion);
3609  Init(DestDir, DestFilename);
3610 }
3612  std::string const &URI, char const * const SrcName, char const * const SrcVersion,
3613  const string &DestDir, const string &DestFilename) :
3614  pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3615 {
3616  Desc.URI = URI;
3617  Init(DestDir, DestFilename);
3618 }
3619 void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename)
3620 {
3621  if (Desc.URI.empty())
3622  {
3623  Status = StatError;
3624  // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3625  strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3626  // Let the error message print something sensible rather than "Failed to fetch /"
3627  if (DestFilename.empty())
3628  DestFile = SrcName + ".changelog";
3629  else
3630  DestFile = DestFilename;
3631  Desc.URI = "changelog:/" + pkgAcquire::URIEncode(DestFile);
3632  return;
3633  }
3634 
3635  std::string DestFileName;
3636  if (DestFilename.empty())
3637  DestFileName = flCombine(DestFile, SrcName + ".changelog");
3638  else
3639  DestFileName = flCombine(DestFile, DestFilename);
3640 
3641  std::string const SandboxUser = _config->Find("APT::Sandbox::User");
3642  std::string const systemTemp = GetTempDir(SandboxUser);
3643  char tmpname[1000];
3644  snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str());
3645  if (NULL == mkdtemp(tmpname))
3646  {
3647  _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str());
3648  Status = StatError;
3649  return;
3650  }
3651  TemporaryDirectory = tmpname;
3652 
3653  ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", TemporaryDirectory.c_str(),
3654  SandboxUser.c_str(), ROOT_GROUP, 0700);
3655 
3656  DestFile = flCombine(TemporaryDirectory, DestFileName);
3657  if (DestDir.empty() == false)
3658  {
3659  d->FinalFile = flCombine(DestDir, DestFileName);
3660  if (RealFileExists(d->FinalFile))
3661  {
3662  FileFd file1, file2;
3663  if (file1.Open(DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive) &&
3664  file2.Open(d->FinalFile, FileFd::ReadOnly) && CopyFile(file2, file1))
3665  {
3666  ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", DestFile.c_str(), "root", ROOT_GROUP, 0644);
3667  struct timeval times[2];
3668  times[0].tv_sec = times[1].tv_sec = file2.ModificationTime();
3669  times[0].tv_usec = times[1].tv_usec = 0;
3670  utimes(DestFile.c_str(), times);
3671  }
3672  }
3673  }
3674 
3675  Desc.ShortDesc = "Changelog";
3676  strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
3677  Desc.Owner = this;
3678  QueueURI(Desc);
3679 }
3680  /*}}}*/
3681 std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver) /*{{{*/
3682 {
3683  std::string const confOnline = "Acquire::Changelogs::AlwaysOnline";
3684  bool AlwaysOnline = _config->FindB(confOnline, false);
3685  if (AlwaysOnline == false)
3686  for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3687  {
3688  pkgCache::PkgFileIterator const PF = VF.File();
3689  if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3690  continue;
3691  pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3692  if (RF->Origin != 0 && _config->FindB(confOnline + "::Origin::" + RF.Origin(), false))
3693  {
3694  AlwaysOnline = true;
3695  break;
3696  }
3697  }
3698  if (AlwaysOnline == false)
3699  {
3700  pkgCache::PkgIterator const Pkg = Ver.ParentPkg();
3701  if (Pkg->CurrentVer != 0 && Pkg.CurrentVer() == Ver)
3702  {
3703  std::string const root = _config->FindDir("Dir");
3704  std::string const basename = root + std::string("usr/share/doc/") + Pkg.Name() + "/changelog";
3705  std::string const debianname = basename + ".Debian";
3706  if (FileExists(debianname))
3707  return "copy://" + debianname;
3708  else if (FileExists(debianname + ".gz"))
3709  return "store://" + debianname + ".gz";
3710  else if (FileExists(basename))
3711  return "copy://" + basename;
3712  else if (FileExists(basename + ".gz"))
3713  return "store://" + basename + ".gz";
3714  }
3715  }
3716 
3717  char const * const SrcName = Ver.SourcePkgName();
3718  char const * const SrcVersion = Ver.SourceVerStr();
3719  // find the first source for this version which promises a changelog
3720  for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3721  {
3722  pkgCache::PkgFileIterator const PF = VF.File();
3723  if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3724  continue;
3725  pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3726  std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion);
3727  if (uri.empty())
3728  continue;
3729  return uri;
3730  }
3731  return "";
3732 }
3734 {
3735  if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0))
3736  return "";
3737  std::string const serverConfig = "Acquire::Changelogs::URI";
3738  std::string server;
3739 #define APT_EMPTY_SERVER \
3740  if (server.empty() == false) \
3741  { \
3742  if (server != "no") \
3743  return server; \
3744  return ""; \
3745  }
3746 #define APT_CHECK_SERVER(X, Y) \
3747  if (Rls->X != 0) \
3748  { \
3749  std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
3750  server = _config->Find(specialServerConfig); \
3751  APT_EMPTY_SERVER \
3752  }
3753  // this way e.g. Debian-Security can fallback to Debian
3754  APT_CHECK_SERVER(Label, "Override::")
3755  APT_CHECK_SERVER(Origin, "Override::")
3756 
3757  if (RealFileExists(Rls.FileName()))
3758  {
3759  _error->PushToStack();
3760  FileFd rf;
3761  /* This can be costly. A caller wanting to get millions of URIs might
3762  want to do this on its own once and use Override settings.
3763  We don't do this here as Origin/Label are not as unique as they
3764  should be so this could produce request order-dependent anomalies */
3765  if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
3766  {
3767  pkgTagFile TagFile(&rf, rf.Size());
3768  pkgTagSection Section;
3769  if (TagFile.Step(Section) == true)
3770  server = Section.FindS("Changelogs");
3771  }
3772  _error->RevertToStack();
3774  }
3775 
3776  APT_CHECK_SERVER(Label, "")
3777  APT_CHECK_SERVER(Origin, "")
3778 #undef APT_CHECK_SERVER
3779 #undef APT_EMPTY_SERVER
3780  return "";
3781 }
3783  char const * const Component, char const * const SrcName,
3784  char const * const SrcVersion)
3785 {
3786  return URI(URITemplate(Rls), Component, SrcName, SrcVersion);
3787 }
3788 std::string pkgAcqChangelog::URI(std::string const &Template,
3789  char const * const Component, char const * const SrcName,
3790  char const * const SrcVersion)
3791 {
3792  if (Template.find("@CHANGEPATH@") == std::string::npos)
3793  return "";
3794 
3795  // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/apt_1.1 or contrib/liba/libapt/libapt_2.0
3796  std::string const Src{SrcName};
3797  std::string path = pkgAcquire::URIEncode(APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1));
3798  path.append("/").append(pkgAcquire::URIEncode(Src)).append("/");
3799  path.append(pkgAcquire::URIEncode(Src)).append("_").append(pkgAcquire::URIEncode(StripEpoch(SrcVersion)));
3800  // we omit component for releases without one (= flat-style repositories)
3801  if (Component != NULL && strlen(Component) != 0)
3802  path = pkgAcquire::URIEncode(Component) + "/" + path;
3803 
3804  return SubstVar(Template, "@CHANGEPATH@", path);
3805 }
3806  /*}}}*/
3807 // AcqChangelog::Failed - Failure handler /*{{{*/
3808 void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
3809 {
3810  Item::Failed(Message,Cnf);
3811 
3812  std::string errText;
3813  // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3814  strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3815 
3816  // Error is probably something techy like 404 Not Found
3817  if (ErrorText.empty())
3818  ErrorText = errText;
3819  else
3820  ErrorText = errText + " (" + ErrorText + ")";
3821 }
3822  /*}}}*/
3823 // AcqChangelog::Done - Item downloaded OK /*{{{*/
3824 void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes,
3825  pkgAcquire::MethodConfig const * const Cnf)
3826 {
3827  Item::Done(Message,CalcHashes,Cnf);
3828  if (d->FinalFile.empty() == false)
3829  {
3830  if (RemoveFile("pkgAcqChangelog::Done", d->FinalFile) == false ||
3831  Rename(DestFile, d->FinalFile) == false)
3832  Status = StatError;
3833  }
3834 
3835  Complete = true;
3836 }
3837  /*}}}*/
3839 {
3840  if (TemporaryDirectory.empty() == false)
3841  {
3842  RemoveFile("~pkgAcqChangelog", DestFile);
3843  rmdir(TemporaryDirectory.c_str());
3844  }
3845  delete d;
3846 }
3847  /*}}}*/
3848 
3849 // AcqFile::pkgAcqFile - Constructor /*{{{*/
3850 pkgAcqFile::pkgAcqFile(pkgAcquire *const Owner, string const &URI, HashStringList const &Hashes,
3851  unsigned long long const Size, string const &Dsc, string const &ShortDesc,
3852  const string &DestDir, const string &DestFilename,
3853  bool const IsIndexFile) : Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
3854 {
3855  ::URI url{URI};
3856  if (url.Path.find(' ') != std::string::npos || url.Path.find('%') == std::string::npos)
3857  url.Path = pkgAcquire::URIEncode(url.Path);
3858 
3859  if(!DestFilename.empty())
3860  DestFile = DestFilename;
3861  else if(!DestDir.empty())
3862  DestFile = DestDir + "/" + DeQuoteString(flNotDir(url.Path));
3863  else
3864  DestFile = DeQuoteString(flNotDir(url.Path));
3865 
3866  // Create the item
3867  Desc.URI = std::string(url);
3868  Desc.Description = Dsc;
3869  Desc.Owner = this;
3870 
3871  // Set the short description to the archive component
3872  Desc.ShortDesc = ShortDesc;
3873 
3874  // Get the transfer sizes
3875  FileSize = Size;
3876  struct stat Buf;
3877  if (stat(DestFile.c_str(),&Buf) == 0)
3878  {
3879  // Hmm, the partial file is too big, erase it
3880  if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
3881  RemoveFile("pkgAcqFile", DestFile);
3882  else
3883  PartialSize = Buf.st_size;
3884  }
3885 
3886  QueueURI(Desc);
3887 }
3888  /*}}}*/
3889 // AcqFile::Done - Item downloaded OK /*{{{*/
3890 void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
3891  pkgAcquire::MethodConfig const * const Cnf)
3892 {
3893  Item::Done(Message,CalcHashes,Cnf);
3894 
3895  std::string const FileName = LookupTag(Message,"Filename");
3896  Complete = true;
3897 
3898  // The files timestamp matches
3899  if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
3900  return;
3901 
3902  // We have to copy it into place
3903  if (RealFileExists(DestFile.c_str()) == false)
3904  {
3905  Local = true;
3906  if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
3907  Cnf->Removable == true)
3908  {
3909  Desc.URI = "copy:" + pkgAcquire::URIEncode(FileName);
3910  QueueURI(Desc);
3911  return;
3912  }
3913 
3914  // Erase the file if it is a symlink so we can overwrite it
3915  struct stat St;
3916  if (lstat(DestFile.c_str(),&St) == 0)
3917  {
3918  if (S_ISLNK(St.st_mode) != 0)
3919  RemoveFile("pkgAcqFile::Done", DestFile);
3920  }
3921 
3922  // Symlink the file
3923  if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
3924  {
3925  _error->PushToStack();
3926  _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str());
3927  std::stringstream msg;
3928  _error->DumpErrors(msg, GlobalError::DEBUG, false);
3929  _error->RevertToStack();
3930  ErrorText = msg.str();
3931  Status = StatError;
3932  Complete = false;
3933  }
3934  }
3935 }
3936  /*}}}*/
3937 string pkgAcqFile::Custom600Headers() const /*{{{*/
3938 {
3939  string Header = pkgAcquire::Item::Custom600Headers();
3940  if (not IsIndexFile)
3941  return Header;
3942  return Header + "\nIndex-File: true";
3943 }
3944  /*}}}*/
3946 
3947 void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/
3948 {
3949  pkgAcqFile::Failed(Message, Cnf);
3950  if (Status == StatIdle)
3951  return;
3952  if (RealFileExists(DestFile))
3953  Rename(DestFile, DestFile + ".FAILED");
3954  Worker->ReplyAux(Desc);
3955 }
3956  /*}}}*/
3957 void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/
3958  pkgAcquire::MethodConfig const *const Cnf)
3959 {
3960  pkgAcqFile::Done(Message, CalcHashes, Cnf);
3961  if (Status == StatDone)
3962  Worker->ReplyAux(Desc);
3963  else if (Status == StatAuthError || Status == StatError)
3964  Worker->ReplyAux(Desc);
3965 }
3966  /*}}}*/
3967 std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/
3968 {
3969  if (MaximumSize == 0)
3971  std::string maxsize;
3972  strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize);
3973  return pkgAcqFile::Custom600Headers().append(maxsize);
3974 }
3975  /*}}}*/
3977 {
3978  auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
3979  if (APT::String::Startswith(DestFile, dirname))
3980  {
3981  // the file is never returned by method requesting it, so fix up the permission now
3982  if (FileExists(DestFile))
3983  {
3984  ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644);
3985  if (Status == StatDone)
3986  return;
3987  }
3988  }
3989  else
3990  {
3991  dirname = flNotFile(DestFile);
3992  RemoveFile("pkgAcqAuxFile::Finished", DestFile);
3993  RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED");
3994  rmdir(dirname.c_str());
3995  }
3996  DestFile.clear();
3997 }
3998  /*}}}*/
3999 // GetAuxFileNameFromURI /*{{{*/
4000 static std::string GetAuxFileNameFromURIInLists(std::string const &uri)
4001 {
4002  // check if we have write permission for our usual location.
4003  auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
4004  char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
4005  std::string const tmpfile_tpl = flCombine(dirname, filetag);
4006  std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
4007  int const fd = mkstemp(tmpfile.get());
4008  if (fd == -1)
4009  return "";
4010  RemoveFile("GetAuxFileNameFromURI", tmpfile.get());
4011  close(fd);
4012  return flCombine(dirname, URItoFileName(uri));
4013 }
4014 static std::string GetAuxFileNameFromURI(std::string const &uri)
4015 {
4016  auto const lists = GetAuxFileNameFromURIInLists(uri);
4017  if (lists.empty() == false)
4018  return lists;
4019 
4020  std::string tmpdir_tpl;
4021  strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str());
4022  std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free };
4023  if (mkdtemp(tmpdir.get()) == nullptr)
4024  {
4025  _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get());
4026  return flCombine("/nonexistent/auxfiles/", URItoFileName(uri));
4027  }
4028  chmod(tmpdir.get(), 0755);
4029  auto const filename = flCombine(tmpdir.get(), URItoFileName(uri));
4030  _error->PushToStack();
4031  FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly);
4032  if (in.IsOpen())
4033  {
4035  CopyFile(in, out);
4036  ChangeOwnerAndPermissionOfFile("GetAuxFileNameFromURI", filename.c_str(), "root", ROOT_GROUP, 0644);
4037  }
4038  _error->RevertToStack();
4039  return filename;
4040 }
4041  /*}}}*/
4043  std::string const &ShortDesc, std::string const &Desc, std::string const &URI,
4044  HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false),
4045  Owner(Owner), Worker(Worker), MaximumSize(MaximumSize)
4046 {
4047  /* very bad failures can happen while constructing which causes
4048  us to hang as the aux request is never answered (e.g. method not available)
4049  Ideally we catch failures earlier, but a safe guard can't hurt. */
4050  if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching)
4051  return;
4052  Failed(std::string("400 URI Failure\n") +
4053  "URI: " + URI + "\n" +
4054  "Filename: " + DestFile,
4055  nullptr);
4056 }
static std::string GetDiffIndexURI(IndexTarget const &Target)
static bool std::string const metaIndex const *const MetaIndexParser
strprintf(m, msg, repo.c_str())
static char const *const std::string const & repo
static bool RemoveFileForBootstrapLinking(std::string &ErrorText, std::string const &For, std::string const &Boot)
static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)
Definition: acquire-item.cc:75
static bool std::string const metaIndex const *const pkgAcqMetaClearSig *const pkgAcquire::Item *const I
static HashStringList GetExpectedHashesFromFor(metaIndex *const Parser, std::string const &MetaKey)
InsecureType
static std::string GetDiffIndexFileName(std::string const &Name)
return false
I Status
static std::string GetAuxFileNameFromURIInLists(std::string const &uri)
static std::string GetFinalFileNameFromURI(std::string const &uri)
Definition: acquire-item.cc:70
static std::string GetPartialFileNameFromURI(std::string const &uri)
Definition: acquire-item.cc:65
static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError
static std::string GetAuxFileNameFromURI(std::string const &uri)
MessageInsecureRepository(true, msgstr, repo)
static std::string GetDiffsPatchFileName(std::string const &Final)
Definition: acquire-item.cc:96
static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string const &FailCode, std::string const &Details)
char const * msgstr
static bool std::string const metaIndex const *const pkgAcqMetaClearSig *const TransactionManager
#define APT_EMPTY_SERVER
static char const *const msg
static void formatHashsum(std::ostream &out, HashString const &hs)
#define APT_CHECK_SERVER(X, Y)
static std::string GetExistingFilename(std::string const &File)
static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type)
static void LoadLastMetaIndexParser(pkgAcqMetaClearSig *const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)
static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)
Definition: acquire-item.cc:90
static std::string GetPartialFileName(std::string const &file)
Definition: acquire-item.cc:58
bool OpenMaybeClearSignedFile(std::string const &ClearSignedFileName, FileFd &MessageFile)
open a file which might be clear-signed
Definition: gpgv.cc:538
CleanupItem(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target)
bool TransactionState(TransactionStates const state) APT_OVERRIDE
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
virtual std::string DescURI() const APT_OVERRIDE
Definition: fileutl.h:39
bool IsOpen()
Definition: fileutl.h:150
@ Extension
Definition: fileutl.h:80
@ Auto
Definition: fileutl.h:78
@ WriteOnly
Definition: fileutl.h:60
@ Exclusive
Definition: fileutl.h:64
@ Create
Definition: fileutl.h:63
@ ReadOnly
Definition: fileutl.h:59
bool Failed()
Definition: fileutl.h:151
bool Open(std::string FileName, unsigned int const Mode, CompressMode Compress, unsigned long const AccessMode=0666)
Definition: fileutl.cc:2415
time_t ModificationTime()
Definition: fileutl.cc:2955
unsigned long long Size()
Definition: fileutl.cc:2967
@ DEBUG
for developers only in areas it is hard to print something directly
Definition: error.h:66
bool usable() const
Definition: hashes.cc:178
const_iterator begin() const
Definition: hashes.h:133
std::vector< HashString >::const_iterator const_iterator
Definition: hashes.h:130
bool empty() const
Definition: hashes.h:119
HashString const * find(char const *const type) const
Definition: hashes.cc:191
bool push_back(const HashString &hashString)
Definition: hashes.cc:232
const_iterator end() const
Definition: hashes.h:136
unsigned long long FileSize() const
Definition: hashes.cc:210
bool usable() const
Definition: hashes.cc:153
std::string toStr() const
Definition: hashes.cc:163
static APT_PURE const char ** SupportedHashes()
Definition: hashes.cc:135
std::string HashValue() const
Definition: hashes.h:44
std::string HashType() const
Definition: hashes.h:43
Definition: hashes.h:170
HashStringList GetHashStringList()
Definition: hashes.cc:431
bool AddFD(int const Fd, unsigned long long Size=0)
Definition: hashes.cc:362
Information about an index file.
Definition: indexfile.h:39
bool IsOptional
Is it okay if the file isn't found in the meta index.
Definition: indexfile.h:55
bool KeepCompressed
If the file is downloaded compressed, do not unpack it.
Definition: indexfile.h:58
std::string Description
A description of the index file.
Definition: indexfile.h:45
std::string Option(OptionKeys const Key) const
Definition: indexfile.cc:97
std::string ShortDesc
A shorter description of the index file.
Definition: indexfile.h:48
std::map< std::string, std::string > Options
options with which this target was created Prefer the usage of Option if at all possible....
Definition: indexfile.h:63
std::string URI
A URI from which the index file can be downloaded.
Definition: indexfile.h:42
@ ARCHITECTURE
Definition: indexfile.h:75
@ ALLOW_DOWNGRADE_TO_INSECURE
Definition: indexfile.h:92
@ COMPRESSIONTYPES
Definition: indexfile.h:83
@ KEEPCOMPRESSEDAS
Definition: indexfile.h:87
@ INRELEASE_PATH
Definition: indexfile.h:93
@ ALLOW_INSECURE
Definition: indexfile.h:90
std::string MetaKey
The key by which this index file should be looked up within the meta index file.
Definition: indexfile.h:52
bool OptionBool(OptionKeys const Key) const
Definition: indexfile.cc:151
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
NoActionItem(pkgAcquire *const Owner, IndexTarget const &Target)
IndexTarget const Target
NoActionItem(pkgAcquire *const Owner, IndexTarget const &Target, std::string const &FinalFile)
virtual std::string DescURI() const APT_OVERRIDE
Definition: rred.cc:378
Definition: strutl.h:193
std::string Access
Definition: strutl.h:198
std::string Path
Definition: strutl.h:202
URI(std::string Path)
Definition: strutl.h:212
static std::string SiteOnly(const std::string &URI)
Definition: strutl.cc:1788
time_t GetValidUntil() const
Definition: metaindex.cc:68
std::string GetVersion() const
Definition: metaindex.cc:62
std::string GetReleaseNotes() const
Definition: metaindex.cc:65
bool GetSupportsAcquireByHash() const
Definition: metaindex.cc:67
std::string GetExpectedDist() const
Definition: metaindex.cc:71
virtual std::vector< IndexTarget > GetIndexTargets() const =0
virtual bool IsArchitectureSupported(std::string const &arch) const
Definition: metaindex.cc:134
bool CheckDist(std::string const &MaybeDist) const
Definition: metaindex.cc:73
std::string GetSuite() const
Definition: metaindex.cc:64
virtual metaIndex * UnloadedClone() const =0
TriState GetTrusted() const
Definition: metaindex.cc:58
time_t GetDate() const
Definition: metaindex.cc:69
signed short GetDefaultPin() const
Definition: metaindex.cc:66
void swapLoad(metaIndex *const OldMetaIndex)
Definition: metaindex.cc:116
std::string GetSignedBy() const
Definition: metaindex.cc:59
std::string GetLabel() const
Definition: metaindex.cc:61
checkSum * Lookup(std::string const &MetaKey) const
Definition: metaindex.cc:92
virtual bool IsArchitectureAllSupportedFor(IndexTarget const &target) const
Definition: metaindex.cc:139
TriState GetLoadedSuccessfully() const
Definition: metaindex.cc:70
std::string GetCodename() const
Definition: metaindex.cc:63
std::string URI
Definition: metaindex.h:39
virtual time_t GetNotBefore() const =0
virtual bool HasSupportForComponent(std::string const &component) const
Definition: metaindex.cc:144
bool Exists(std::string const &MetaKey) const
tests if a checksum for this file is available
Definition: metaindex.cc:100
virtual bool Load(std::string const &Filename, std::string *const ErrorText)=0
std::string GetOrigin() const
Definition: metaindex.cc:60
pkgRecords * Recs
A package records object, used to look up the file corresponding to each version of the package.
Definition: acquire-item.h:962
pkgCache::VerIterator Version
The package version being fetched.
Definition: acquire-item.h:952
bool Trusted
true if this version file is being downloaded from a trusted source.
Definition: acquire-item.h:972
HashStringList ExpectedHashes
Definition: acquire-item.h:948
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual void Finished() APT_OVERRIDE
std::string & StoreFilename
A location in which the actual filename of the package should be stored.
Definition: acquire-item.h:967
virtual bool IsTrusted() const APT_OVERRIDE
virtual std::string GetFinalFilename() const APT_OVERRIDE
Get the full pathname of the final file for the current URI.
virtual std::string DescURI() const APT_OVERRIDE
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
bool QueueNext()
Queue up the next available file for this version.
virtual ~pkgAcqArchive()
virtual bool HashesRequired() const APT_OVERRIDE
pkgSourceList * Sources
The list of sources from which to pick archives to download this package from.
Definition: acquire-item.h:957
virtual std::string ShortDesc() const APT_OVERRIDE
pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sources, pkgRecords *const Recs, pkgCache::VerIterator const &Version, std::string &StoreFilename)
Create a new pkgAcqArchive.
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, std::string const &ShortDesc, std::string const &Desc, std::string const &URI, HashStringList const &Hashes, unsigned long long const MaximumSize)
unsigned long long MaximumSize
virtual ~pkgAcqAuxFile()
virtual void Finished() APT_OVERRIDE
virtual std::string Custom600Headers() const APT_OVERRIDE
virtual void Done(std::string const &Message, HashStringList const &CalcHashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
pkgAcquire::Worker *const Worker
Common base class for all classes that deal with fetching indexes {{{.
Definition: acquire-item.h:597
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual std::string GetFinalFilename() const APT_OVERRIDE
Get the full pathname of the final file for the current URI.
virtual ~pkgAcqBaseIndex()
pkgAcqBaseIndex(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target) APT_NONNULL(2
Retrieve the changelog for the given version {{{.
static std::string URI(pkgCache::VerIterator const &Ver)
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
APT_HIDDEN void Init(std::string const &DestDir, std::string const &DestFilename)
std::string TemporaryDirectory
std::string const SrcName
pkgAcqChangelog(pkgAcquire *const Owner, pkgCache::VerIterator const &Ver, std::string const &DestDir="", std::string const &DestFilename="")
Create a new pkgAcqChangelog object.
std::string const SrcVersion
virtual ~pkgAcqChangelog()
virtual void Done(std::string const &Message, HashStringList const &CalcHashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
static std::string URITemplate(pkgCache::RlsFileIterator const &Rls)
Private *const d
An item that is responsible for fetching an index file of {{{ package list diffs and starting the pac...
Definition: acquire-item.h:707
std::vector< pkgAcqIndexMergeDiffs * > * diffs
Definition: acquire-item.h:709
virtual bool VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
bool ParseDiffIndex(std::string const &IndexDiffFile)
Parse the Index file for a set of Packages diffs.
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
std::vector< DiffInfo > available_patches
Definition: acquire-item.h:710
APT_HIDDEN void QueueOnIMSHit() const
pkgAcqDiffIndex(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target) APT_NONNULL(2
Create a new pkgAcqDiffIndex.
virtual std::string GetFinalFilename() const APT_OVERRIDE
Get the full pathname of the final file for the current URI.
virtual std::string GetMetaKey() const APT_OVERRIDE
bool Debug
If true, debugging information will be written to std::clog.
Definition: acquire-item.h:715
virtual ~pkgAcqDiffIndex()
virtual bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE
Retrieve an arbitrary file to the current directory. {{{.
virtual std::string Custom600Headers() const APT_OVERRIDE
bool IsIndexFile
Should this file be considered a index file.
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
virtual bool HashesRequired() const APT_OVERRIDE
HashStringList const ExpectedHashes
virtual void Done(std::string const &Message, HashStringList const &CalcHashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
pkgAcqFile(pkgAcquire *const Owner, std::string const &URI, HashStringList const &Hashes, unsigned long long const Size, std::string const &Desc, std::string const &ShortDesc, std::string const &DestDir="", std::string const &DestFilename="", bool const IsIndexFile=false)
Create a new pkgAcqFile object.
virtual ~pkgAcqFile()
An item that is responsible for fetching server-merge patches {{{ that need to be applied to a given ...
Definition: acquire-item.h:848
virtual bool AcquireByHash() const APT_OVERRIDE
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
Called when the patch file failed to be downloaded.
pkgAcqIndexDiffs(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target, std::vector< DiffInfo > const &diffs=std::vector< DiffInfo >()) APT_NONNULL(2
Create an index diff item.
APT_HIDDEN bool QueueNextDiff()
Queue up the next diff download.
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual ~pkgAcqIndexDiffs()
virtual std::string Custom600Headers() const APT_OVERRIDE
std::vector< DiffInfo > available_patches
Definition: acquire-item.h:890
APT_HIDDEN void Finish(bool const allDone=false)
Handle tasks that must be performed after the item finishes downloading.
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
@ StateApplyDiff
The diff is currently being applied.
Definition: acquire-item.h:899
@ StateFetchDiff
The diff is currently being fetched.
Definition: acquire-item.h:896
virtual bool HashesRequired() const APT_OVERRIDE
bool Debug
If true, debugging output will be written to std::clog.
Definition: acquire-item.h:880
enum pkgAcqIndexDiffs::DiffState State
An item that is responsible for fetching client-merge patches {{{ that need to be applied to a given ...
Definition: acquire-item.h:773
pkgAcqIndexMergeDiffs(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target, DiffInfo const &patch, std::vector< pkgAcqIndexMergeDiffs * > const *const allPatches) APT_NONNULL(2
Create an index merge-diff item.
virtual ~pkgAcqIndexMergeDiffs()
virtual bool AcquireByHash() const APT_OVERRIDE
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
Called when the patch file failed to be downloaded.
std::vector< pkgAcqIndexMergeDiffs * > const *const allPatches
list of all download items for the patches
Definition: acquire-item.h:785
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
struct DiffInfo const patch
information about the current patch
Definition: acquire-item.h:782
virtual std::string Custom600Headers() const APT_OVERRIDE
enum pkgAcqIndexMergeDiffs::DiffState State
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
@ StateErrorDiff
something bad happened and fallback was triggered
Definition: acquire-item.h:800
@ StateApplyDiff
The diff is currently being applied.
Definition: acquire-item.h:794
@ StateFetchDiff
The diff is currently being fetched.
Definition: acquire-item.h:791
@ StateDoneDiff
the work with this diff is done
Definition: acquire-item.h:797
virtual bool HashesRequired() const APT_OVERRIDE
bool Debug
If true, debugging output will be written to std::clog.
Definition: acquire-item.h:779
An acquire item that is responsible for fetching an index {{{ file (e.g., Packages or Sources).
Definition: acquire-item.h:618
std::string CompressionExtensions
The compression-related file extensions that are being added to the downloaded file one by one if fir...
Definition: acquire-item.h:650
void StageDecompressDone()
Handle what needs to be done when the decompression/copy is done.
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE
virtual std::string Custom600Headers() const APT_OVERRIDE
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual ~pkgAcqIndex()
std::string CurrentCompressionExtension
The actual compression extension currently used.
Definition: acquire-item.h:653
@ STAGE_DECOMPRESS_AND_VERIFY
Definition: acquire-item.h:630
virtual std::string GetFinalFilename() const APT_OVERRIDE
Get the full pathname of the final file for the current URI.
virtual std::string GetMetaKey() const APT_OVERRIDE
APT_HIDDEN bool CommonFailed(std::string const &TargetURI, std::string const &Message, pkgAcquire::MethodConfig const *const Cnf)
APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc)
pkgAcqIndex(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target, bool const Derived=false) APT_NONNULL(2
std::string EraseFileName
If set, this partially downloaded file will be removed when the download completes.
Definition: acquire-item.h:645
AllStages Stage
Definition: acquire-item.h:632
void StageDownloadDone(std::string const &Message)
Handle what needs to be done when the download is done.
the manager of a transaction
Definition: acquire-item.h:404
std::vector< pkgAcqTransactionItem * > Transaction
Definition: acquire-item.h:407
void TransactionStageRemoval(pkgAcqTransactionItem *const I, const std::string &FinalFile)
Stage (queue) a removal action when the transaction is committed.
void CommitTransaction()
bool CheckAuthDone(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf)
Called when authentication succeeded.
std::string BaseURI
Definition: acquire-item.h:457
bool AuthPass
If true, the index's signature is currently being verified.
Definition: acquire-item.h:411
void AbortTransaction()
TransactionStates State
Definition: acquire-item.h:456
virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE
virtual std::string Custom600Headers() const APT_OVERRIDE
void TransactionStageCopy(pkgAcqTransactionItem *const I, const std::string &From, const std::string &To)
Stage (queue) a copy action when the transaction is committed.
virtual std::string GetFinalFilename() const APT_OVERRIDE
Get the full pathname of the final file for the current URI.
bool CheckStopAuthentication(pkgAcquire::Item *const I, const std::string &Message)
bool TransactionHasError() const
void Add(pkgAcqTransactionItem *const I)
bool VerifyVendor(std::string const &Message)
Check that the release file is a release file for the correct distribution.
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
bool CheckDownloadDone(pkgAcqTransactionItem *const I, const std::string &Message, HashStringList const &Hashes) const
Called when a file is finished being retrieved.
virtual bool HashesRequired() const APT_OVERRIDE
void QueueForSignatureVerify(pkgAcqTransactionItem *const I, std::string const &File, std::string const &Signature)
Queue the downloaded Signature for verification.
virtual ~pkgAcqMetaBase()
virtual bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE
An item responsible for downloading clearsigned metaindexes {{{.
Definition: acquire-item.h:560
void QueueIndexes(bool const verify)
Starts downloading the individual index files.
virtual bool VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
metaIndex * MetaIndexParser
A package-system-specific parser for the meta-index file.
Definition: acquire-item.h:566
virtual std::string Custom600Headers() const APT_OVERRIDE
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual void Finished() APT_OVERRIDE
metaIndex * LastMetaIndexParser
Definition: acquire-item.h:567
IndexTarget const DetachedDataTarget
Definition: acquire-item.h:562
virtual ~pkgAcqMetaClearSig()
An item that is responsible for downloading the meta-index {{{ file (i.e., Release) itself and verify...
Definition: acquire-item.h:497
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual ~pkgAcqMetaIndex()
friend class pkgAcqMetaSig
Definition: acquire-item.h:518
pkgAcqMetaIndex(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget) APT_NONNULL(2
Create a new pkgAcqMetaIndex.
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
IndexTarget const DetachedSigTarget
Definition: acquire-item.h:500
virtual std::string DescURI() const APT_OVERRIDE
An acquire item that downloads the detached signature {{{ of a meta-index (Release) file,...
Definition: acquire-item.h:530
pkgAcqMetaIndex *const MetaIndex
Definition: acquire-item.h:533
virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE
virtual std::string Custom600Headers() const APT_OVERRIDE
virtual std::string GetFinalFilename() const APT_OVERRIDE
Get the full pathname of the final file for the current URI.
std::string MetaIndexFileSignature
The file we use to verify the MetaIndexFile with (not always set!)
Definition: acquire-item.h:536
pkgAcqMetaSig(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target, pkgAcqMetaIndex *const MetaIndex) APT_NONNULL(2
Create a new pkgAcqMetaSig.
virtual ~pkgAcqMetaSig()
baseclass for the indexes files to manage them all together
Definition: acquire-item.h:366
virtual ~pkgAcqTransactionItem()
virtual bool TransactionState(TransactionStates const state)
friend class pkgAcqMetaBase
Definition: acquire-item.h:398
pkgAcqMetaClearSig *const TransactionManager
TransactionManager.
Definition: acquire-item.h:380
HashStringList GetExpectedHashesFor(std::string const &MetaKey) const
virtual bool AcquireByHash() const
friend class pkgAcqMetaClearSig
Definition: acquire-item.h:399
virtual HashStringList GetExpectedHashes() const APT_OVERRIDE
IndexTarget const Target
Definition: acquire-item.h:374
virtual bool HashesRequired() const APT_OVERRIDE
std::string PartialFile
storge name until a transaction is finished
Definition: acquire-item.h:377
virtual std::string GetMetaKey() const
virtual std::string DescURI() const APT_OVERRIDE
Definition: acquire-item.h:389
pkgAcqTransactionItem(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, IndexTarget const &Target) APT_NONNULL(2
bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE
static APT_HIDDEN bool ReleaseInfoChangesAsGlobalErrors(std::vector< ReleaseInfoChange > &&Changes)
Definition: acquire.cc:1485
std::unordered_map< std::string, std::string > CustomFields
std::list< AlternateURI > AlternativeURIs
std::vector< std::string > BadAlternativeSites
std::vector< std::string > PastRedirections
Represents the process by which a pkgAcquire object should retrieve a file or a collection of files.
Definition: acquire-item.h:59
MethodConfig()
Set up the default method parameters.
friend class Item
Definition: acquire.h:106
std::string URI
URI from which to download this item.
Definition: acquire.h:380
static APT_HIDDEN std::string URIEncode(std::string const &part)
Definition: acquire.cc:54
Worker(Queue *OwnerQ, MethodConfig *Config, pkgAcquireStatus *Log)
Create a new Worker to download files.
IndexTarget GetIndexTarget() const APT_HIDDEN
Definition: indexfile.cc:248
virtual std::string ArchiveURI(std::string const &) const
Definition: indexfile.h:135
bool IsTrusted() const
Definition: indexfile.h:147
virtual std::string ArchiveInfo(pkgCache::VerIterator const &Ver) const
Definition: indexfile.cc:69
virtual std::string Describe(bool const Short=false) const =0
Parser & Lookup(pkgCache::VerFileIterator const &Ver)
Definition: pkgrecords.cc:62
bool FindIndex(pkgCache::PkgFileIterator File, pkgIndexFile *&Found) const
Definition: sourcelist.cc:471
bool Step(pkgTagSection &Section)
Definition: tagfile.cc:204
std::string FindS(APT::StringView sv) const
Definition: tagfile.h:70
Configuration * _config
string flNotFile(string File)
Definition: fileutl.cc:676
string flExtension(string File)
Definition: fileutl.cc:688
bool ExecWait(pid_t Pid, const char *Name, bool Reap)
Definition: fileutl.cc:942
string flNotDir(string File)
Definition: fileutl.cc:664
bool FileExists(string File)
Definition: fileutl.cc:326
std::string GetTempDir()
Definition: fileutl.cc:3103
bool ChangeOwnerAndPermissionOfFile(char const *const requester, char const *const file, char const *const user, char const *const group, mode_t const mode)
Definition: fileutl.cc:1015
string flCombine(string Dir, string File)
Definition: fileutl.cc:740
bool RemoveFile(char const *const Function, std::string const &FileName)
Definition: fileutl.cc:198
bool Rename(std::string From, std::string To)
Definition: fileutl.cc:3187
bool StartsWithGPGClearTextSignature(string const &FileName)
Definition: fileutl.cc:984
bool RealFileExists(string File)
Definition: fileutl.cc:337
bool RunScripts(const char *Cnf)
Definition: fileutl.cc:91
bool CopyFile(FileFd &From, FileFd &To)
Definition: fileutl.cc:164
pid_t ExecFork()
Definition: fileutl.cc:881
#define APT_OVERRIDE
Definition: macros.h:111
#define APT_PURE
Definition: macros.h:56
#define APT_HIDDEN
Definition: macros.h:78
TRI_YES
Definition: metaindex.h:31
APT_PUBLIC std::vector< std::string > const getCompressorExtensions()
Return a vector of extensions supported for data.tar's.
APT_PUBLIC std::vector< std::string > const getCompressionTypes(bool const &Cached=true)
Returns a vector of usable Compression Types.
bool Endswith(const std::string &s, const std::string &end)
Definition: strutl.cc:77
bool Startswith(const std::string &s, const std::string &start)
Definition: strutl.cc:84
pkgCache - Structure definitions for the cache file
HashStringList result_hashes
Definition: acquire-item.h:688
HashStringList download_hashes
Definition: acquire-item.h:694
std::string file
Definition: acquire-item.h:685
HashStringList Hashes
Definition: metaindex.h:26
std::unordered_map< std::string, std::string > changefields
AlternateURI(std::string &&u, decltype(changefields) &&cf)
int StringToBool(const string &Text, int Default)
Definition: strutl.cc:820
string SubstVar(const string &Str, const string &Subst, const string &Contents)
Definition: strutl.cc:502
string URItoFileName(const string &URI)
Definition: strutl.cc:553
vector< string > VectorizeString(string const &haystack, char const &split)
Definition: strutl.cc:1308
string TimeToStr(unsigned long Sec)
Definition: strutl.cc:473
string DeQuoteString(const string &Str)
Definition: strutl.cc:404
string QuoteString(const string &Str, const char *Bad)
Definition: strutl.cc:384
std::string LookupTag(const std::string &Message, const char *TagC, const char *Default)
Definition: strutl.cc:742
string StripEpoch(const string &VerStr)
Definition: strutl.cc:1486
string TimeRFC1123(time_t Date, bool const NumericTimezone)
Definition: strutl.cc:853