"Fossies" - the Fresh Open Source Software Archive

Member "apt-2.2.4/apt-pkg/acquire-item.cc" (10 Jun 2021, 146672 Bytes) of package /linux/misc/apt-2.2.4.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 // -*- mode: cpp; mode: fold -*-
    2 // Description                              /*{{{*/
    3 /* ######################################################################
    4 
    5    Acquire Item - Item to acquire
    6 
    7    Each item can download to exactly one file at a time. This means you
    8    cannot create an item that fetches two uri's to two files at the same 
    9    time. The pkgAcqIndex class creates a second class upon instantiation
   10    to fetch the other index files because of this.
   11 
   12    ##################################################################### */
   13                                     /*}}}*/
   14 // Include Files                            /*{{{*/
   15 #include <config.h>
   16 
   17 #include <apt-pkg/acquire-item.h>
   18 #include <apt-pkg/acquire-worker.h>
   19 #include <apt-pkg/acquire.h>
   20 #include <apt-pkg/aptconfiguration.h>
   21 #include <apt-pkg/configuration.h>
   22 #include <apt-pkg/error.h>
   23 #include <apt-pkg/fileutl.h>
   24 #include <apt-pkg/gpgv.h>
   25 #include <apt-pkg/hashes.h>
   26 #include <apt-pkg/indexfile.h>
   27 #include <apt-pkg/metaindex.h>
   28 #include <apt-pkg/pkgcache.h>
   29 #include <apt-pkg/pkgrecords.h>
   30 #include <apt-pkg/sourcelist.h>
   31 #include <apt-pkg/strutl.h>
   32 #include <apt-pkg/tagfile.h>
   33 
   34 #include <algorithm>
   35 #include <ctime>
   36 #include <chrono>
   37 #include <iostream>
   38 #include <memory>
   39 #include <numeric>
   40 #include <random>
   41 #include <sstream>
   42 #include <string>
   43 #include <unordered_set>
   44 #include <vector>
   45 #include <errno.h>
   46 #include <stddef.h>
   47 #include <stdio.h>
   48 #include <stdlib.h>
   49 #include <string.h>
   50 #include <sys/stat.h>
   51 #include <unistd.h>
   52 
   53 #include <apti18n.h>
   54                                     /*}}}*/
   55 
   56 using namespace std;
   57 
   58 static std::string GetPartialFileName(std::string const &file)      /*{{{*/
   59 {
   60    std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/";
   61    DestFile += file;
   62    return DestFile;
   63 }
   64                                     /*}}}*/
   65 static std::string GetPartialFileNameFromURI(std::string const &uri)    /*{{{*/
   66 {
   67    return GetPartialFileName(URItoFileName(uri));
   68 }
   69                                     /*}}}*/
   70 static std::string GetFinalFileNameFromURI(std::string const &uri)  /*{{{*/
   71 {
   72    return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
   73 }
   74                                     /*}}}*/
   75 static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/
   76 {
   77    if (Target.KeepCompressed == false)
   78       return file;
   79 
   80    std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
   81    if (KeepCompressedAs.empty() == false)
   82    {
   83       std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
   84       if (ext != "uncompressed")
   85      file.append(".").append(ext);
   86    }
   87    return file;
   88 }
   89                                     /*}}}*/
   90 static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
   91 {
   92    // rred expects the patch as $FinalFile.ed.$patchname.gz
   93    return Final + ".ed." + Patch + ".gz";
   94 }
   95                                     /*}}}*/
   96 static std::string GetDiffsPatchFileName(std::string const &Final)  /*{{{*/
   97 {
   98    // rred expects the patch as $FinalFile.ed
   99    return Final + ".ed";
  100 }
  101                                     /*}}}*/
  102 static std::string GetExistingFilename(std::string const &File)     /*{{{*/
  103 {
  104    if (RealFileExists(File))
  105       return File;
  106    for (auto const &type : APT::Configuration::getCompressorExtensions())
  107    {
  108       std::string const Final = File + type;
  109       if (RealFileExists(Final))
  110      return Final;
  111    }
  112    return "";
  113 }
  114                                     /*}}}*/
  115 static std::string GetDiffIndexFileName(std::string const &Name)    /*{{{*/
  116 {
  117    return Name + ".diff/Index";
  118 }
  119                                     /*}}}*/
  120 static std::string GetDiffIndexURI(IndexTarget const &Target)       /*{{{*/
  121 {
  122    return Target.URI + ".diff/Index";
  123 }
  124                                     /*}}}*/
  125 
  126 static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string const &FailCode, std::string const &Details)/*{{{*/
  127 {
  128    // we only act if a mirror was used at all
  129    if(I.UsedMirror.empty())
  130       return;
  131 #if 0
  132    std::cerr << "\nReportMirrorFailure: "
  133          << UsedMirror
  134          << " Uri: " << DescURI()
  135          << " FailCode: "
  136          << FailCode << std::endl;
  137 #endif
  138    string const report = _config->Find("Methods::Mirror::ProblemReporting",
  139                  LIBEXEC_DIR "/apt-report-mirror-failure");
  140    if(!FileExists(report))
  141       return;
  142 
  143    std::vector<char const*> const Args = {
  144       report.c_str(),
  145       I.UsedMirror.c_str(),
  146       I.DescURI().c_str(),
  147       FailCode.c_str(),
  148       Details.c_str(),
  149       NULL
  150    };
  151 
  152    pid_t pid = ExecFork();
  153    if(pid < 0)
  154    {
  155       _error->Error("ReportMirrorFailure Fork failed");
  156       return;
  157    }
  158    else if(pid == 0)
  159    {
  160       execvp(Args[0], (char**)Args.data());
  161       std::cerr << "Could not exec " << Args[0] << std::endl;
  162       _exit(100);
  163    }
  164    if(!ExecWait(pid, "report-mirror-failure"))
  165       _error->Warning("Couldn't report problem to '%s'", report.c_str());
  166 }
  167                                     /*}}}*/
  168 
  169 static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/
  170 {
  171    std::string m;
  172    strprintf(m, msg, repo.c_str());
  173    if (isError)
  174    {
  175       _error->Error("%s", m.c_str());
  176       _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default."));
  177    }
  178    else
  179    {
  180       _error->Warning("%s", m.c_str());
  181       _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use."));
  182    }
  183    _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details."));
  184    return false;
  185 }
  186                                     /*}}}*/
  187 // AllowInsecureRepositories                        /*{{{*/
  188 enum class InsecureType { UNSIGNED, WEAK, NORELEASE };
  189 static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type)
  190 {
  191    if (_config->FindB("Acquire::AllowInsecureRepositories"))
  192       return true;
  193 
  194    if (Target.OptionBool(IndexTarget::ALLOW_INSECURE))
  195       return true;
  196 
  197    switch (type)
  198    {
  199       case InsecureType::UNSIGNED: break;
  200       case InsecureType::NORELEASE: break;
  201       case InsecureType::WEAK:
  202      if (_config->FindB("Acquire::AllowWeakRepositories"))
  203         return true;
  204      if (Target.OptionBool(IndexTarget::ALLOW_WEAK))
  205         return true;
  206      break;
  207    }
  208    return false;
  209 }
  210 static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo,
  211       metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I)
  212 {
  213    // we skip weak downgrades as its unlikely that a repository gets really weaker –
  214    // its more realistic that apt got pickier in a newer version
  215    if (msg != InsecureType::WEAK)
  216    {
  217       std::string const FinalInRelease = TransactionManager->GetFinalFilename();
  218       std::string const FinalReleasegpg = FinalInRelease.substr(0, FinalInRelease.length() - strlen("InRelease")) + "Release.gpg";
  219       if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
  220       {
  221      char const * msgstr = nullptr;
  222      switch (msg)
  223      {
  224         case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is no longer signed."); break;
  225         case InsecureType::NORELEASE: msgstr = _("The repository '%s' no longer has a Release file."); break;
  226         case InsecureType::WEAK: /* unreachable */ break;
  227      }
  228      if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") ||
  229            TransactionManager->Target.OptionBool(IndexTarget::ALLOW_DOWNGRADE_TO_INSECURE))
  230      {
  231         // meh, the users wants to take risks (we still mark the packages
  232         // from this repository as unauthenticated)
  233         _error->Warning(msgstr, repo.c_str());
  234         _error->Warning(_("This is normally not allowed, but the option "
  235              "Acquire::AllowDowngradeToInsecureRepositories was "
  236              "given to override it."));
  237      } else {
  238         MessageInsecureRepository(true, msgstr, repo);
  239         TransactionManager->AbortTransaction();
  240         I->Status = pkgAcquire::Item::StatError;
  241         return false;
  242      }
  243       }
  244    }
  245 
  246    if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES)
  247       return true;
  248 
  249    char const * msgstr = nullptr;
  250    switch (msg)
  251    {
  252       case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is not signed."); break;
  253       case InsecureType::NORELEASE: msgstr = _("The repository '%s' does not have a Release file."); break;
  254       case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break;
  255    }
  256 
  257    if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true)
  258    {
  259       MessageInsecureRepository(false, msgstr, repo);
  260       return true;
  261    }
  262 
  263    MessageInsecureRepository(true, msgstr, repo);
  264    TransactionManager->AbortTransaction();
  265    I->Status = pkgAcquire::Item::StatError;
  266    return false;
  267 }
  268                                     /*}}}*/
  269 static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/
  270 {
  271    if (Parser == NULL)
  272       return HashStringList();
  273    metaIndex::checkSum * const R = Parser->Lookup(MetaKey);
  274    if (R == NULL)
  275       return HashStringList();
  276    return R->Hashes;
  277 }
  278                                     /*}}}*/
  279 
  280 class pkgAcquire::Item::Private /*{{{*/
  281 {
  282 public:
  283    struct AlternateURI
  284    {
  285       std::string URI;
  286       std::unordered_map<std::string, std::string> changefields;
  287       AlternateURI(std::string &&u, decltype(changefields) &&cf) : URI(u), changefields(cf) {}
  288    };
  289    std::list<AlternateURI> AlternativeURIs;
  290    std::vector<std::string> BadAlternativeSites;
  291    std::vector<std::string> PastRedirections;
  292    std::unordered_map<std::string, std::string> CustomFields;
  293 
  294    Private()
  295    {
  296    }
  297 };
  298                                     /*}}}*/
  299 
  300 // all ::HashesRequired and ::GetExpectedHashes implementations     /*{{{*/
  301 /* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
  302    It is best to implement it as broadly as possible, while ::HashesRequired defaults
  303    to true and should be as restrictive as possible for false cases. Note that if
  304    a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't
  305    ::HashesRequired is called to evaluate if its okay to have no hashes. */
  306 APT_PURE bool pkgAcqTransactionItem::HashesRequired() const
  307 {
  308    /* signed repositories obviously have a parser and good hashes.
  309       unsigned repositories, too, as even if we can't trust them for security,
  310       we can at least trust them for integrity of the download itself.
  311       Only repositories without a Release file can (obviously) not have
  312       hashes – and they are very uncommon and strongly discouraged */
  313    if (TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_YES)
  314       return false;
  315    if (TargetIsAllowedToBe(Target, InsecureType::WEAK))
  316    {
  317       /* If we allow weak hashes, we check that we have some (weak) and then
  318          declare hashes not needed. That will tip us in the right direction
  319      as if hashes exist, they will be used, even if not required */
  320       auto const hsl = GetExpectedHashes();
  321       if (hsl.usable())
  322      return true;
  323       if (hsl.empty() == false)
  324      return false;
  325    }
  326    return true;
  327 }
  328 HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
  329 {
  330    return GetExpectedHashesFor(GetMetaKey());
  331 }
  332 
  333 APT_PURE bool pkgAcqMetaBase::HashesRequired() const
  334 {
  335    // Release and co have no hashes 'by design'.
  336    return false;
  337 }
  338 HashStringList pkgAcqMetaBase::GetExpectedHashes() const
  339 {
  340    return HashStringList();
  341 }
  342 
  343 APT_PURE bool pkgAcqIndexDiffs::HashesRequired() const
  344 {
  345    /* We can't check hashes of rred result as we don't know what the
  346       hash of the file will be. We just know the hash of the patch(es),
  347       the hash of the file they will apply on and the hash of the resulting
  348       file. */
  349    if (State == StateFetchDiff)
  350       return true;
  351    return false;
  352 }
  353 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
  354 {
  355    if (State == StateFetchDiff)
  356       return available_patches[0].download_hashes;
  357    return HashStringList();
  358 }
  359 
  360 APT_PURE bool pkgAcqIndexMergeDiffs::HashesRequired() const
  361 {
  362    /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
  363       we can check the rred result after all patches are applied as
  364       we know the expected result rather than potentially apply more patches */
  365    if (State == StateFetchDiff)
  366       return true;
  367    return State == StateApplyDiff;
  368 }
  369 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
  370 {
  371    if (State == StateFetchDiff)
  372       return patch.download_hashes;
  373    else if (State == StateApplyDiff)
  374       return GetExpectedHashesFor(Target.MetaKey);
  375    return HashStringList();
  376 }
  377 
  378 APT_PURE bool pkgAcqArchive::HashesRequired() const
  379 {
  380    return LocalSource == false;
  381 }
  382 HashStringList pkgAcqArchive::GetExpectedHashes() const
  383 {
  384    // figured out while parsing the records
  385    return ExpectedHashes;
  386 }
  387 
  388 APT_PURE bool pkgAcqFile::HashesRequired() const
  389 {
  390    // supplied as parameter at creation time, so the caller decides
  391    return ExpectedHashes.usable();
  392 }
  393 HashStringList pkgAcqFile::GetExpectedHashes() const
  394 {
  395    return ExpectedHashes;
  396 }
  397                                     /*}}}*/
  398 // Acquire::Item::QueueURI and specialisations from child classes   /*{{{*/
  399 bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item)
  400 {
  401    Owner->Enqueue(Item);
  402    return true;
  403 }
  404 /* The idea here is that an item isn't queued if it exists on disk and the
  405    transition manager was a hit as this means that the files it contains
  406    the checksums for can't be updated either (or they are and we are asking
  407    for a hashsum mismatch to happen which helps nobody) */
  408 bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
  409 {
  410    if (TransactionManager->State != TransactionStarted)
  411    {
  412       if (_config->FindB("Debug::Acquire::Transaction", false))
  413      std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl;
  414       return false;
  415    }
  416    std::string const FinalFile = GetFinalFilename();
  417    if (TransactionManager->IMSHit == true && FileExists(FinalFile) == true)
  418    {
  419       PartialFile = DestFile = FinalFile;
  420       Status = StatDone;
  421       return false;
  422    }
  423    // this ensures we rewrite only once and only the first step
  424    auto const OldBaseURI = Target.Option(IndexTarget::BASE_URI);
  425    if (OldBaseURI.empty() || APT::String::Startswith(Item.URI, OldBaseURI) == false)
  426       return pkgAcquire::Item::QueueURI(Item);
  427    // the given URI is our last resort
  428    PushAlternativeURI(std::string(Item.URI), {}, false);
  429    // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too
  430    std::string SameMirrorURI;
  431    if (TransactionManager->BaseURI.empty() == false && TransactionManager->UsedMirror.empty() == false &&
  432        URI::SiteOnly(Item.URI) != URI::SiteOnly(TransactionManager->BaseURI))
  433    {
  434       auto ExtraPath = Item.URI.substr(OldBaseURI.length());
  435       auto newURI = flCombine(TransactionManager->BaseURI, std::move(ExtraPath));
  436       if (IsGoodAlternativeURI(newURI))
  437       {
  438      SameMirrorURI = std::move(newURI);
  439      PushAlternativeURI(std::string(SameMirrorURI), {}, false);
  440       }
  441    }
  442    // add URI and by-hash based on it
  443    if (AcquireByHash())
  444    {
  445       // if we use the mirror transport, ask it for by-hash uris
  446       // we need to stick to the same mirror only for non-unique filenames
  447       auto const sameMirrorException = [&]() {
  448      if (Item.URI.find("mirror") == std::string::npos)
  449         return false;
  450      ::URI uri(Item.URI);
  451      return uri.Access == "mirror" || APT::String::Startswith(uri.Access, "mirror+") ||
  452         APT::String::Endswith(uri.Access, "+mirror") || uri.Access.find("+mirror+") != std::string::npos;
  453       }();
  454       if (sameMirrorException)
  455      SameMirrorURI.clear();
  456       // now add the actual by-hash uris
  457       auto const Expected = GetExpectedHashes();
  458       auto const TargetHash = Expected.find(nullptr);
  459       auto const PushByHashURI = [&](std::string U) {
  460      if (unlikely(TargetHash == nullptr))
  461         return false;
  462      auto const trailing_slash = U.find_last_of("/");
  463      if (unlikely(trailing_slash == std::string::npos))
  464         return false;
  465      auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
  466      U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix));
  467      PushAlternativeURI(std::move(U), {}, false);
  468      return true;
  469       };
  470       PushByHashURI(Item.URI);
  471       if (SameMirrorURI.empty() == false && PushByHashURI(SameMirrorURI) == false)
  472      SameMirrorURI.clear();
  473    }
  474    // the last URI added is the first one tried
  475    if (unlikely(PopAlternativeURI(Item.URI) == false))
  476       return false;
  477    if (SameMirrorURI.empty() == false)
  478    {
  479       UsedMirror = TransactionManager->UsedMirror;
  480       if (Item.Description.find(" ") != string::npos)
  481      Item.Description.replace(0, Item.Description.find(" "), UsedMirror);
  482    }
  483    return pkgAcquire::Item::QueueURI(Item);
  484 }
  485 /* The transition manager InRelease itself (or its older sisters-in-law
  486    Release & Release.gpg) is always queued as this allows us to rerun gpgv
  487    on it to verify that we aren't stalled with old files */
  488 bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
  489 {
  490    return pkgAcquire::Item::QueueURI(Item);
  491 }
  492 /* the Diff/Index needs to queue also the up-to-date complete index file
  493    to ensure that the list cleaner isn't eating it */
  494 bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
  495 {
  496    if (pkgAcqTransactionItem::QueueURI(Item) == true)
  497       return true;
  498    QueueOnIMSHit();
  499    return false;
  500 }
  501                                     /*}}}*/
  502 // Acquire::Item::GetFinalFilename and specialisations for child classes    /*{{{*/
  503 std::string pkgAcquire::Item::GetFinalFilename() const
  504 {
  505    // Beware: Desc.URI is modified by redirections
  506    return GetFinalFileNameFromURI(Desc.URI);
  507 }
  508 std::string pkgAcqDiffIndex::GetFinalFilename() const
  509 {
  510    std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
  511    // we don't want recompress, so lets keep whatever we got
  512    if (CurrentCompressionExtension == "uncompressed")
  513       return FinalFile;
  514    return FinalFile + "." + CurrentCompressionExtension;
  515 }
  516 std::string pkgAcqIndex::GetFinalFilename() const
  517 {
  518    std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
  519    return GetKeepCompressedFileName(FinalFile, Target);
  520 }
  521 std::string pkgAcqMetaSig::GetFinalFilename() const
  522 {
  523    return GetFinalFileNameFromURI(Target.URI);
  524 }
  525 std::string pkgAcqBaseIndex::GetFinalFilename() const
  526 {
  527    return GetFinalFileNameFromURI(Target.URI);
  528 }
  529 std::string pkgAcqMetaBase::GetFinalFilename() const
  530 {
  531    return GetFinalFileNameFromURI(Target.URI);
  532 }
  533 std::string pkgAcqArchive::GetFinalFilename() const
  534 {
  535    return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
  536 }
  537                                     /*}}}*/
  538 // pkgAcqTransactionItem::GetMetaKey and specialisations for child classes  /*{{{*/
  539 std::string pkgAcqTransactionItem::GetMetaKey() const
  540 {
  541    return Target.MetaKey;
  542 }
  543 std::string pkgAcqIndex::GetMetaKey() const
  544 {
  545    if (Stage == STAGE_DECOMPRESS_AND_VERIFY || CurrentCompressionExtension == "uncompressed")
  546       return Target.MetaKey;
  547    return Target.MetaKey + "." + CurrentCompressionExtension;
  548 }
  549 std::string pkgAcqDiffIndex::GetMetaKey() const
  550 {
  551    auto const metakey = GetDiffIndexFileName(Target.MetaKey);
  552    if (CurrentCompressionExtension == "uncompressed")
  553       return metakey;
  554    return metakey + "." + CurrentCompressionExtension;
  555 }
  556                                     /*}}}*/
  557 //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
  558 bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
  559 {
  560    bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
  561    switch(state)
  562    {
  563       case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
  564       case TransactionAbort:
  565      if(Debug == true)
  566         std::clog << "  Cancel: " << DestFile << std::endl;
  567      if (Status == pkgAcquire::Item::StatIdle)
  568      {
  569         Status = pkgAcquire::Item::StatDone;
  570         Dequeue();
  571      }
  572      break;
  573       case TransactionCommit:
  574      if(PartialFile.empty() == false)
  575      {
  576         bool sameFile = (PartialFile == DestFile);
  577         // we use symlinks on IMS-Hit to avoid copies
  578         if (RealFileExists(DestFile))
  579         {
  580            struct stat Buf;
  581            if (lstat(PartialFile.c_str(), &Buf) != -1)
  582            {
  583           if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
  584           {
  585              char partial[Buf.st_size + 1];
  586              ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
  587              if (sp == -1)
  588             _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
  589              else
  590              {
  591             partial[sp] = '\0';
  592             sameFile = (DestFile == partial);
  593              }
  594           }
  595            }
  596            else
  597           _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
  598         }
  599         if (sameFile == false)
  600         {
  601            // ensure that even without lists-cleanup all compressions are nuked
  602            std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
  603            if (FileExists(FinalFile))
  604            {
  605           if(Debug == true)
  606              std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl;
  607           if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false)
  608              return false;
  609            }
  610            for (auto const &ext: APT::Configuration::getCompressorExtensions())
  611            {
  612           auto const Final = FinalFile + ext;
  613           if (FileExists(Final))
  614           {
  615              if(Debug == true)
  616             std::clog << "rm " << Final << " # " << DescURI() << std::endl;
  617              if (RemoveFile("TransactionStates-Cleanup", Final) == false)
  618             return false;
  619           }
  620            }
  621            if(Debug == true)
  622           std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
  623            if (Rename(PartialFile, DestFile) == false)
  624           return false;
  625         }
  626         else if(Debug == true)
  627            std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl;
  628 
  629      } else {
  630         if(Debug == true)
  631            std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
  632         if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
  633            return false;
  634      }
  635      break;
  636    }
  637    return true;
  638 }
  639 bool pkgAcqMetaBase::TransactionState(TransactionStates const state)
  640 {
  641    // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
  642    if (TransactionManager->IMSHit == false)
  643       return pkgAcqTransactionItem::TransactionState(state);
  644    return true;
  645 }
  646 bool pkgAcqIndex::TransactionState(TransactionStates const state)
  647 {
  648    if (pkgAcqTransactionItem::TransactionState(state) == false)
  649       return false;
  650 
  651    switch (state)
  652    {
  653       case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break;
  654       case TransactionAbort:
  655      if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
  656      {
  657         // keep the compressed file, but drop the decompressed
  658         EraseFileName.clear();
  659         if (PartialFile.empty() == false && flExtension(PartialFile) != CurrentCompressionExtension)
  660            RemoveFile("TransactionAbort", PartialFile);
  661      }
  662      break;
  663       case TransactionCommit:
  664      if (EraseFileName.empty() == false)
  665         RemoveFile("AcqIndex::TransactionCommit", EraseFileName);
  666      break;
  667    }
  668    return true;
  669 }
  670 bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)
  671 {
  672    if (pkgAcqTransactionItem::TransactionState(state) == false)
  673       return false;
  674 
  675    switch (state)
  676    {
  677       case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
  678       case TransactionCommit:
  679      break;
  680       case TransactionAbort:
  681      std::string const Partial = GetPartialFileNameFromURI(Target.URI);
  682      RemoveFile("TransactionAbort", Partial);
  683      break;
  684    }
  685 
  686    return true;
  687 }
  688                                     /*}}}*/
  689 // pkgAcqTransactionItem::AcquireByHash and specialisations for child classes   /*{{{*/
  690 bool pkgAcqTransactionItem::AcquireByHash() const
  691 {
  692    if (TransactionManager->MetaIndexParser == nullptr)
  693       return false;
  694    auto const useByHashConf = Target.Option(IndexTarget::BY_HASH);
  695    if (useByHashConf == "force")
  696       return true;
  697    return StringToBool(useByHashConf) == true && TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
  698 }
  699 // pdiff patches have a unique name already, no need for by-hash
  700 bool pkgAcqIndexMergeDiffs::AcquireByHash() const
  701 {
  702    return false;
  703 }
  704 bool pkgAcqIndexDiffs::AcquireByHash() const
  705 {
  706    return false;
  707 }
  708                                     /*}}}*/
  709 
  710 class APT_HIDDEN NoActionItem : public pkgAcquire::Item         /*{{{*/
  711 /* The sole purpose of this class is having an item which does nothing to
  712    reach its done state to prevent cleanup deleting the mentioned file.
  713    Handy in cases in which we know we have the file already, like IMS-Hits. */
  714 {
  715    IndexTarget const Target;
  716    public:
  717    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
  718    virtual HashStringList GetExpectedHashes()  const APT_OVERRIDE {return HashStringList();};
  719 
  720    NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) :
  721       pkgAcquire::Item(Owner), Target(Target)
  722    {
  723       Status = StatDone;
  724       DestFile = GetFinalFileNameFromURI(Target.URI);
  725    }
  726    NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) :
  727       pkgAcquire::Item(Owner), Target(Target)
  728    {
  729       Status = StatDone;
  730       DestFile = FinalFile;
  731    }
  732 };
  733                                     /*}}}*/
  734 class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem     /*{{{*/
  735 /* This class ensures that a file which was configured but isn't downloaded
  736    for various reasons isn't kept in an old version in the lists directory.
  737    In a way its the reverse of NoActionItem as it helps with removing files
  738    even if the lists-cleanup is deactivated. */
  739 {
  740    public:
  741    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
  742    virtual HashStringList GetExpectedHashes()  const APT_OVERRIDE {return HashStringList();};
  743 
  744    CleanupItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) :
  745       pkgAcqTransactionItem(Owner, TransactionManager, Target)
  746    {
  747       Status = StatDone;
  748       DestFile = GetFinalFileNameFromURI(Target.URI);
  749    }
  750    bool TransactionState(TransactionStates const state) APT_OVERRIDE
  751    {
  752       switch (state)
  753       {
  754      case TransactionStarted:
  755         break;
  756      case TransactionAbort:
  757         break;
  758      case TransactionCommit:
  759         if (_config->FindB("Debug::Acquire::Transaction", false) == true)
  760            std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
  761         if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
  762            return false;
  763         break;
  764       }
  765       return true;
  766    }
  767 };
  768                                     /*}}}*/
  769 
  770 // Acquire::Item::Item - Constructor                    /*{{{*/
  771 pkgAcquire::Item::Item(pkgAcquire * const owner) :
  772    FileSize(0), PartialSize(0), ID(0), Complete(false), Local(false),
  773     QueueCounter(0), ExpectedAdditionalItems(0), Retries(_config->FindI("Acquire::Retries", 0)), Owner(owner), d(new Private())
  774 {
  775    Owner->Add(this);
  776    Status = StatIdle;
  777 }
  778                                     /*}}}*/
  779 // Acquire::Item::~Item - Destructor                    /*{{{*/
  780 pkgAcquire::Item::~Item()
  781 {
  782    Owner->Remove(this);
  783    delete d;
  784 }
  785                                     /*}}}*/
  786 std::string pkgAcquire::Item::Custom600Headers() const          /*{{{*/
  787 {
  788    std::ostringstream header;
  789    for (auto const &f : d->CustomFields)
  790       if (f.second.empty() == false)
  791      header << '\n'
  792         << f.first << ": " << f.second;
  793    return header.str();
  794 }
  795                                     /*}}}*/
  796 std::unordered_map<std::string, std::string> &pkgAcquire::Item::ModifyCustomFields() /*{{{*/
  797 {
  798    return d->CustomFields;
  799 }
  800                                     /*}}}*/
  801 bool pkgAcquire::Item::PopAlternativeURI(std::string &NewURI) /*{{{*/
  802 {
  803    if (d->AlternativeURIs.empty())
  804       return false;
  805    auto const AltUri = d->AlternativeURIs.front();
  806    d->AlternativeURIs.pop_front();
  807    NewURI = AltUri.URI;
  808    auto &CustomFields = ModifyCustomFields();
  809    for (auto const &f : AltUri.changefields)
  810    {
  811       if (f.second.empty())
  812      CustomFields.erase(f.first);
  813       else
  814      CustomFields[f.first] = f.second;
  815    }
  816    return true;
  817 }
  818                                     /*}}}*/
  819 bool pkgAcquire::Item::IsGoodAlternativeURI(std::string const &AltUri) const/*{{{*/
  820 {
  821    return std::find(d->PastRedirections.cbegin(), d->PastRedirections.cend(), AltUri) == d->PastRedirections.cend() &&
  822      std::find(d->BadAlternativeSites.cbegin(), d->BadAlternativeSites.cend(), URI::SiteOnly(AltUri)) == d->BadAlternativeSites.cend();
  823 }
  824                                     /*}}}*/
  825 void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back) /*{{{*/
  826 {
  827    if (IsGoodAlternativeURI(NewURI) == false)
  828       return;
  829    if (at_the_back)
  830       d->AlternativeURIs.emplace_back(std::move(NewURI), std::move(fields));
  831    else
  832       d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields));
  833 }
  834                                     /*}}}*/
  835 void pkgAcquire::Item::RemoveAlternativeSite(std::string &&OldSite) /*{{{*/
  836 {
  837    d->AlternativeURIs.erase(std::remove_if(d->AlternativeURIs.begin(), d->AlternativeURIs.end(),
  838                        [&](decltype(*d->AlternativeURIs.cbegin()) AltUri) {
  839                           return URI::SiteOnly(AltUri.URI) == OldSite;
  840                        }),
  841                 d->AlternativeURIs.end());
  842    d->BadAlternativeSites.push_back(std::move(OldSite));
  843 }
  844                                     /*}}}*/
  845 std::string pkgAcquire::Item::ShortDesc() const             /*{{{*/
  846 {
  847    return DescURI();
  848 }
  849                                     /*}}}*/
  850 void pkgAcquire::Item::Finished()                   /*{{{*/
  851 {
  852 }
  853                                     /*}}}*/
  854 APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const        /*{{{*/
  855 {
  856    return Owner;
  857 }
  858                                     /*}}}*/
  859 APT_PURE pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc()      /*{{{*/
  860 {
  861    return Desc;
  862 }
  863                                     /*}}}*/
  864 APT_PURE bool pkgAcquire::Item::IsTrusted() const           /*{{{*/
  865 {
  866    return false;
  867 }
  868                                     /*}}}*/
  869 // Acquire::Item::Failed - Item failed to download          /*{{{*/
  870 // ---------------------------------------------------------------------
  871 /* We return to an idle state if there are still other queues that could
  872    fetch this object */
  873 static void formatHashsum(std::ostream &out, HashString const &hs)
  874 {
  875    auto const type = hs.HashType();
  876    if (type == "Checksum-FileSize")
  877       out << " - Filesize";
  878    else
  879       out << " - " << type;
  880    out << ':' << hs.HashValue();
  881    if (hs.usable() == false)
  882       out << " [weak]";
  883    out << std::endl;
  884 }
  885 void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
  886 {
  887    if (QueueCounter <= 1)
  888    {
  889       /* This indicates that the file is not available right now but might
  890          be sometime later. If we do a retry cycle then this should be
  891      retried [CDROMs] */
  892       if (Cnf != NULL && Cnf->LocalOnly == true &&
  893       StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  894       {
  895      Status = StatIdle;
  896      Dequeue();
  897      return;
  898       }
  899 
  900       switch (Status)
  901       {
  902      case StatIdle:
  903      case StatFetching:
  904      case StatDone:
  905         Status = StatError;
  906         break;
  907      case StatAuthError:
  908      case StatError:
  909      case StatTransientNetworkError:
  910         break;
  911       }
  912       Complete = false;
  913       Dequeue();
  914    }
  915 
  916    FailMessage(Message);
  917 
  918    if (QueueCounter > 1)
  919       Status = StatIdle;
  920 }
  921 void pkgAcquire::Item::FailMessage(string const &Message)
  922 {
  923    string const FailReason = LookupTag(Message, "FailReason");
  924    enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, REDIRECTION_LOOP, OTHER } failreason = OTHER;
  925    if ( FailReason == "MaximumSizeExceeded")
  926       failreason = MAXIMUM_SIZE_EXCEEDED;
  927    else if ( FailReason == "WeakHashSums")
  928       failreason = WEAK_HASHSUMS;
  929    else if (FailReason == "RedirectionLoop")
  930       failreason = REDIRECTION_LOOP;
  931    else if (Status == StatAuthError)
  932       failreason = HASHSUM_MISMATCH;
  933 
  934    if(ErrorText.empty())
  935    {
  936       std::ostringstream out;
  937       switch (failreason)
  938       {
  939      case HASHSUM_MISMATCH:
  940         out << _("Hash Sum mismatch") << std::endl;
  941         break;
  942      case WEAK_HASHSUMS:
  943         out << _("Insufficient information available to perform this download securely") << std::endl;
  944         break;
  945      case REDIRECTION_LOOP:
  946         out << "Redirection loop encountered" << std::endl;
  947         break;
  948      case MAXIMUM_SIZE_EXCEEDED:
  949         out << LookupTag(Message, "Message") << std::endl;
  950         break;
  951      case OTHER:
  952         out << LookupTag(Message, "Message");
  953         break;
  954       }
  955 
  956       if (Status == StatAuthError)
  957       {
  958      auto const ExpectedHashes = GetExpectedHashes();
  959      if (ExpectedHashes.empty() == false)
  960      {
  961         out << "Hashes of expected file:" << std::endl;
  962         for (auto const &hs: ExpectedHashes)
  963            formatHashsum(out, hs);
  964      }
  965      if (failreason == HASHSUM_MISMATCH)
  966      {
  967         out << "Hashes of received file:" << std::endl;
  968         for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
  969         {
  970            std::string const tagname = std::string(*type) + "-Hash";
  971            std::string const hashsum = LookupTag(Message, tagname.c_str());
  972            if (hashsum.empty() == false)
  973           formatHashsum(out, HashString(*type, hashsum));
  974         }
  975      }
  976      auto const lastmod = LookupTag(Message, "Last-Modified", "");
  977      if (lastmod.empty() == false)
  978         out << "Last modification reported: " << lastmod << std::endl;
  979       }
  980       ErrorText = out.str();
  981    }
  982 
  983    switch (failreason)
  984    {
  985       case MAXIMUM_SIZE_EXCEEDED: RenameOnError(MaximumSizeExceeded); break;
  986       case HASHSUM_MISMATCH: RenameOnError(HashSumMismatch); break;
  987       case WEAK_HASHSUMS: break;
  988       case REDIRECTION_LOOP: break;
  989       case OTHER: break;
  990    }
  991 
  992    if (FailReason.empty() == false)
  993       ReportMirrorFailureToCentral(*this, FailReason, ErrorText);
  994    else
  995       ReportMirrorFailureToCentral(*this, ErrorText, ErrorText);
  996 }
  997                                     /*}}}*/
  998 // Acquire::Item::Start - Item has begun to download            /*{{{*/
  999 // ---------------------------------------------------------------------
 1000 /* Stash status and the file size. Note that setting Complete means
 1001    sub-phases of the acquire process such as decompresion are operating */
 1002 void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size)
 1003 {
 1004    Status = StatFetching;
 1005    ErrorText.clear();
 1006    if (FileSize == 0 && Complete == false)
 1007       FileSize = Size;
 1008 }
 1009                                     /*}}}*/
 1010 // Acquire::Item::VerifyDone - check if Item was downloaded OK      /*{{{*/
 1011 /* Note that hash-verification is 'hardcoded' in acquire-worker and has
 1012  * already passed if this method is called. */
 1013 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
 1014      pkgAcquire::MethodConfig const * const /*Cnf*/)
 1015 {
 1016    std::string const FileName = LookupTag(Message,"Filename");
 1017    if (FileName.empty() == true)
 1018    {
 1019       Status = StatError;
 1020       ErrorText = "Method gave a blank filename";
 1021       return false;
 1022    }
 1023 
 1024    return true;
 1025 }
 1026                                     /*}}}*/
 1027 // Acquire::Item::Done - Item downloaded OK             /*{{{*/
 1028 void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes,
 1029                 pkgAcquire::MethodConfig const * const /*Cnf*/)
 1030 {
 1031    // We just downloaded something..
 1032    if (FileSize == 0)
 1033    {
 1034       unsigned long long const downloadedSize = Hashes.FileSize();
 1035       if (downloadedSize != 0)
 1036       {
 1037      FileSize = downloadedSize;
 1038       }
 1039    }
 1040    Status = StatDone;
 1041    ErrorText.clear();
 1042    Dequeue();
 1043 }
 1044                                     /*}}}*/
 1045 // Acquire::Item::Rename - Rename a file                /*{{{*/
 1046 // ---------------------------------------------------------------------
 1047 /* This helper function is used by a lot of item methods as their final
 1048    step */
 1049 bool pkgAcquire::Item::Rename(string const &From,string const &To)
 1050 {
 1051    if (From == To || rename(From.c_str(),To.c_str()) == 0)
 1052       return true;
 1053 
 1054    std::string S;
 1055    strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
 1056      From.c_str(),To.c_str());
 1057    Status = StatError;
 1058    if (ErrorText.empty())
 1059       ErrorText = S;
 1060    else
 1061       ErrorText = ErrorText + ": " + S;
 1062    return false;
 1063 }
 1064                                     /*}}}*/
 1065 void pkgAcquire::Item::Dequeue()                    /*{{{*/
 1066 {
 1067    d->AlternativeURIs.clear();
 1068    Owner->Dequeue(this);
 1069 }
 1070                                     /*}}}*/
 1071 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 1072 {
 1073    if (RealFileExists(DestFile))
 1074       Rename(DestFile, DestFile + ".FAILED");
 1075 
 1076    std::string errtext;
 1077    switch (error)
 1078    {
 1079       case HashSumMismatch:
 1080      errtext = _("Hash Sum mismatch");
 1081      break;
 1082       case SizeMismatch:
 1083      errtext = _("Size mismatch");
 1084      Status = StatAuthError;
 1085      break;
 1086       case InvalidFormat:
 1087      errtext = _("Invalid file format");
 1088      Status = StatError;
 1089      // do not report as usually its not the mirrors fault, but Portal/Proxy
 1090      break;
 1091       case SignatureError:
 1092      errtext = _("Signature error");
 1093      Status = StatError;
 1094      break;
 1095       case NotClearsigned:
 1096      strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT");
 1097      Status = StatAuthError;
 1098      break;
 1099       case MaximumSizeExceeded:
 1100      // the method is expected to report a good error for this
 1101      break;
 1102       case PDiffError:
 1103      // no handling here, done by callers
 1104      break;
 1105    }
 1106    if (ErrorText.empty())
 1107       ErrorText = errtext;
 1108    return false;
 1109 }
 1110                                     /*}}}*/
 1111 void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/
 1112 {
 1113       ActiveSubprocess = subprocess;
 1114 }
 1115                                     /*}}}*/
 1116 std::string pkgAcquire::Item::HashSum() const               /*{{{*/
 1117 {
 1118    HashStringList const hashes = GetExpectedHashes();
 1119    HashString const * const hs = hashes.find(NULL);
 1120    return hs != NULL ? hs->toStr() : "";
 1121 }
 1122                                     /*}}}*/
 1123 bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/
 1124 {
 1125    // store can fail due to permission errors and the item will "loop" then
 1126    if (APT::String::Startswith(NewURI, "store:"))
 1127       return false;
 1128    if (d->PastRedirections.empty())
 1129    {
 1130       d->PastRedirections.push_back(NewURI);
 1131       return false;
 1132    }
 1133    auto const LastURI = std::prev(d->PastRedirections.end());
 1134    // redirections to the same file are a way of restarting/resheduling,
 1135    // individual methods will have to make sure that they aren't looping this way
 1136    if (*LastURI == NewURI)
 1137       return false;
 1138    if (std::find(d->PastRedirections.begin(), LastURI, NewURI) != LastURI)
 1139       return true;
 1140    d->PastRedirections.push_back(NewURI);
 1141    return false;
 1142 }
 1143                                     /*}}}*/
 1144 int pkgAcquire::Item::Priority()                    /*{{{*/
 1145 {
 1146    // Stage 0: Files requested by methods
 1147    // - they will usually not end up here, but if they do we make sure
 1148    //   to get them as soon as possible as they are probably blocking
 1149    //   the processing of files by the requesting method
 1150    if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr)
 1151       return 5000;
 1152    // Stage 1: Meta indices and diff indices
 1153    // - those need to be fetched first to have progress reporting working
 1154    //   for the rest
 1155    if (dynamic_cast<pkgAcqMetaSig*>(this) != nullptr
 1156        || dynamic_cast<pkgAcqMetaBase*>(this) != nullptr
 1157        || dynamic_cast<pkgAcqDiffIndex*>(this) != nullptr)
 1158       return 1000;
 1159    // Stage 2: Diff files
 1160    // - fetch before complete indexes so we can apply the diffs while fetching
 1161    //   larger files.
 1162    if (dynamic_cast<pkgAcqIndexDiffs*>(this) != nullptr ||
 1163        dynamic_cast<pkgAcqIndexMergeDiffs*>(this) != nullptr)
 1164       return 800;
 1165 
 1166    // Stage 3: The rest - complete index files and other stuff
 1167    return 500;
 1168 }
 1169                                     /*}}}*/
 1170 
 1171 pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner,  /*{{{*/
 1172       pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) :
 1173    pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager)
 1174 {
 1175    if (TransactionManager != this)
 1176       TransactionManager->Add(this);
 1177    ModifyCustomFields() = {
 1178       {"Target-Site", Target.Option(IndexTarget::SITE)},
 1179       {"Target-Repo-URI", Target.Option(IndexTarget::REPO_URI)},
 1180       {"Target-Base-URI", Target.Option(IndexTarget::BASE_URI)},
 1181       {"Target-Component", Target.Option(IndexTarget::COMPONENT)},
 1182       {"Target-Release", Target.Option(IndexTarget::RELEASE)},
 1183       {"Target-Architecture", Target.Option(IndexTarget::ARCHITECTURE)},
 1184       {"Target-Language", Target.Option(IndexTarget::LANGUAGE)},
 1185       {"Target-Type", "index"},
 1186    };
 1187 }
 1188                                     /*}}}*/
 1189 pkgAcqTransactionItem::~pkgAcqTransactionItem()             /*{{{*/
 1190 {
 1191 }
 1192                                     /*}}}*/
 1193 HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const    /*{{{*/
 1194 {
 1195    return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey);
 1196 }
 1197                                     /*}}}*/
 1198 
 1199 static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/
 1200 {
 1201    if (TransactionManager->IMSHit == true)
 1202       return;
 1203    if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
 1204    {
 1205       TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
 1206       if (TransactionManager->LastMetaIndexParser != NULL)
 1207       {
 1208      _error->PushToStack();
 1209      if (RealFileExists(FinalInRelease))
 1210         TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
 1211      else
 1212         TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
 1213      // its unlikely to happen, but if what we have is bad ignore it
 1214      if (_error->PendingError())
 1215      {
 1216         delete TransactionManager->LastMetaIndexParser;
 1217         TransactionManager->LastMetaIndexParser = NULL;
 1218      }
 1219      _error->RevertToStack();
 1220       }
 1221    }
 1222 }
 1223                                     /*}}}*/
 1224 
 1225 // AcqMetaBase - Constructor                        /*{{{*/
 1226 pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner,
 1227       pkgAcqMetaClearSig * const TransactionManager,
 1228       IndexTarget const &DataTarget)
 1229 : pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL),
 1230    AuthPass(false), IMSHit(false), State(TransactionStarted)
 1231 {
 1232 }
 1233                                     /*}}}*/
 1234 // AcqMetaBase::Add - Add a item to the current Transaction     /*{{{*/
 1235 void pkgAcqMetaBase::Add(pkgAcqTransactionItem * const I)
 1236 {
 1237    Transaction.push_back(I);
 1238 }
 1239                                     /*}}}*/
 1240 // AcqMetaBase::AbortTransaction - Abort the current Transaction    /*{{{*/
 1241 void pkgAcqMetaBase::AbortTransaction()
 1242 {
 1243    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
 1244       std::clog << "AbortTransaction: " << TransactionManager << std::endl;
 1245 
 1246    switch (TransactionManager->State)
 1247    {
 1248       case TransactionStarted: break;
 1249       case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return;
 1250       case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now committed", TransactionManager->Target.URI.c_str()); return;
 1251    }
 1252    TransactionManager->State = TransactionAbort;
 1253    TransactionManager->ExpectedAdditionalItems = 0;
 1254 
 1255    // ensure the toplevel is in error state too
 1256    for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
 1257         I != Transaction.end(); ++I)
 1258    {
 1259       (*I)->ExpectedAdditionalItems = 0;
 1260       if ((*I)->Status != pkgAcquire::Item::StatFetching)
 1261      (*I)->Dequeue();
 1262       (*I)->TransactionState(TransactionAbort);
 1263    }
 1264    Transaction.clear();
 1265 }
 1266                                     /*}}}*/
 1267 // AcqMetaBase::TransactionHasError - Check for errors in Transaction   /*{{{*/
 1268 APT_PURE bool pkgAcqMetaBase::TransactionHasError() const
 1269 {
 1270    for (std::vector<pkgAcqTransactionItem*>::const_iterator I = Transaction.begin();
 1271         I != Transaction.end(); ++I)
 1272    {
 1273       switch((*I)->Status) {
 1274      case StatDone: break;
 1275      case StatIdle: break;
 1276      case StatAuthError: return true;
 1277      case StatError: return true;
 1278      case StatTransientNetworkError: return true;
 1279      case StatFetching: break;
 1280       }
 1281    }
 1282    return false;
 1283 }
 1284                                     /*}}}*/
 1285 // AcqMetaBase::CommitTransaction - Commit a transaction        /*{{{*/
 1286 void pkgAcqMetaBase::CommitTransaction()
 1287 {
 1288    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
 1289       std::clog << "CommitTransaction: " << this << std::endl;
 1290 
 1291    switch (TransactionManager->State)
 1292    {
 1293       case TransactionStarted: break;
 1294       case TransactionAbort: _error->Fatal("Transaction %s was already committed and is now aborted", TransactionManager->Target.URI.c_str()); return;
 1295       case TransactionCommit: _error->Fatal("Transaction %s was already committed and is again committed", TransactionManager->Target.URI.c_str()); return;
 1296    }
 1297    TransactionManager->State = TransactionCommit;
 1298 
 1299    // move new files into place *and* remove files that are not
 1300    // part of the transaction but are still on disk
 1301    for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
 1302         I != Transaction.end(); ++I)
 1303    {
 1304       (*I)->TransactionState(TransactionCommit);
 1305    }
 1306    Transaction.clear();
 1307 }
 1308                                     /*}}}*/
 1309 // AcqMetaBase::TransactionStageCopy - Stage a file for copying     /*{{{*/
 1310 void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem * const I,
 1311                                           const std::string &From,
 1312                                           const std::string &To)
 1313 {
 1314    I->PartialFile = From;
 1315    I->DestFile = To;
 1316 }
 1317                                     /*}}}*/
 1318 // AcqMetaBase::TransactionStageRemoval - Stage a file for removal  /*{{{*/
 1319 void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem * const I,
 1320                                              const std::string &FinalFile)
 1321 {
 1322    I->PartialFile = "";
 1323    I->DestFile = FinalFile;
 1324 }
 1325                                     /*}}}*/
 1326 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error    /*{{{*/
 1327 /* This method is called from ::Failed handlers. If it returns true,
 1328    no fallback to other files or modi is performed */
 1329 bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
 1330 {
 1331    string const Final = I->GetFinalFilename();
 1332    std::string const GPGError = LookupTag(Message, "Message");
 1333    if (FileExists(Final))
 1334    {
 1335       I->Status = StatTransientNetworkError;
 1336       _error->Warning(_("An error occurred during the signature verification. "
 1337            "The repository is not updated and the previous index files will be used. "
 1338            "GPG error: %s: %s"),
 1339         Desc.Description.c_str(),
 1340         GPGError.c_str());
 1341       RunScripts("APT::Update::Auth-Failure");
 1342       return true;
 1343    } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
 1344       /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
 1345       _error->Error(_("GPG error: %s: %s"),
 1346         Desc.Description.c_str(),
 1347         GPGError.c_str());
 1348       I->Status = StatAuthError;
 1349       return true;
 1350    } else {
 1351       _error->Warning(_("GPG error: %s: %s"),
 1352         Desc.Description.c_str(),
 1353         GPGError.c_str());
 1354    }
 1355    // gpgv method failed
 1356    ReportMirrorFailureToCentral(*this, "GPGFailure", GPGError);
 1357    return false;
 1358 }
 1359                                     /*}}}*/
 1360 // AcqMetaBase::Custom600Headers - Get header for AcqMetaBase       /*{{{*/
 1361 // ---------------------------------------------------------------------
 1362 string pkgAcqMetaBase::Custom600Headers() const
 1363 {
 1364    std::string Header = pkgAcqTransactionItem::Custom600Headers();
 1365    Header.append("\nIndex-File: true");
 1366    std::string MaximumSize;
 1367    strprintf(MaximumSize, "\nMaximum-Size: %i",
 1368              _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
 1369    Header += MaximumSize;
 1370 
 1371    string const FinalFile = GetFinalFilename();
 1372    struct stat Buf;
 1373    if (stat(FinalFile.c_str(),&Buf) == 0)
 1374       Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
 1375 
 1376    return Header;
 1377 }
 1378                                     /*}}}*/
 1379 // AcqMetaBase::QueueForSignatureVerify                 /*{{{*/
 1380 void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature)
 1381 {
 1382    AuthPass = true;
 1383    I->Desc.URI = "gpgv:" + pkgAcquire::URIEncode(Signature);
 1384    I->DestFile = File;
 1385    QueueURI(I->Desc);
 1386    I->SetActiveSubprocess("gpgv");
 1387 }
 1388                                     /*}}}*/
 1389 // AcqMetaBase::CheckDownloadDone                   /*{{{*/
 1390 bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const
 1391 {
 1392    // We have just finished downloading a Release file (it is not
 1393    // verified yet)
 1394 
 1395    // Save the final base URI we got this Release file from
 1396    if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
 1397    {
 1398       auto InReleasePath = Target.Option(IndexTarget::INRELEASE_PATH);
 1399       if (InReleasePath.empty())
 1400      InReleasePath = "InRelease";
 1401 
 1402       if (APT::String::Endswith(I->Desc.URI, InReleasePath))
 1403       {
 1404      TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - InReleasePath.length());
 1405      TransactionManager->UsedMirror = I->UsedMirror;
 1406       }
 1407       else if (APT::String::Endswith(I->Desc.URI, "Release"))
 1408       {
 1409      TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release"));
 1410      TransactionManager->UsedMirror = I->UsedMirror;
 1411       }
 1412    }
 1413 
 1414    std::string const FileName = LookupTag(Message,"Filename");
 1415    if (FileName != I->DestFile && RealFileExists(I->DestFile) == false)
 1416    {
 1417       I->Local = true;
 1418       I->Desc.URI = "copy:" + pkgAcquire::URIEncode(FileName);
 1419       I->QueueURI(I->Desc);
 1420       return false;
 1421    }
 1422 
 1423    // make sure to verify against the right file on I-M-S hit
 1424    bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
 1425    if (IMSHit == false && Hashes.usable())
 1426    {
 1427       // detect IMS-Hits servers haven't detected by Hash comparison
 1428       std::string const FinalFile = I->GetFinalFilename();
 1429       if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
 1430       {
 1431      IMSHit = true;
 1432      RemoveFile("CheckDownloadDone", I->DestFile);
 1433       }
 1434    }
 1435 
 1436    if(IMSHit == true)
 1437    {
 1438       // for simplicity, the transaction manager is always InRelease
 1439       // even if it doesn't exist.
 1440       TransactionManager->IMSHit = true;
 1441       I->PartialFile = I->DestFile = I->GetFinalFilename();
 1442    }
 1443 
 1444    // set Item to complete as the remaining work is all local (verify etc)
 1445    I->Complete = true;
 1446 
 1447    return true;
 1448 }
 1449                                     /*}}}*/
 1450 bool pkgAcqMetaBase::CheckAuthDone(string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/
 1451 {
 1452    /* If we work with a recent version of our gpgv method, we expect that it tells us
 1453       which key(s) have signed the file so stuff like CVE-2018-0501 is harder in the future */
 1454    if (Cnf->Version != "1.0" && LookupTag(Message, "Signed-By").empty())
 1455    {
 1456       std::string errmsg;
 1457       strprintf(errmsg, "Internal Error: Signature on %s seems good, but expected details are missing! (%s)", Target.URI.c_str(), "Signed-By");
 1458       if (ErrorText.empty())
 1459      ErrorText = errmsg;
 1460       Status = StatAuthError;
 1461       return _error->Error("%s", errmsg.c_str());
 1462    }
 1463 
 1464    // At this point, the gpgv method has succeeded, so there is a
 1465    // valid signature from a key in the trusted keyring.  We
 1466    // perform additional verification of its contents, and use them
 1467    // to verify the indexes we are about to download
 1468    if (_config->FindB("Debug::pkgAcquire::Auth", false))
 1469       std::cerr << "Signature verification succeeded: " << DestFile << std::endl;
 1470 
 1471    if (TransactionManager->IMSHit == false)
 1472    {
 1473       // open the last (In)Release if we have it
 1474       std::string const FinalFile = GetFinalFilename();
 1475       std::string FinalRelease;
 1476       std::string FinalInRelease;
 1477       if (APT::String::Endswith(FinalFile, "InRelease"))
 1478       {
 1479      FinalInRelease = FinalFile;
 1480      FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
 1481       }
 1482       else
 1483       {
 1484      FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
 1485      FinalRelease = FinalFile;
 1486       }
 1487       LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
 1488    }
 1489 
 1490    bool const GoodAuth = TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText);
 1491    if (GoodAuth == false && AllowInsecureRepositories(InsecureType::WEAK, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == false)
 1492    {
 1493       Status = StatAuthError;
 1494       return false;
 1495    }
 1496 
 1497    if (!VerifyVendor(Message))
 1498    {
 1499       Status = StatAuthError;
 1500       return false;
 1501    }
 1502 
 1503    // Download further indexes with verification
 1504    TransactionManager->QueueIndexes(GoodAuth);
 1505 
 1506    return GoodAuth;
 1507 }
 1508                                     /*}}}*/
 1509 void pkgAcqMetaClearSig::QueueIndexes(bool const verify)            /*{{{*/
 1510 {
 1511    // at this point the real Items are loaded in the fetcher
 1512    ExpectedAdditionalItems = 0;
 1513 
 1514    std::unordered_set<std::string> targetsSeen, componentsSeen;
 1515    bool const hasReleaseFile = TransactionManager->MetaIndexParser != NULL;
 1516    bool hasHashes = true;
 1517    auto IndexTargets = TransactionManager->MetaIndexParser->GetIndexTargets();
 1518    if (hasReleaseFile && verify == false)
 1519       hasHashes = std::any_of(IndexTargets.begin(), IndexTargets.end(),
 1520         [&](IndexTarget const &Target) { return TransactionManager->MetaIndexParser->Exists(Target.MetaKey); });
 1521    if (_config->FindB("Acquire::IndexTargets::Randomized", true) && likely(IndexTargets.empty() == false))
 1522    {
 1523       /* For fallback handling and to have some reasonable progress information
 1524      we can't randomize everything, but at least the order in the same type
 1525      can be as we shouldn't be telling the mirrors (and everyone else watching)
 1526      which is native/foreign arch, specific order of preference of translations, … */
 1527       auto range_start = IndexTargets.begin();
 1528       auto seed = (std::chrono::high_resolution_clock::now().time_since_epoch() /  std::chrono::nanoseconds(1)) ^ getpid();
 1529       std::default_random_engine g(seed);
 1530       do {
 1531      auto const type = range_start->Option(IndexTarget::CREATED_BY);
 1532      auto const range_end = std::find_if_not(range_start, IndexTargets.end(),
 1533            [&type](IndexTarget const &T) { return type == T.Option(IndexTarget::CREATED_BY); });
 1534      std::shuffle(range_start, range_end, g);
 1535      range_start = range_end;
 1536       } while (range_start != IndexTargets.end());
 1537    }
 1538    /* Collect all components for which files exist to prevent apt from warning users
 1539       about "hidden" components for which not all files exist like main/debian-installer
 1540       and Translation files */
 1541    if (hasReleaseFile == true)
 1542       for (auto const &Target : IndexTargets)
 1543      if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey))
 1544      {
 1545         auto component = Target.Option(IndexTarget::COMPONENT);
 1546         if (component.empty() == false)
 1547            componentsSeen.emplace(std::move(component));
 1548      }
 1549 
 1550    for (auto&& Target: IndexTargets)
 1551    {
 1552       // if we have seen a target which is created-by a target this one here is declared a
 1553       // fallback to, we skip acquiring the fallback (but we make sure we clean up)
 1554       if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
 1555       {
 1556      targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 1557      new CleanupItem(Owner, TransactionManager, Target);
 1558      continue;
 1559       }
 1560       // all is an implementation detail. Users shouldn't use this as arch
 1561       // We need this support trickery here as e.g. Debian has binary-all files already,
 1562       // but arch:all packages are still in the arch:any files, so we would waste precious
 1563       // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all
 1564       // in the set of supported architectures, so we can filter based on this property rather
 1565       // than invent an entirely new flag we would need to carry for all of eternity.
 1566       if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
 1567       {
 1568      if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)
 1569      {
 1570         new CleanupItem(Owner, TransactionManager, Target);
 1571         continue;
 1572      }
 1573       }
 1574 
 1575       bool trypdiff = Target.OptionBool(IndexTarget::PDIFFS);
 1576       if (hasReleaseFile == true)
 1577       {
 1578      if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false)
 1579      {
 1580         auto const component = Target.Option(IndexTarget::COMPONENT);
 1581         if (component.empty() == false &&
 1582         componentsSeen.find(component) == std::end(componentsSeen) &&
 1583         TransactionManager->MetaIndexParser->HasSupportForComponent(component) == false)
 1584         {
 1585            new CleanupItem(Owner, TransactionManager, Target);
 1586            _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' doesn't have the component '%s' (component misspelt in sources.list?)"),
 1587              Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), component.c_str());
 1588            continue;
 1589 
 1590         }
 1591 
 1592         // optional targets that we do not have in the Release file are skipped
 1593         if (hasHashes == true && Target.IsOptional)
 1594         {
 1595            new CleanupItem(Owner, TransactionManager, Target);
 1596            continue;
 1597         }
 1598 
 1599         std::string const &arch = Target.Option(IndexTarget::ARCHITECTURE);
 1600         if (arch.empty() == false)
 1601         {
 1602            if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false)
 1603            {
 1604           new CleanupItem(Owner, TransactionManager, Target);
 1605           _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"),
 1606             Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str());
 1607           continue;
 1608            }
 1609            // if the architecture is officially supported but currently no packages for it available,
 1610            // ignore silently as this is pretty much the same as just shipping an empty file.
 1611            // if we don't know which architectures are supported, we do NOT ignore it to notify user about this
 1612            if (hasHashes == true && TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false)
 1613            {
 1614           new CleanupItem(Owner, TransactionManager, Target);
 1615           continue;
 1616            }
 1617         }
 1618 
 1619         if (hasHashes == true)
 1620         {
 1621            new CleanupItem(Owner, TransactionManager, Target);
 1622            _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' does not seem to provide it (sources.list entry misspelt?)"),
 1623              Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
 1624            continue;
 1625         }
 1626         else
 1627         {
 1628            new pkgAcqIndex(Owner, TransactionManager, Target);
 1629            continue;
 1630         }
 1631      }
 1632      else if (verify)
 1633      {
 1634         auto const hashes = GetExpectedHashesFor(Target.MetaKey);
 1635         if (hashes.empty() == false)
 1636         {
 1637            if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false)
 1638            {
 1639           new CleanupItem(Owner, TransactionManager, Target);
 1640           _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
 1641             Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
 1642           continue;
 1643            }
 1644            // empty files are skipped as acquiring the very small compressed files is a waste of time
 1645            else if (hashes.FileSize() == 0)
 1646            {
 1647           new CleanupItem(Owner, TransactionManager, Target);
 1648           targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 1649           continue;
 1650            }
 1651         }
 1652      }
 1653 
 1654      // autoselect the compression method
 1655      std::vector<std::string> types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
 1656      types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) {
 1657         if (t == "uncompressed")
 1658            return TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false;
 1659         std::string const MetaKey = Target.MetaKey + "." + t;
 1660         return TransactionManager->MetaIndexParser->Exists(MetaKey) == false;
 1661      }), types.end());
 1662      if (types.empty() == false)
 1663      {
 1664         std::ostringstream os;
 1665         std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "));
 1666         os << *types.rbegin();
 1667         Target.Options["COMPRESSIONTYPES"] = os.str();
 1668      }
 1669      else
 1670         Target.Options["COMPRESSIONTYPES"].clear();
 1671 
 1672      std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
 1673      if (filename.empty() == false)
 1674      {
 1675         // if the Release file is a hit and we have an index it must be the current one
 1676         if (TransactionManager->IMSHit == true)
 1677            ;
 1678         else if (TransactionManager->LastMetaIndexParser != NULL)
 1679         {
 1680            // see if the file changed since the last Release file
 1681            // we use the uncompressed files as we might compress differently compared to the server,
 1682            // so the hashes might not match, even if they contain the same data.
 1683            HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target.MetaKey);
 1684            HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
 1685            if (newFile != oldFile)
 1686           filename.clear();
 1687         }
 1688         else
 1689            filename.clear();
 1690      }
 1691      else
 1692         trypdiff = false; // no file to patch
 1693 
 1694      if (filename.empty() == false)
 1695      {
 1696         new NoActionItem(Owner, Target, filename);
 1697         std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
 1698         if (FileExists(idxfilename))
 1699            new NoActionItem(Owner, Target, idxfilename);
 1700         targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 1701         continue;
 1702      }
 1703 
 1704      // check if we have patches available
 1705      trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey));
 1706       }
 1707       else
 1708       {
 1709      // if we have no file to patch, no point in trying
 1710      trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false);
 1711       }
 1712 
 1713       // no point in patching from local sources
 1714       if (trypdiff)
 1715       {
 1716      std::string const proto = Target.URI.substr(0, strlen("file:/"));
 1717      if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:")
 1718         trypdiff = false;
 1719       }
 1720 
 1721       // Queue the Index file (Packages, Sources, Translation-$foo, …)
 1722       targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 1723       if (trypdiff)
 1724          new pkgAcqDiffIndex(Owner, TransactionManager, Target);
 1725       else
 1726          new pkgAcqIndex(Owner, TransactionManager, Target);
 1727    }
 1728 }
 1729                                     /*}}}*/
 1730 bool pkgAcqMetaBase::VerifyVendor(string const &)           /*{{{*/
 1731 {
 1732    if (TransactionManager->MetaIndexParser->GetValidUntil() > 0)
 1733    {
 1734       time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil();
 1735       if (invalid_since > 0)
 1736       {
 1737      std::string errmsg;
 1738      strprintf(errmsg,
 1739            // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
 1740            // the time since then the file is invalid - formatted in the same way as in
 1741            // the download progress display (e.g. 7d 3h 42min 1s)
 1742            _("Release file for %s is expired (invalid since %s). "
 1743           "Updates for this repository will not be applied."),
 1744            Target.URI.c_str(), TimeToStr(invalid_since).c_str());
 1745      if (ErrorText.empty())
 1746         ErrorText = errmsg;
 1747      return _error->Error("%s", errmsg.c_str());
 1748       }
 1749    }
 1750 
 1751    if (TransactionManager->MetaIndexParser->GetNotBefore() > 0)
 1752    {
 1753       time_t const invalid_for = TransactionManager->MetaIndexParser->GetNotBefore() - time(nullptr);
 1754       if (invalid_for > 0)
 1755       {
 1756      std::string errmsg;
 1757      strprintf(errmsg,
 1758            // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
 1759            // the time until the file will be valid - formatted in the same way as in
 1760            // the download progress display (e.g. 7d 3h 42min 1s)
 1761            _("Release file for %s is not valid yet (invalid for another %s). "
 1762              "Updates for this repository will not be applied."),
 1763            Target.URI.c_str(), TimeToStr(invalid_for).c_str());
 1764      if (ErrorText.empty())
 1765         ErrorText = errmsg;
 1766      return _error->Error("%s", errmsg.c_str());
 1767       }
 1768    }
 1769 
 1770    /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
 1771       as a prevention of downgrading us to older (still valid) files */
 1772    if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL &&
 1773      TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate())
 1774    {
 1775       TransactionManager->IMSHit = true;
 1776       RemoveFile("VerifyVendor", DestFile);
 1777       PartialFile = DestFile = GetFinalFilename();
 1778       // load the 'old' file in the 'new' one instead of flipping pointers as
 1779       // the new one isn't owned by us, while the old one is so cleanup would be confused.
 1780       TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser);
 1781       delete TransactionManager->LastMetaIndexParser;
 1782       TransactionManager->LastMetaIndexParser = NULL;
 1783    }
 1784 
 1785    if (_config->FindB("Debug::pkgAcquire::Auth", false))
 1786    {
 1787       std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl;
 1788       std::cerr << "Got Suite: " << TransactionManager->MetaIndexParser->GetSuite() << std::endl;
 1789       std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl;
 1790    }
 1791 
 1792    // One day that might become fatal…
 1793    auto const ExpectedDist = TransactionManager->MetaIndexParser->GetExpectedDist();
 1794    auto const NowCodename = TransactionManager->MetaIndexParser->GetCodename();
 1795    if (TransactionManager->MetaIndexParser->CheckDist(ExpectedDist) == false)
 1796       _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
 1797         Desc.Description.c_str(), ExpectedDist.c_str(), NowCodename.c_str());
 1798 
 1799    // changed info potentially breaks user config like pinning
 1800    if (TransactionManager->LastMetaIndexParser != nullptr)
 1801    {
 1802       std::vector<pkgAcquireStatus::ReleaseInfoChange> Changes;
 1803       auto const AllowInfoChange = _config->FindB("Acquire::AllowReleaseInfoChange", false);
 1804       auto const quietInfoChange = _config->FindB("quiet::ReleaseInfoChange", false);
 1805       struct {
 1806      char const * const Type;
 1807      bool const Allowed;
 1808      decltype(&metaIndex::GetOrigin) const Getter;
 1809       } checkers[] = {
 1810      { "Origin", AllowInfoChange, &metaIndex::GetOrigin },
 1811      { "Label", AllowInfoChange, &metaIndex::GetLabel },
 1812      { "Version", true, &metaIndex::GetVersion }, // numbers change all the time, that is okay
 1813      { "Suite", true, &metaIndex::GetSuite },
 1814      { "Codename", AllowInfoChange, &metaIndex::GetCodename },
 1815      { nullptr, false, nullptr }
 1816       };
 1817       auto const CheckReleaseInfo = [&](char const * const Type, bool const AllowChange, decltype(checkers[0].Getter) const Getter) {
 1818      std::string const Last = (TransactionManager->LastMetaIndexParser->*Getter)();
 1819      std::string const Now = (TransactionManager->MetaIndexParser->*Getter)();
 1820      if (Last == Now)
 1821         return;
 1822      auto const Allow = _config->FindB(std::string("Acquire::AllowReleaseInfoChange::").append(Type), AllowChange);
 1823      if (Allow == true && _config->FindB(std::string("quiet::ReleaseInfoChange::").append(Type), quietInfoChange) == true)
 1824         return;
 1825      std::string msg;
 1826      strprintf(msg, _("Repository '%s' changed its '%s' value from '%s' to '%s'"),
 1827            Desc.Description.c_str(), Type, Last.c_str(), Now.c_str());
 1828      Changes.push_back({Type, std::move(Last), std::move(Now), std::move(msg), Allow});
 1829       };
 1830       for (short i = 0; checkers[i].Type != nullptr; ++i)
 1831      CheckReleaseInfo(checkers[i].Type, checkers[i].Allowed, checkers[i].Getter);
 1832 
 1833       {
 1834      auto const Last = TransactionManager->LastMetaIndexParser->GetDefaultPin();
 1835      auto const Now = TransactionManager->MetaIndexParser->GetDefaultPin();
 1836      if (Last != Now)
 1837      {
 1838         auto const Allow = _config->FindB("Acquire::AllowReleaseInfoChange::DefaultPin", AllowInfoChange);
 1839         if (Allow == false || _config->FindB("quiet::ReleaseInfoChange::DefaultPin", quietInfoChange) == false)
 1840         {
 1841            std::string msg;
 1842            strprintf(msg, _("Repository '%s' changed its default priority for %s from %hi to %hi."),
 1843              Desc.Description.c_str(), "apt_preferences(5)", Last, Now);
 1844            Changes.push_back({"DefaultPin", std::to_string(Last), std::to_string(Now), std::move(msg), Allow});
 1845         }
 1846      }
 1847       }
 1848       if (Changes.empty() == false)
 1849       {
 1850      auto const notes = TransactionManager->MetaIndexParser->GetReleaseNotes();
 1851      if (notes.empty() == false)
 1852      {
 1853         std::string msg;
 1854         // TRANSLATOR: the "this" refers to changes in the repository like a new release or owner change
 1855         strprintf(msg, _("More information about this can be found online in the Release notes at: %s"), notes.c_str());
 1856         Changes.push_back({"Release-Notes", "", std::move(notes), std::move(msg), true});
 1857      }
 1858      if (std::any_of(Changes.begin(),Changes.end(),[](pkgAcquireStatus::ReleaseInfoChange const &c) { return c.DefaultAction == false; }))
 1859      {
 1860         std::string msg;
 1861         // TRANSLATOR: %s is the name of the manpage in question, e.g. apt-secure(8)
 1862         strprintf(msg, _("This must be accepted explicitly before updates for "
 1863              "this repository can be applied. See %s manpage for details."), "apt-secure(8)");
 1864         Changes.push_back({"Confirmation", "", "", std::move(msg), true});
 1865      }
 1866 
 1867       }
 1868       if (Owner->Log == nullptr)
 1869      return pkgAcquireStatus::ReleaseInfoChangesAsGlobalErrors(std::move(Changes));
 1870       return Owner->Log->ReleaseInfoChanges(TransactionManager->LastMetaIndexParser, TransactionManager->MetaIndexParser, std::move(Changes));
 1871    }
 1872    return true;
 1873 }
 1874                                     /*}}}*/
 1875 pkgAcqMetaBase::~pkgAcqMetaBase()
 1876 {
 1877 }
 1878 
 1879 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,    /*{{{*/
 1880       IndexTarget const &ClearsignedTarget,
 1881       IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
 1882       metaIndex * const MetaIndexParser) :
 1883    pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),
 1884    d(NULL), DetachedDataTarget(DetachedDataTarget),
 1885    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
 1886 {
 1887    // index targets + (worst case:) Release/Release.gpg
 1888    ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max();
 1889    TransactionManager->Add(this);
 1890 }
 1891                                     /*}}}*/
 1892 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()               /*{{{*/
 1893 {
 1894    if (LastMetaIndexParser != NULL)
 1895       delete LastMetaIndexParser;
 1896 }
 1897                                     /*}}}*/
 1898 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
 1899 string pkgAcqMetaClearSig::Custom600Headers() const
 1900 {
 1901    string Header = pkgAcqMetaBase::Custom600Headers();
 1902    Header += "\nFail-Ignore: true";
 1903    std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
 1904    if (key.empty() == false)
 1905       Header += "\nSigned-By: " + key;
 1906 
 1907    return Header;
 1908 }
 1909                                     /*}}}*/
 1910 void pkgAcqMetaClearSig::Finished()                 /*{{{*/
 1911 {
 1912    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
 1913       std::clog << "Finished: " << DestFile <<std::endl;
 1914    if(TransactionManager->State == TransactionStarted &&
 1915       TransactionManager->TransactionHasError() == false)
 1916       TransactionManager->CommitTransaction();
 1917 }
 1918                                     /*}}}*/
 1919 bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,     /*{{{*/
 1920      pkgAcquire::MethodConfig const * const Cnf)
 1921 {
 1922    if (Item::VerifyDone(Message, Cnf) == false)
 1923       return false;
 1924 
 1925    if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
 1926       return RenameOnError(NotClearsigned);
 1927 
 1928    return true;
 1929 }
 1930                                     /*}}}*/
 1931 // pkgAcqMetaClearSig::Done - We got a file             /*{{{*/
 1932 void pkgAcqMetaClearSig::Done(std::string const &Message,
 1933                               HashStringList const &Hashes,
 1934                               pkgAcquire::MethodConfig const * const Cnf)
 1935 {
 1936    Item::Done(Message, Hashes, Cnf);
 1937 
 1938    if(AuthPass == false)
 1939    {
 1940       if(CheckDownloadDone(this, Message, Hashes) == true)
 1941          QueueForSignatureVerify(this, DestFile, DestFile);
 1942       return;
 1943    }
 1944    else if (CheckAuthDone(Message, Cnf) == true)
 1945    {
 1946       if (TransactionManager->IMSHit == false)
 1947      TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
 1948       else if (RealFileExists(GetFinalFilename()) == false)
 1949       {
 1950      // We got an InRelease file IMSHit, but we haven't one, which means
 1951      // we had a valid Release/Release.gpg combo stepping in, which we have
 1952      // to 'acquire' now to ensure list cleanup isn't removing them
 1953      new NoActionItem(Owner, DetachedDataTarget);
 1954      new NoActionItem(Owner, DetachedSigTarget);
 1955       }
 1956    }
 1957    else if (Status != StatAuthError)
 1958    {
 1959       string const FinalFile = GetFinalFileNameFromURI(DetachedDataTarget.URI);
 1960       string const OldFile = GetFinalFilename();
 1961       if (TransactionManager->IMSHit == false)
 1962      TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
 1963       else if (RealFileExists(OldFile) == false)
 1964      new NoActionItem(Owner, DetachedDataTarget);
 1965       else
 1966      TransactionManager->TransactionStageCopy(this, OldFile, FinalFile);
 1967    }
 1968 }
 1969                                     /*}}}*/
 1970 void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/
 1971 {
 1972    Item::Failed(Message, Cnf);
 1973 
 1974    if (AuthPass == false)
 1975    {
 1976       if (Status == StatTransientNetworkError)
 1977       {
 1978      TransactionManager->AbortTransaction();
 1979      return;
 1980       }
 1981       auto const failreason = LookupTag(Message, "FailReason");
 1982       auto const httperror = "HttpError";
 1983       if (Status == StatAuthError ||
 1984       Target.Option(IndexTarget::INRELEASE_PATH).empty() == false || /* do not fallback if InRelease was requested */
 1985       (strncmp(failreason.c_str(), httperror, strlen(httperror)) == 0 &&
 1986        failreason != "HttpError404"))
 1987       {
 1988      // if we expected a ClearTextSignature (InRelease) but got a network
 1989      // error or got a file, but it wasn't valid, we end up here (see VerifyDone).
 1990      // As these is usually called by web-portals we do not try Release/Release.gpg
 1991      // as this is going to fail anyway and instead abort our try (LP#346386)
 1992      _error->PushToStack();
 1993      _error->Error(_("Failed to fetch %s  %s"), Target.URI.c_str(), ErrorText.c_str());
 1994      if (Target.Option(IndexTarget::INRELEASE_PATH).empty() == true && AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
 1995         _error->RevertToStack();
 1996      else
 1997         return;
 1998       }
 1999 
 2000       // Queue the 'old' InRelease file for removal if we try Release.gpg
 2001       // as otherwise the file will stay around and gives a false-auth
 2002       // impression (CVE-2012-0214)
 2003       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
 2004       Status = StatDone;
 2005 
 2006       new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget);
 2007    }
 2008    else
 2009    {
 2010       if(CheckStopAuthentication(this, Message))
 2011          return;
 2012 
 2013       if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
 2014       {
 2015      Status = StatDone;
 2016 
 2017      /* InRelease files become Release files, otherwise
 2018       * they would be considered as trusted later on */
 2019      string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI);
 2020      string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI);
 2021      string const FinalInRelease = GetFinalFilename();
 2022      Rename(DestFile, PartialRelease);
 2023      TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
 2024      LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
 2025 
 2026      // we parse the indexes here because at this point the user wanted
 2027      // a repository that may potentially harm him
 2028      if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
 2029         /* expired Release files are still a problem you need extra force for */;
 2030      else
 2031         TransactionManager->QueueIndexes(true);
 2032       }
 2033    }
 2034 }
 2035                                     /*}}}*/
 2036 
 2037 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner,      /*{{{*/
 2038                                  pkgAcqMetaClearSig * const TransactionManager,
 2039                  IndexTarget const &DataTarget,
 2040                  IndexTarget const &DetachedSigTarget) :
 2041    pkgAcqMetaBase(Owner, TransactionManager, DataTarget), d(NULL),
 2042    DetachedSigTarget(DetachedSigTarget)
 2043 {
 2044    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
 2045       std::clog << "New pkgAcqMetaIndex with TransactionManager "
 2046                 << this->TransactionManager << std::endl;
 2047 
 2048    DestFile = GetPartialFileNameFromURI(DataTarget.URI);
 2049 
 2050    // Create the item
 2051    Desc.Description = DataTarget.Description;
 2052    Desc.Owner = this;
 2053    Desc.ShortDesc = DataTarget.ShortDesc;
 2054 
 2055    // Rewrite the description URI if INRELEASE_PATH was specified so
 2056    // we download the specified file instead.
 2057    auto InReleasePath = DataTarget.Option(IndexTarget::INRELEASE_PATH);
 2058    if (InReleasePath.empty() == false && APT::String::Endswith(DataTarget.URI, "/InRelease"))
 2059    {
 2060       Desc.URI = DataTarget.URI.substr(0, DataTarget.URI.size() - strlen("InRelease")) + InReleasePath;
 2061    }
 2062    else
 2063    {
 2064       Desc.URI = DataTarget.URI;
 2065    }
 2066 
 2067    QueueURI(Desc);
 2068 }
 2069                                     /*}}}*/
 2070 void pkgAcqMetaIndex::Done(string const &Message,           /*{{{*/
 2071                            HashStringList const &Hashes,
 2072                pkgAcquire::MethodConfig const * const Cfg)
 2073 {
 2074    Item::Done(Message,Hashes,Cfg);
 2075 
 2076    if(CheckDownloadDone(this, Message, Hashes))
 2077    {
 2078       // we have a Release file, now download the Signature, all further
 2079       // verify/queue for additional downloads will be done in the
 2080       // pkgAcqMetaSig::Done() code
 2081       new pkgAcqMetaSig(Owner, TransactionManager, DetachedSigTarget, this);
 2082    }
 2083 }
 2084                                     /*}}}*/
 2085 // pkgAcqMetaIndex::Failed - no Release file present            /*{{{*/
 2086 void pkgAcqMetaIndex::Failed(string const &Message,
 2087                              pkgAcquire::MethodConfig const * const Cnf)
 2088 {
 2089    pkgAcquire::Item::Failed(Message, Cnf);
 2090    Status = StatDone;
 2091 
 2092    // No Release file was present so fall
 2093    // back to queueing Packages files without verification
 2094    // only allow going further if the user explicitly wants it
 2095    if(AllowInsecureRepositories(InsecureType::NORELEASE, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
 2096    {
 2097       // ensure old Release files are removed
 2098       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
 2099 
 2100       // queue without any kind of hashsum support
 2101       TransactionManager->QueueIndexes(false);
 2102    }
 2103 }
 2104                                     /*}}}*/
 2105 std::string pkgAcqMetaIndex::DescURI() const                /*{{{*/
 2106 {
 2107    return Target.URI;
 2108 }
 2109                                     /*}}}*/
 2110 pkgAcqMetaIndex::~pkgAcqMetaIndex() {}
 2111 
 2112 // AcqMetaSig::AcqMetaSig - Constructor                 /*{{{*/
 2113 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner,
 2114       pkgAcqMetaClearSig * const TransactionManager,
 2115       IndexTarget const &Target,
 2116       pkgAcqMetaIndex * const MetaIndex) :
 2117    pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex)
 2118 {
 2119    DestFile = GetPartialFileNameFromURI(Target.URI);
 2120 
 2121    // remove any partial downloaded sig-file in partial/.
 2122    // it may confuse proxies and is too small to warrant a
 2123    // partial download anyway
 2124    RemoveFile("pkgAcqMetaSig", DestFile);
 2125 
 2126    // set the TransactionManager
 2127    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
 2128       std::clog << "New pkgAcqMetaSig with TransactionManager "
 2129                 << TransactionManager << std::endl;
 2130 
 2131    // Create the item
 2132    Desc.Description = Target.Description;
 2133    Desc.Owner = this;
 2134    Desc.ShortDesc = Target.ShortDesc;
 2135    Desc.URI = Target.URI;
 2136 
 2137    // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
 2138    // so we skip the download step and go instantly to verification
 2139    if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
 2140    {
 2141       Complete = true;
 2142       Status = StatDone;
 2143       PartialFile = DestFile = GetFinalFilename();
 2144       MetaIndexFileSignature = DestFile;
 2145       MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
 2146    }
 2147    else
 2148       QueueURI(Desc);
 2149 }
 2150                                     /*}}}*/
 2151 pkgAcqMetaSig::~pkgAcqMetaSig()                     /*{{{*/
 2152 {
 2153 }
 2154                                     /*}}}*/
 2155 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers  /*{{{*/
 2156 std::string pkgAcqMetaSig::Custom600Headers() const
 2157 {
 2158    std::string Header = pkgAcqTransactionItem::Custom600Headers();
 2159    std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
 2160    if (key.empty() == false)
 2161       Header += "\nSigned-By: " + key;
 2162    return Header;
 2163 }
 2164                                     /*}}}*/
 2165 // AcqMetaSig::Done - The signature was downloaded/verified     /*{{{*/
 2166 void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes,
 2167              pkgAcquire::MethodConfig const * const Cfg)
 2168 {
 2169    if (MetaIndexFileSignature.empty() == false)
 2170    {
 2171       DestFile = MetaIndexFileSignature;
 2172       MetaIndexFileSignature.clear();
 2173    }
 2174    Item::Done(Message, Hashes, Cfg);
 2175 
 2176    if(MetaIndex->AuthPass == false)
 2177    {
 2178       if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
 2179       {
 2180      // destfile will be modified to point to MetaIndexFile for the
 2181      // gpgv method, so we need to save it here
 2182      MetaIndexFileSignature = DestFile;
 2183      MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
 2184       }
 2185       return;
 2186    }
 2187    else if (MetaIndex->CheckAuthDone(Message, Cfg) == true)
 2188    {
 2189       auto const Releasegpg = GetFinalFilename();
 2190       auto const Release = MetaIndex->GetFinalFilename();
 2191       // if this is an IMS-Hit on Release ensure we also have the Release.gpg file stored
 2192       // (previously an unknown pubkey) – but only if the Release file exists locally (unlikely
 2193       // event of InRelease removed from the mirror causing fallback but still an IMS-Hit)
 2194       if (TransactionManager->IMSHit == false ||
 2195         (FileExists(Releasegpg) == false && FileExists(Release) == true))
 2196       {
 2197      TransactionManager->TransactionStageCopy(this, DestFile, Releasegpg);
 2198      TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, Release);
 2199       }
 2200    }
 2201    else if (MetaIndex->Status != StatAuthError)
 2202    {
 2203       std::string const FinalFile = MetaIndex->GetFinalFilename();
 2204       if (TransactionManager->IMSHit == false)
 2205      TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalFile);
 2206       else
 2207      TransactionManager->TransactionStageCopy(MetaIndex, FinalFile, FinalFile);
 2208    }
 2209 }
 2210                                     /*}}}*/
 2211 void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 2212 {
 2213    Item::Failed(Message,Cnf);
 2214 
 2215    // check if we need to fail at this point
 2216    if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
 2217          return;
 2218 
 2219    // ensures that a Release.gpg file in the lists/ is removed by the transaction
 2220    if (not MetaIndexFileSignature.empty())
 2221    {
 2222       DestFile = MetaIndexFileSignature;
 2223       MetaIndexFileSignature.clear();
 2224    }
 2225    TransactionManager->TransactionStageRemoval(this, DestFile);
 2226 
 2227    // only allow going further if the user explicitly wants it
 2228    if (AllowInsecureRepositories(InsecureType::UNSIGNED, MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
 2229    {
 2230       string const FinalRelease = MetaIndex->GetFinalFilename();
 2231       string const FinalInRelease = TransactionManager->GetFinalFilename();
 2232       LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
 2233 
 2234       // we parse the indexes here because at this point the user wanted
 2235       // a repository that may potentially harm him
 2236       bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText);
 2237       if (MetaIndex->VerifyVendor(Message) == false)
 2238      /* expired Release files are still a problem you need extra force for */;
 2239       else
 2240      TransactionManager->QueueIndexes(GoodLoad);
 2241 
 2242       TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalRelease);
 2243    }
 2244    else if (TransactionManager->IMSHit == false)
 2245       Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
 2246 
 2247    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
 2248    if (Cnf->LocalOnly == true ||
 2249        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 2250    {
 2251       // Ignore this
 2252       Status = StatDone;
 2253    }
 2254 }
 2255                                     /*}}}*/
 2256 
 2257 
 2258 // AcqBaseIndex - Constructor                       /*{{{*/
 2259 pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner,
 2260       pkgAcqMetaClearSig * const TransactionManager,
 2261       IndexTarget const &Target)
 2262 : pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL)
 2263 {
 2264 }
 2265                                     /*}}}*/
 2266 void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 2267 {
 2268    pkgAcquire::Item::Failed(Message, Cnf);
 2269    if (Status != StatAuthError)
 2270       return;
 2271 
 2272    ErrorText.append("Release file created at: ");
 2273    auto const timespec = TransactionManager->MetaIndexParser->GetDate();
 2274    if (timespec == 0)
 2275       ErrorText.append("<unknown>");
 2276    else
 2277       ErrorText.append(TimeRFC1123(timespec, true));
 2278    ErrorText.append("\n");
 2279 }
 2280                                     /*}}}*/
 2281 pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
 2282 
 2283 // AcqDiffIndex::AcqDiffIndex - Constructor             /*{{{*/
 2284 // ---------------------------------------------------------------------
 2285 /* Get the DiffIndex file first and see if there are patches available
 2286  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 2287  * patches. If anything goes wrong in that process, it will fall back to
 2288  * the original packages file
 2289  */
 2290 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
 2291                                  pkgAcqMetaClearSig * const TransactionManager,
 2292                                  IndexTarget const &Target)
 2293    : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL)
 2294 {
 2295    // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
 2296    ExpectedAdditionalItems = 40;
 2297    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 2298 
 2299    CompressionExtensions.clear();
 2300    {
 2301       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 2302       if (types.empty() == false)
 2303       {
 2304      std::ostringstream os;
 2305      std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) {
 2306            if (type == "uncompressed")
 2307               return true;
 2308            return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type);
 2309      });
 2310      os << *types.rbegin();
 2311      CompressionExtensions = os.str();
 2312       }
 2313    }
 2314    Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc);
 2315 
 2316    if(Debug)
 2317       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 2318 }
 2319                                     /*}}}*/
 2320 void pkgAcqDiffIndex::QueueOnIMSHit() const             /*{{{*/
 2321 {
 2322    // list cleanup needs to know that this file as well as the already
 2323    // present index is ours, so we create an empty diff to save it for us
 2324    new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
 2325 }
 2326                                     /*}}}*/
 2327 static bool RemoveFileForBootstrapLinking(std::string &ErrorText, std::string const &For, std::string const &Boot)/*{{{*/
 2328 {
 2329    if (FileExists(Boot) && RemoveFile("Bootstrap-linking", Boot) == false)
 2330    {
 2331       strprintf(ErrorText, "Bootstrap for patching %s by removing stale %s failed!", For.c_str(), Boot.c_str());
 2332       return false;
 2333    }
 2334    return true;
 2335 }
 2336                                     /*}}}*/
 2337 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)   /*{{{*/
 2338 {
 2339    available_patches.clear();
 2340    ExpectedAdditionalItems = 0;
 2341    // failing here is fine: our caller will take care of trying to
 2342    // get the complete file if patching fails
 2343    if(Debug)
 2344       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 2345      << std::endl;
 2346 
 2347    FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension);
 2348    pkgTagFile TF(&Fd);
 2349    if (Fd.IsOpen() == false || Fd.Failed())
 2350       return false;
 2351 
 2352    pkgTagSection Tags;
 2353    if(unlikely(TF.Step(Tags) == false))
 2354       return false;
 2355 
 2356    HashStringList ServerHashes;
 2357    unsigned long long ServerSize = 0;
 2358 
 2359    auto const &posix = std::locale::classic();
 2360    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 2361    {
 2362       std::string tagname = *type;
 2363       tagname.append("-Current");
 2364       std::string const tmp = Tags.FindS(tagname.c_str());
 2365       if (tmp.empty() == true)
 2366      continue;
 2367 
 2368       string hash;
 2369       unsigned long long size;
 2370       std::stringstream ss(tmp);
 2371       ss.imbue(posix);
 2372       ss >> hash >> size;
 2373       if (unlikely(hash.empty() == true))
 2374      continue;
 2375       if (unlikely(ServerSize != 0 && ServerSize != size))
 2376      continue;
 2377       ServerHashes.push_back(HashString(*type, hash));
 2378       ServerSize = size;
 2379    }
 2380 
 2381    if (ServerHashes.usable() == false)
 2382    {
 2383       ErrorText = "Did not find a good hashsum in the index";
 2384       return false;
 2385    }
 2386 
 2387    std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI);
 2388    HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
 2389    if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes)
 2390    {
 2391       ErrorText = "Index has different hashes than parser (probably older)";
 2392       return false;
 2393    }
 2394 
 2395    HashStringList LocalHashes;
 2396    // try avoiding calculating the hash here as this is costly
 2397    if (TransactionManager->LastMetaIndexParser != NULL)
 2398       LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
 2399    if (LocalHashes.usable() == false)
 2400    {
 2401       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto);
 2402       Hashes LocalHashesCalc(ServerHashes);
 2403       LocalHashesCalc.AddFD(fd);
 2404       LocalHashes = LocalHashesCalc.GetHashStringList();
 2405    }
 2406 
 2407    if (ServerHashes == LocalHashes)
 2408    {
 2409       available_patches.clear();
 2410       return true;
 2411    }
 2412 
 2413    if(Debug)
 2414       std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
 2415      << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
 2416 
 2417    // historically, older hashes have more info than newer ones, so start
 2418    // collecting with older ones first to avoid implementing complicated
 2419    // information merging techniques… a failure is after all always
 2420    // recoverable with a complete file and hashes aren't changed that often.
 2421    std::vector<char const *> types;
 2422    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 2423       types.push_back(*type);
 2424 
 2425    // parse all of (provided) history
 2426    bool firstAcceptedHashes = true;
 2427    for (auto type = types.crbegin(); type != types.crend(); ++type)
 2428    {
 2429       if (LocalHashes.find(*type) == NULL)
 2430      continue;
 2431 
 2432       std::string tagname = *type;
 2433       tagname.append("-History");
 2434       std::string const tmp = Tags.FindS(tagname.c_str());
 2435       if (tmp.empty() == true)
 2436      continue;
 2437 
 2438       string hash, filename;
 2439       unsigned long long size;
 2440       std::stringstream ss(tmp);
 2441       ss.imbue(posix);
 2442 
 2443       while (ss >> hash >> size >> filename)
 2444       {
 2445      if (unlikely(hash.empty() == true || filename.empty() == true))
 2446         continue;
 2447 
 2448      // see if we have a record for this file already
 2449      std::vector<DiffInfo>::iterator cur = available_patches.begin();
 2450      for (; cur != available_patches.end(); ++cur)
 2451      {
 2452         if (cur->file != filename)
 2453            continue;
 2454         cur->result_hashes.push_back(HashString(*type, hash));
 2455         break;
 2456      }
 2457      if (cur != available_patches.end())
 2458         continue;
 2459      if (firstAcceptedHashes == true)
 2460      {
 2461         DiffInfo next;
 2462         next.file = filename;
 2463         next.result_hashes.push_back(HashString(*type, hash));
 2464         next.result_hashes.FileSize(size);
 2465         available_patches.push_back(next);
 2466      }
 2467      else
 2468      {
 2469         if (Debug == true)
 2470            std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
 2471           << " wasn't in the list for the first parsed hash! (history)" << std::endl;
 2472         break;
 2473      }
 2474       }
 2475       firstAcceptedHashes = false;
 2476    }
 2477 
 2478    if (unlikely(available_patches.empty() == true))
 2479    {
 2480       ErrorText = "Couldn't find any patches for the patch series";
 2481       return false;
 2482    }
 2483 
 2484    for (auto type = types.crbegin(); type != types.crend(); ++type)
 2485    {
 2486       if (LocalHashes.find(*type) == NULL)
 2487      continue;
 2488 
 2489       std::string tagname = *type;
 2490       tagname.append("-Patches");
 2491       std::string const tmp = Tags.FindS(tagname.c_str());
 2492       if (tmp.empty() == true)
 2493      continue;
 2494 
 2495       string hash, filename;
 2496       unsigned long long size;
 2497       std::stringstream ss(tmp);
 2498       ss.imbue(posix);
 2499 
 2500       while (ss >> hash >> size >> filename)
 2501       {
 2502      if (unlikely(hash.empty() == true || filename.empty() == true))
 2503         continue;
 2504 
 2505      // see if we have a record for this file already
 2506      std::vector<DiffInfo>::iterator cur = available_patches.begin();
 2507      for (; cur != available_patches.end(); ++cur)
 2508      {
 2509         if (cur->file != filename)
 2510            continue;
 2511         if (cur->patch_hashes.empty())
 2512            cur->patch_hashes.FileSize(size);
 2513         cur->patch_hashes.push_back(HashString(*type, hash));
 2514         break;
 2515      }
 2516      if (cur != available_patches.end())
 2517            continue;
 2518      if (Debug == true)
 2519         std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
 2520            << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
 2521      break;
 2522       }
 2523    }
 2524 
 2525    for (auto type = types.crbegin(); type != types.crend(); ++type)
 2526    {
 2527       std::string tagname = *type;
 2528       tagname.append("-Download");
 2529       std::string const tmp = Tags.FindS(tagname.c_str());
 2530       if (tmp.empty() == true)
 2531      continue;
 2532 
 2533       string hash, filename;
 2534       unsigned long long size;
 2535       std::stringstream ss(tmp);
 2536       ss.imbue(posix);
 2537 
 2538       // FIXME: all of pdiff supports only .gz compressed patches
 2539       while (ss >> hash >> size >> filename)
 2540       {
 2541      if (unlikely(hash.empty() == true || filename.empty() == true))
 2542         continue;
 2543      if (unlikely(APT::String::Endswith(filename, ".gz") == false))
 2544         continue;
 2545      filename.erase(filename.length() - 3);
 2546 
 2547      // see if we have a record for this file already
 2548      std::vector<DiffInfo>::iterator cur = available_patches.begin();
 2549      for (; cur != available_patches.end(); ++cur)
 2550      {
 2551         if (cur->file != filename)
 2552            continue;
 2553         if (cur->download_hashes.empty())
 2554            cur->download_hashes.FileSize(size);
 2555         cur->download_hashes.push_back(HashString(*type, hash));
 2556         break;
 2557      }
 2558      if (cur != available_patches.end())
 2559            continue;
 2560      if (Debug == true)
 2561         std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
 2562            << " wasn't in the list for the first parsed hash! (download)" << std::endl;
 2563      break;
 2564       }
 2565    }
 2566 
 2567    {
 2568       auto const foundStart = std::find_if(available_patches.rbegin(), available_patches.rend(),
 2569         [&](auto const &cur) { return LocalHashes == cur.result_hashes; });
 2570       if (foundStart == available_patches.rend() || unlikely(available_patches.empty()))
 2571       {
 2572      ErrorText = "Couldn't find the start of the patch series";
 2573      return false;
 2574       }
 2575       available_patches.erase(available_patches.begin(), std::prev(foundStart.base()));
 2576 
 2577       auto const patch = std::find_if(available_patches.cbegin(), available_patches.cend(), [](auto const &patch) {
 2578      return not patch.result_hashes.usable() ||
 2579         not patch.patch_hashes.usable() ||
 2580         not patch.download_hashes.usable();
 2581       });
 2582       if (patch != available_patches.cend())
 2583       {
 2584      strprintf(ErrorText, "Provides no usable hashes for %s", patch->file.c_str());
 2585      return false;
 2586       }
 2587    }
 2588 
 2589    // patching with too many files is rather slow compared to a fast download
 2590    unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 2591    if (fileLimit != 0 && fileLimit < available_patches.size())
 2592    {
 2593       strprintf(ErrorText, "Need %lu diffs, but limit is %lu", available_patches.size(), fileLimit);
 2594       return false;
 2595    }
 2596 
 2597    /* decide if we should download patches one by one or in one go:
 2598       The first is good if the server merges patches, but many don't so client
 2599       based merging can be attempt in which case the second is better.
 2600       "bad things" will happen if patches are merged on the server,
 2601       but client side merging is attempt as well */
 2602    pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 2603    if (pdiff_merge == true)
 2604    {
 2605       // reprepro and dak add this flag if they merge patches on the server
 2606       std::string const precedence = Tags.FindS("X-Patch-Precedence");
 2607       pdiff_merge = (precedence != "merged");
 2608    }
 2609 
 2610    // calculate the size of all patches we have to get
 2611    unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 2612    if (sizeLimitPercent > 0)
 2613    {
 2614       unsigned long long downloadSize = 0;
 2615       if (pdiff_merge)
 2616      downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu,
 2617                     [](unsigned long long const T, DiffInfo const &I) {
 2618                        return T + I.download_hashes.FileSize();
 2619                     });
 2620       // if server-side merging, assume we will need only the first patch
 2621       else if (not available_patches.empty())
 2622      downloadSize = available_patches.front().download_hashes.FileSize();
 2623       if (downloadSize != 0)
 2624       {
 2625      unsigned long long downloadSizeIdx = 0;
 2626      auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
 2627      for (auto const &t : types)
 2628      {
 2629         std::string MetaKey = Target.MetaKey;
 2630         if (t != "uncompressed")
 2631            MetaKey += '.' + t;
 2632         HashStringList const hsl = GetExpectedHashesFor(MetaKey);
 2633         if (unlikely(hsl.usable() == false))
 2634            continue;
 2635         downloadSizeIdx = hsl.FileSize();
 2636         break;
 2637      }
 2638      unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
 2639      if ((sizeLimit/100) < downloadSize)
 2640      {
 2641         strprintf(ErrorText, "Need %llu compressed bytes, but limit is %llu and original is %llu", downloadSize, (sizeLimit/100), downloadSizeIdx);
 2642         return false;
 2643      }
 2644       }
 2645    }
 2646 
 2647    // clean the plate
 2648    {
 2649       std::string const Final = GetExistingFilename(CurrentPackagesFile);
 2650       if (unlikely(Final.empty())) // because we wouldn't be called in such a case
 2651      return false;
 2652       std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);
 2653       std::string const PatchedFile = GetKeepCompressedFileName(PartialFile + "-patched", Target);
 2654       if (not RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile) ||
 2655       not RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile))
 2656      return false;
 2657       {
 2658      auto const exts = APT::Configuration::getCompressorExtensions();
 2659      if (not std::all_of(exts.cbegin(), exts.cend(), [&](auto const &ext) {
 2660         return RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile + ext) &&
 2661                RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile + ext);
 2662          }))
 2663         return false;
 2664       }
 2665       std::string const Ext = Final.substr(CurrentPackagesFile.length());
 2666       std::string const Partial = PartialFile + Ext;
 2667       if (symlink(Final.c_str(), Partial.c_str()) != 0)
 2668       {
 2669      strprintf(ErrorText, "Bootstrap for patching by linking %s to %s failed!", Final.c_str(), Partial.c_str());
 2670      return false;
 2671       }
 2672    }
 2673 
 2674    return true;
 2675 }
 2676                                     /*}}}*/
 2677 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 2678 {
 2679    if (CommonFailed(GetDiffIndexURI(Target), Message, Cnf))
 2680       return;
 2681 
 2682    RenameOnError(PDiffError);
 2683    Status = StatDone;
 2684    ExpectedAdditionalItems = 0;
 2685 
 2686    if(Debug)
 2687       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 2688         << "Falling back to normal index file acquire" << std::endl;
 2689 
 2690    new pkgAcqIndex(Owner, TransactionManager, Target);
 2691 }
 2692                                     /*}}}*/
 2693 bool pkgAcqDiffIndex::VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const)/*{{{*/
 2694 {
 2695    string const FinalFile = GetFinalFilename();
 2696    if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
 2697       DestFile = FinalFile;
 2698 
 2699    if (ParseDiffIndex(DestFile))
 2700       return true;
 2701 
 2702    Status = StatError;
 2703    if (ErrorText.empty())
 2704       ErrorText = "Couldn't parse pdiff index";
 2705    return false;
 2706 }
 2707                                     /*}}}*/
 2708 void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes,  /*{{{*/
 2709                pkgAcquire::MethodConfig const * const Cnf)
 2710 {
 2711    if(Debug)
 2712       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 2713 
 2714    Item::Done(Message, Hashes, Cnf);
 2715 
 2716    if (available_patches.empty())
 2717    {
 2718       // we have the same sha1 as the server so we are done here
 2719       if(Debug)
 2720      std::clog << "pkgAcqDiffIndex: Package file is up-to-date" << std::endl;
 2721       QueueOnIMSHit();
 2722    }
 2723    else
 2724    {
 2725       if (pdiff_merge == false)
 2726      new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
 2727       else
 2728       {
 2729      diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 2730      for(size_t i = 0; i < available_patches.size(); ++i)
 2731         (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
 2732                             Target,
 2733                             available_patches[i],
 2734                             diffs);
 2735       }
 2736    }
 2737 
 2738    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
 2739 
 2740    Complete = true;
 2741    Status = StatDone;
 2742    Dequeue();
 2743 
 2744    return;
 2745 }
 2746                                     /*}}}*/
 2747 pkgAcqDiffIndex::~pkgAcqDiffIndex()
 2748 {
 2749    if (diffs != NULL)
 2750       delete diffs;
 2751 }
 2752 
 2753 // AcqIndexDiffs::AcqIndexDiffs - Constructor               /*{{{*/
 2754 // ---------------------------------------------------------------------
 2755 /* The package diff is added to the queue. one object is constructed
 2756  * for each diff and the index
 2757  */
 2758 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *const Owner,
 2759                    pkgAcqMetaClearSig *const TransactionManager,
 2760                    IndexTarget const &Target,
 2761                    vector<DiffInfo> const &diffs)
 2762     : pkgAcqBaseIndex(Owner, TransactionManager, Target),
 2763       available_patches(diffs)
 2764 {
 2765    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
 2766 
 2767    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 2768 
 2769    Desc.Owner = this;
 2770    Desc.ShortDesc = Target.ShortDesc;
 2771 
 2772    if(available_patches.empty() == true)
 2773    {
 2774       // we are done (yeah!), check hashes against the final file
 2775       DestFile = GetKeepCompressedFileName(GetFinalFileNameFromURI(Target.URI), Target);
 2776       Finish(true);
 2777    }
 2778    else
 2779    {
 2780       State = StateFetchDiff;
 2781       QueueNextDiff();
 2782    }
 2783 }
 2784                                     /*}}}*/
 2785 void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 2786 {
 2787    pkgAcqBaseIndex::Failed(Message,Cnf);
 2788    Status = StatDone;
 2789 
 2790    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
 2791    if(Debug)
 2792       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 2793         << "Falling back to normal index file acquire " << std::endl;
 2794    RenameOnError(PDiffError);
 2795    std::string const patchname = GetDiffsPatchFileName(DestFile);
 2796    if (RealFileExists(patchname))
 2797       Rename(patchname, patchname + ".FAILED");
 2798    std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
 2799    if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
 2800       Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
 2801    new pkgAcqIndex(Owner, TransactionManager, Target);
 2802    Finish();
 2803 }
 2804                                     /*}}}*/
 2805 // Finish - helper that cleans the item out of the fetcher queue    /*{{{*/
 2806 void pkgAcqIndexDiffs::Finish(bool allDone)
 2807 {
 2808    if(Debug)
 2809       std::clog << "pkgAcqIndexDiffs::Finish(): " 
 2810                 << allDone << " "
 2811                 << Desc.URI << std::endl;
 2812 
 2813    // we restore the original name, this is required, otherwise
 2814    // the file will be cleaned
 2815    if(allDone)
 2816    {
 2817       std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target);
 2818       TransactionManager->TransactionStageCopy(this, DestFile, Final);
 2819 
 2820       // this is for the "real" finish
 2821       Complete = true;
 2822       Status = StatDone;
 2823       Dequeue();
 2824       if(Debug)
 2825      std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 2826       return;
 2827    }
 2828    else
 2829       DestFile.clear();
 2830 
 2831    if(Debug)
 2832       std::clog << "Finishing: " << Desc.URI << std::endl;
 2833    Complete = false;
 2834    Status = StatDone;
 2835    Dequeue();
 2836    return;
 2837 }
 2838                                     /*}}}*/
 2839 bool pkgAcqIndexDiffs::QueueNextDiff()                  /*{{{*/
 2840 {
 2841    // calc sha1 of the just patched file
 2842    std::string const PartialFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
 2843    if(unlikely(PartialFile.empty()))
 2844    {
 2845       Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL);
 2846       return false;
 2847    }
 2848 
 2849    FileFd fd(PartialFile, FileFd::ReadOnly, FileFd::Extension);
 2850    Hashes LocalHashesCalc;
 2851    LocalHashesCalc.AddFD(fd);
 2852    HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 2853 
 2854    if(Debug)
 2855       std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
 2856 
 2857    HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
 2858    if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false))
 2859    {
 2860       Failed("Local/Expected hashes are not usable for " + PartialFile, NULL);
 2861       return false;
 2862    }
 2863 
 2864    // final file reached before all patches are applied
 2865    if(LocalHashes == TargetFileHashes)
 2866    {
 2867       Finish(true);
 2868       return true;
 2869    }
 2870 
 2871    // remove all patches until the next matching patch is found
 2872    // this requires the Index file to be ordered
 2873    available_patches.erase(available_patches.begin(),
 2874      std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) {
 2875         return I.result_hashes == LocalHashes;
 2876         }));
 2877 
 2878    // error checking and falling back if no patch was found
 2879    if(available_patches.empty() == true)
 2880    {
 2881       Failed("No patches left to reach target for " + PartialFile, NULL);
 2882       return false;
 2883    }
 2884 
 2885    // queue the right diff
 2886    auto const BaseFileURI = Target.URI + ".diff/" + pkgAcquire::URIEncode(available_patches[0].file);
 2887    Desc.URI = BaseFileURI + ".gz";
 2888    Desc.Description = Target.Description + " " + available_patches[0].file + string(".pdiff");
 2889    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(BaseFileURI), Target);
 2890 
 2891    if(Debug)
 2892       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 2893 
 2894    QueueURI(Desc);
 2895 
 2896    return true;
 2897 }
 2898                                     /*}}}*/
 2899 void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes,    /*{{{*/
 2900                 pkgAcquire::MethodConfig const * const Cnf)
 2901 {
 2902    if (Debug)
 2903       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 2904 
 2905    Item::Done(Message, Hashes, Cnf);
 2906 
 2907    std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
 2908    std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
 2909    std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile);
 2910    std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
 2911 
 2912    switch (State)
 2913    {
 2914       // success in downloading a diff, enter ApplyDiff state
 2915       case StateFetchDiff:
 2916      Rename(DestFile, PatchFile);
 2917      DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
 2918      if(Debug)
 2919         std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
 2920      State = StateApplyDiff;
 2921      Local = true;
 2922      Desc.URI = "rred:" + pkgAcquire::URIEncode(UnpatchedFile);
 2923      QueueURI(Desc);
 2924      SetActiveSubprocess("rred");
 2925      return;
 2926       // success in download/apply a diff, queue next (if needed)
 2927       case StateApplyDiff:
 2928      // remove the just applied patch and base file
 2929      available_patches.erase(available_patches.begin());
 2930      RemoveFile("pkgAcqIndexDiffs::Done", PatchFile);
 2931      RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile);
 2932      if(Debug)
 2933         std::clog << "Moving patched file in place: " << std::endl
 2934            << DestFile << " -> " << PatchedFile << std::endl;
 2935      Rename(DestFile, PatchedFile);
 2936 
 2937      // see if there is more to download
 2938      if(available_patches.empty() == false)
 2939      {
 2940         new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
 2941         Finish();
 2942      } else {
 2943         DestFile = PatchedFile;
 2944         Finish(true);
 2945      }
 2946      return;
 2947    }
 2948 }
 2949                                     /*}}}*/
 2950 std::string pkgAcqIndexDiffs::Custom600Headers() const          /*{{{*/
 2951 {
 2952    if(State != StateApplyDiff)
 2953       return pkgAcqBaseIndex::Custom600Headers();
 2954    std::ostringstream patchhashes;
 2955    for (auto && hs : available_patches[0].result_hashes)
 2956       patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
 2957    for (auto && hs : available_patches[0].patch_hashes)
 2958       patchhashes <<  "\nPatch-0-" << hs.HashType() << "-Hash: " << hs.HashValue();
 2959    patchhashes << pkgAcqBaseIndex::Custom600Headers();
 2960    return patchhashes.str();
 2961 }
 2962                                     /*}}}*/
 2963 pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {}
 2964 
 2965 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor         /*{{{*/
 2966 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *const Owner,
 2967                          pkgAcqMetaClearSig *const TransactionManager,
 2968                          IndexTarget const &Target,
 2969                          DiffInfo const &patch,
 2970                          std::vector<pkgAcqIndexMergeDiffs *> const *const allPatches)
 2971     : pkgAcqBaseIndex(Owner, TransactionManager, Target),
 2972       patch(patch), allPatches(allPatches), State(StateFetchDiff)
 2973 {
 2974    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 2975 
 2976    Desc.Owner = this;
 2977    Desc.ShortDesc = Target.ShortDesc;
 2978    Desc.URI = Target.URI + ".diff/" + pkgAcquire::URIEncode(patch.file) + ".gz";
 2979    Desc.Description = Target.Description + " " + patch.file + ".pdiff";
 2980    DestFile = GetPartialFileNameFromURI(Desc.URI);
 2981 
 2982    if(Debug)
 2983       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 2984 
 2985    QueueURI(Desc);
 2986 }
 2987                                     /*}}}*/
 2988 void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 2989 {
 2990    if(Debug)
 2991       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 2992 
 2993    pkgAcqBaseIndex::Failed(Message,Cnf);
 2994    Status = StatDone;
 2995 
 2996    // check if we are the first to fail, otherwise we are done here
 2997    State = StateDoneDiff;
 2998    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 2999      I != allPatches->end(); ++I)
 3000       if ((*I)->State == StateErrorDiff)
 3001       {
 3002      State = StateErrorDiff;
 3003      return;
 3004       }
 3005 
 3006    // first failure means we should fallback
 3007    State = StateErrorDiff;
 3008    if (Debug)
 3009       std::clog << "Falling back to normal index file acquire" << std::endl;
 3010    RenameOnError(PDiffError);
 3011    std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
 3012    if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
 3013       Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
 3014    DestFile.clear();
 3015    new pkgAcqIndex(Owner, TransactionManager, Target);
 3016 }
 3017                                     /*}}}*/
 3018 void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes,   /*{{{*/
 3019                 pkgAcquire::MethodConfig const * const Cnf)
 3020 {
 3021    if(Debug)
 3022       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 3023 
 3024    Item::Done(Message, Hashes, Cnf);
 3025 
 3026    if (std::any_of(allPatches->begin(), allPatches->end(),
 3027         [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; }))
 3028    {
 3029       if(Debug)
 3030      std::clog << "Another patch failed already, no point in processing this one." << std::endl;
 3031       State = StateErrorDiff;
 3032       return;
 3033    }
 3034 
 3035    std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
 3036    std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
 3037    if (UnpatchedFile.empty())
 3038    {
 3039       _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str());
 3040       State = StateErrorDiff;
 3041       return;
 3042    }
 3043    std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
 3044 
 3045    switch (State)
 3046    {
 3047       case StateFetchDiff:
 3048      // check if this is the last completed diff
 3049      State = StateDoneDiff;
 3050      for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 3051            I != allPatches->end(); ++I)
 3052         if ((*I)->State != StateDoneDiff)
 3053         {
 3054            if(Debug)
 3055           std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 3056            return;
 3057         }
 3058      for (auto * diff : *allPatches)
 3059         Rename(diff->DestFile, GetMergeDiffsPatchFileName(UnpatchedFile, diff->patch.file));
 3060      // this is the last completed diff, so we are ready to apply now
 3061      DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
 3062      if(Debug)
 3063         std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
 3064      State = StateApplyDiff;
 3065      Local = true;
 3066      Desc.URI = "rred:" + pkgAcquire::URIEncode(UnpatchedFile);
 3067      QueueURI(Desc);
 3068      SetActiveSubprocess("rred");
 3069      return;
 3070       case StateApplyDiff:
 3071      // success in download & apply all diffs, finialize and clean up
 3072      if(Debug)
 3073         std::clog << "Queue patched file in place: " << std::endl
 3074            << DestFile << " -> " << PatchedFile << std::endl;
 3075 
 3076      // queue for copy by the transaction manager
 3077      TransactionManager->TransactionStageCopy(this, DestFile, GetKeepCompressedFileName(GetFinalFilename(), Target));
 3078 
 3079      // ensure the ed's are gone regardless of list-cleanup
 3080      for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 3081            I != allPatches->end(); ++I)
 3082         RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file));
 3083      RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile);
 3084 
 3085      // all set and done
 3086      Complete = true;
 3087      if(Debug)
 3088         std::clog << "allDone: " << DestFile << "\n" << std::endl;
 3089      return;
 3090       case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", patch.file.c_str()); break;
 3091       case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", patch.file.c_str()); break;
 3092    }
 3093 }
 3094                                     /*}}}*/
 3095 std::string pkgAcqIndexMergeDiffs::Custom600Headers() const     /*{{{*/
 3096 {
 3097    if(State != StateApplyDiff)
 3098       return pkgAcqBaseIndex::Custom600Headers();
 3099    std::ostringstream patchhashes;
 3100    unsigned int seen_patches = 0;
 3101    for (auto && hs : (*allPatches)[0]->patch.result_hashes)
 3102       patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
 3103    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 3104      I != allPatches->end(); ++I)
 3105    {
 3106       HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
 3107       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
 3108      patchhashes <<  "\nPatch-" << std::to_string(seen_patches) << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
 3109       ++seen_patches;
 3110    }
 3111    patchhashes << pkgAcqBaseIndex::Custom600Headers();
 3112    return patchhashes.str();
 3113 }
 3114                                     /*}}}*/
 3115 pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
 3116 
 3117 // AcqIndex::AcqIndex - Constructor                 /*{{{*/
 3118 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
 3119                          pkgAcqMetaClearSig * const TransactionManager,
 3120                          IndexTarget const &Target, bool const Derived)
 3121    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
 3122    CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
 3123 {
 3124    if (Derived)
 3125       return;
 3126    Init(Target.URI, Target.Description, Target.ShortDesc);
 3127 
 3128    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
 3129       std::clog << "New pkgIndex with TransactionManager "
 3130                 << TransactionManager << std::endl;
 3131 }
 3132                                     /*}}}*/
 3133 // AcqIndex::Init - deferred Constructor                /*{{{*/
 3134 void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
 3135                        string const &ShortDesc)
 3136 {
 3137    Stage = STAGE_DOWNLOAD;
 3138 
 3139    DestFile = GetPartialFileNameFromURI(URI);
 3140    size_t const nextExt = CompressionExtensions.find(' ');
 3141    if (nextExt == std::string::npos)
 3142    {
 3143       CurrentCompressionExtension = CompressionExtensions;
 3144       CompressionExtensions.clear();
 3145    }
 3146    else
 3147    {
 3148       CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt);
 3149       CompressionExtensions = CompressionExtensions.substr(nextExt+1);
 3150    }
 3151 
 3152    if (CurrentCompressionExtension == "uncompressed")
 3153    {
 3154       Desc.URI = URI;
 3155    }
 3156    else if (unlikely(CurrentCompressionExtension.empty()))
 3157       return;
 3158    else
 3159    {
 3160       Desc.URI = URI + '.' + CurrentCompressionExtension;
 3161       DestFile = DestFile + '.' + CurrentCompressionExtension;
 3162    }
 3163 
 3164    // store file size of the download to ensure the fetcher gives
 3165    // accurate progress reporting
 3166    FileSize = GetExpectedHashes().FileSize();
 3167 
 3168    Desc.Description = URIDesc;
 3169    Desc.Owner = this;
 3170    Desc.ShortDesc = ShortDesc;
 3171 
 3172    QueueURI(Desc);
 3173 }
 3174                                     /*}}}*/
 3175 // AcqIndex::Custom600Headers - Insert custom request headers       /*{{{*/
 3176 // ---------------------------------------------------------------------
 3177 /* The only header we use is the last-modified header. */
 3178 string pkgAcqIndex::Custom600Headers() const
 3179 {
 3180    std::string msg = pkgAcqBaseIndex::Custom600Headers();
 3181    msg.append("\nIndex-File: true");
 3182 
 3183    if (TransactionManager->LastMetaIndexParser == NULL)
 3184    {
 3185       std::string const Final = GetFinalFilename();
 3186 
 3187       struct stat Buf;
 3188       if (stat(Final.c_str(),&Buf) == 0)
 3189      msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
 3190    }
 3191 
 3192    if(Target.IsOptional)
 3193       msg += "\nFail-Ignore: true";
 3194 
 3195    return msg;
 3196 }
 3197                                     /*}}}*/
 3198 // AcqIndex::Failed - getting the indexfile failed          /*{{{*/
 3199 bool pkgAcqIndex::CommonFailed(std::string const &TargetURI,
 3200                    std::string const &Message, pkgAcquire::MethodConfig const *const Cnf)
 3201 {
 3202    pkgAcqBaseIndex::Failed(Message,Cnf);
 3203    // authorisation matches will not be fixed by other compression types
 3204    if (Status != StatAuthError)
 3205    {
 3206       if (CompressionExtensions.empty() == false)
 3207       {
 3208      Init(TargetURI, Desc.Description, Desc.ShortDesc);
 3209      Status = StatIdle;
 3210      return true;
 3211       }
 3212    }
 3213    return false;
 3214 }
 3215 void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 3216 {
 3217    if (CommonFailed(Target.URI, Message, Cnf))
 3218       return;
 3219 
 3220    if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
 3221       Status = StatDone;
 3222    else
 3223       TransactionManager->AbortTransaction();
 3224 }
 3225                                     /*}}}*/
 3226 // AcqIndex::Done - Finished a fetch                    /*{{{*/
 3227 // ---------------------------------------------------------------------
 3228 /* This goes through a number of states.. On the initial fetch the
 3229    method could possibly return an alternate filename which points
 3230    to the uncompressed version of the file. If this is so the file
 3231    is copied into the partial directory. In all other cases the file
 3232    is decompressed with a compressed uri. */
 3233 void pkgAcqIndex::Done(string const &Message,
 3234                        HashStringList const &Hashes,
 3235                pkgAcquire::MethodConfig const * const Cfg)
 3236 {
 3237    Item::Done(Message,Hashes,Cfg);
 3238 
 3239    switch(Stage) 
 3240    {
 3241       case STAGE_DOWNLOAD:
 3242          StageDownloadDone(Message);
 3243          break;
 3244       case STAGE_DECOMPRESS_AND_VERIFY:
 3245          StageDecompressDone();
 3246          break;
 3247    }
 3248 }
 3249                                     /*}}}*/
 3250 // AcqIndex::StageDownloadDone - Queue for decompress and verify    /*{{{*/
 3251 void pkgAcqIndex::StageDownloadDone(string const &Message)
 3252 {
 3253    Local = true;
 3254    Complete = true;
 3255 
 3256    std::string const AltFilename = LookupTag(Message,"Alt-Filename");
 3257    std::string Filename = LookupTag(Message,"Filename");
 3258 
 3259    // we need to verify the file against the current Release file again
 3260    // on if-modfied-since hit to avoid a stale attack against us
 3261    if (StringToBool(LookupTag(Message, "IMS-Hit"), false))
 3262    {
 3263       Filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
 3264       EraseFileName = DestFile = flCombine(flNotFile(DestFile), flNotDir(Filename));
 3265       if (symlink(Filename.c_str(), DestFile.c_str()) != 0)
 3266      _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
 3267       Stage = STAGE_DECOMPRESS_AND_VERIFY;
 3268       Desc.URI = "store:" + pkgAcquire::URIEncode(DestFile);
 3269       QueueURI(Desc);
 3270       SetActiveSubprocess(::URI(Desc.URI).Access);
 3271       return;
 3272    }
 3273    // methods like file:// give us an alternative (uncompressed) file
 3274    else if (Target.KeepCompressed == false && AltFilename.empty() == false)
 3275    {
 3276       Filename = AltFilename;
 3277       EraseFileName.clear();
 3278    }
 3279    // Methods like e.g. "file:" will give us a (compressed) FileName that is
 3280    // not the "DestFile" we set, in this case we uncompress from the local file
 3281    else if (Filename != DestFile && RealFileExists(DestFile) == false)
 3282    {
 3283       // symlinking ensures that the filename can be used for compression detection
 3284       // that is e.g. needed for by-hash which has no extension over file
 3285       if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
 3286      _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
 3287       else
 3288       {
 3289      EraseFileName = DestFile;
 3290      Filename = DestFile;
 3291       }
 3292    }
 3293 
 3294    Stage = STAGE_DECOMPRESS_AND_VERIFY;
 3295    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
 3296    if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
 3297       Desc.URI = "copy:" + pkgAcquire::URIEncode(Filename);
 3298    else
 3299       Desc.URI = "store:" + pkgAcquire::URIEncode(Filename);
 3300    if (DestFile == Filename)
 3301    {
 3302       if (CurrentCompressionExtension == "uncompressed")
 3303      return StageDecompressDone();
 3304       DestFile = "/dev/null";
 3305    }
 3306 
 3307    if (EraseFileName.empty() && Filename != AltFilename)
 3308       EraseFileName = Filename;
 3309 
 3310    // queue uri for the next stage
 3311    QueueURI(Desc);
 3312    SetActiveSubprocess(::URI(Desc.URI).Access);
 3313 }
 3314                                     /*}}}*/
 3315 // AcqIndex::StageDecompressDone - Final verification           /*{{{*/
 3316 void pkgAcqIndex::StageDecompressDone()
 3317 {
 3318    if (DestFile == "/dev/null")
 3319       DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
 3320 
 3321    // Done, queue for rename on transaction finished
 3322    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
 3323 }
 3324                                     /*}}}*/
 3325 pkgAcqIndex::~pkgAcqIndex() {}
 3326 
 3327 // AcqArchive::AcqArchive - Constructor                 /*{{{*/
 3328 // ---------------------------------------------------------------------
 3329 /* This just sets up the initial fetch environment and queues the first
 3330    possibilitiy */
 3331 pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sources,
 3332                  pkgRecords *const Recs, pkgCache::VerIterator const &Version,
 3333                  string &StoreFilename) : Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
 3334                               StoreFilename(StoreFilename),
 3335                               Trusted(false)
 3336 {
 3337    if (Version.Arch() == 0)
 3338    {
 3339       _error->Error(_("I wasn't able to locate a file for the %s package. "
 3340               "This might mean you need to manually fix this package. "
 3341               "(due to missing arch)"),
 3342             Version.ParentPkg().FullName().c_str());
 3343       return;
 3344    }
 3345 
 3346    // check if we have one trusted source for the package. if so, switch
 3347    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
 3348    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
 3349    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
 3350    bool seenUntrusted = false;
 3351    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
 3352    {
 3353       pkgIndexFile *Index;
 3354       if (Sources->FindIndex(i.File(),Index) == false)
 3355          continue;
 3356 
 3357       if (debugAuth == true)
 3358          std::cerr << "Checking index: " << Index->Describe()
 3359                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
 3360 
 3361       if (Index->IsTrusted() == true)
 3362       {
 3363          Trusted = true;
 3364      if (allowUnauth == false)
 3365         break;
 3366       }
 3367       else
 3368          seenUntrusted = true;
 3369    }
 3370 
 3371    // "allow-unauthenticated" restores apts old fetching behaviour
 3372    // that means that e.g. unauthenticated file:// uris are higher
 3373    // priority than authenticated http:// uris
 3374    if (allowUnauth == true && seenUntrusted == true)
 3375       Trusted = false;
 3376 
 3377    StoreFilename.clear();
 3378    for (auto Vf = Version.FileList(); Vf.end() == false; ++Vf)
 3379    {
 3380       auto const PkgF = Vf.File();
 3381       if (unlikely(PkgF.end()))
 3382      continue;
 3383       if (PkgF.Flagged(pkgCache::Flag::NotSource))
 3384      continue;
 3385       pkgIndexFile *Index;
 3386       if (Sources->FindIndex(PkgF, Index) == false)
 3387      continue;
 3388       if (Trusted && Index->IsTrusted() == false)
 3389      continue;
 3390 
 3391       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
 3392       // collect the hashes from the indexes
 3393       auto hsl = Parse.Hashes();
 3394       if (ExpectedHashes.empty())
 3395      ExpectedHashes = hsl;
 3396       else
 3397       {
 3398      // bad things will likely happen, but the user might be "lucky" still
 3399      // if the sources provide the same hashtypes (so that they aren't mixed)
 3400      for (auto const &hs : hsl)
 3401         if (ExpectedHashes.push_back(hs) == false)
 3402         {
 3403            _error->Warning("Sources disagree on hashes for supposely identical version '%s' of '%s'.",
 3404                    Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
 3405            break;
 3406         }
 3407       }
 3408       // only allow local volatile sources to have no hashes
 3409       if (PkgF.Flagged(pkgCache::Flag::LocalSource))
 3410      LocalSource = true;
 3411       else if (hsl.empty())
 3412      continue;
 3413 
 3414       std::string poolfilename = Parse.FileName();
 3415       if (poolfilename.empty())
 3416      continue;
 3417 
 3418       std::remove_reference<decltype(ModifyCustomFields())>::type fields;
 3419       {
 3420      auto const debIndex = dynamic_cast<pkgDebianIndexTargetFile const *const>(Index);
 3421      if (debIndex != nullptr)
 3422      {
 3423         auto const IT = debIndex->GetIndexTarget();
 3424         fields.emplace("Target-Repo-URI", IT.Option(IndexTarget::REPO_URI));
 3425         fields.emplace("Target-Release", IT.Option(IndexTarget::RELEASE));
 3426         fields.emplace("Target-Site", IT.Option(IndexTarget::SITE));
 3427      }
 3428       }
 3429       fields.emplace("Target-Base-URI", Index->ArchiveURI(""));
 3430       if (PkgF->Component != 0)
 3431      fields.emplace("Target-Component", PkgF.Component());
 3432       auto const RelF = PkgF.ReleaseFile();
 3433       if (RelF.end() == false)
 3434       {
 3435      if (RelF->Codename != 0)
 3436         fields.emplace("Target-Codename", RelF.Codename());
 3437      if (RelF->Archive != 0)
 3438         fields.emplace("Target-Suite", RelF.Archive());
 3439       }
 3440       fields.emplace("Target-Architecture", Version.Arch());
 3441       fields.emplace("Target-Type", flExtension(poolfilename));
 3442 
 3443       if (StoreFilename.empty())
 3444       {
 3445      /* We pick a filename based on the information we have for the version,
 3446         but we don't know what extension such a file should have, so we look
 3447         at the filenames used online and assume that they are the same for
 3448         all repositories containing this file */
 3449      StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' +
 3450              QuoteString(Version.VerStr(), "_:") + '_' +
 3451              QuoteString(Version.Arch(), "_:.") +
 3452              "." + flExtension(poolfilename);
 3453 
 3454      Desc.URI = Index->ArchiveURI(poolfilename);
 3455      Desc.Description = Index->ArchiveInfo(Version);
 3456      Desc.Owner = this;
 3457      Desc.ShortDesc = Version.ParentPkg().FullName(true);
 3458      auto &customfields = ModifyCustomFields();
 3459      for (auto const &f : fields)
 3460         customfields[f.first] = f.second;
 3461      FileSize = Version->Size;
 3462       }
 3463       else
 3464      PushAlternativeURI(Index->ArchiveURI(poolfilename), std::move(fields), true);
 3465    }
 3466    if (StoreFilename.empty())
 3467    {
 3468       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
 3469             Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
 3470       return;
 3471    }
 3472    if (FileSize == 0 && not _config->FindB("Acquire::AllowUnsizedPackages", false))
 3473    {
 3474       _error->Warning("Repository is broken: %s (= %s) has no Size information",
 3475               Version.ParentPkg().FullName(false).c_str(),
 3476               Version.VerStr());
 3477    }
 3478 
 3479    // Check if we already downloaded the file
 3480    struct stat Buf;
 3481    auto FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
 3482    if (stat(FinalFile.c_str(), &Buf) == 0)
 3483    {
 3484       // Make sure the size matches
 3485       if ((unsigned long long)Buf.st_size == Version->Size)
 3486       {
 3487      Complete = true;
 3488      Local = true;
 3489      Status = StatDone;
 3490      StoreFilename = DestFile = FinalFile;
 3491      return;
 3492       }
 3493 
 3494       /* Hmm, we have a file and its size does not match, this shouldn't
 3495      happen.. */
 3496       RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
 3497    }
 3498 
 3499    // Check the destination file
 3500    DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
 3501    if (stat(DestFile.c_str(), &Buf) == 0)
 3502    {
 3503       // Hmm, the partial file is too big, erase it
 3504       if ((unsigned long long)Buf.st_size > Version->Size)
 3505      RemoveFile("pkgAcqArchive::QueueNext", DestFile);
 3506       else
 3507      PartialSize = Buf.st_size;
 3508    }
 3509 
 3510    // Disables download of archives - useful if no real installation follows,
 3511    // e.g. if we are just interested in proposed installation order
 3512    if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
 3513    {
 3514       Complete = true;
 3515       Local = true;
 3516       Status = StatDone;
 3517       StoreFilename = DestFile = FinalFile;
 3518       return;
 3519    }
 3520 
 3521    // Create the item
 3522    Local = false;
 3523    QueueURI(Desc);
 3524 }
 3525                                     /*}}}*/
 3526 bool pkgAcqArchive::QueueNext() /*{{{*/
 3527 {
 3528    return false;
 3529 }
 3530                                     /*}}}*/
 3531 // AcqArchive::Done - Finished fetching                 /*{{{*/
 3532 // ---------------------------------------------------------------------
 3533 /* */
 3534 void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
 3535              pkgAcquire::MethodConfig const * const Cfg)
 3536 {
 3537    Item::Done(Message, Hashes, Cfg);
 3538 
 3539    // Grab the output filename
 3540    std::string const FileName = LookupTag(Message,"Filename");
 3541    if (DestFile !=  FileName && RealFileExists(DestFile) == false)
 3542    {
 3543       StoreFilename = DestFile = FileName;
 3544       Local = true;
 3545       Complete = true;
 3546       return;
 3547    }
 3548 
 3549    // Done, move it into position
 3550    string const FinalFile = GetFinalFilename();
 3551    Rename(DestFile,FinalFile);
 3552    StoreFilename = DestFile = FinalFile;
 3553    Complete = true;
 3554 }
 3555                                     /*}}}*/
 3556 // AcqArchive::Failed - Failure handler                 /*{{{*/
 3557 // ---------------------------------------------------------------------
 3558 /* Here we try other sources */
 3559 void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 3560 {
 3561    Item::Failed(Message,Cnf);
 3562 }
 3563                                     /*}}}*/
 3564 APT_PURE bool pkgAcqArchive::IsTrusted() const              /*{{{*/
 3565 {
 3566    return Trusted;
 3567 }
 3568                                     /*}}}*/
 3569 void pkgAcqArchive::Finished()                      /*{{{*/
 3570 {
 3571    if (Status == pkgAcquire::Item::StatDone &&
 3572        Complete == true)
 3573       return;
 3574    StoreFilename = string();
 3575 }
 3576                                     /*}}}*/
 3577 std::string pkgAcqArchive::DescURI() const              /*{{{*/
 3578 {
 3579    return Desc.URI;
 3580 }
 3581                                     /*}}}*/
 3582 std::string pkgAcqArchive::ShortDesc() const                /*{{{*/
 3583 {
 3584    return Desc.ShortDesc;
 3585 }
 3586                                     /*}}}*/
 3587 pkgAcqArchive::~pkgAcqArchive() {}
 3588 
 3589 // AcqChangelog::pkgAcqChangelog - Constructors             /*{{{*/
 3590 class pkgAcqChangelog::Private
 3591 {
 3592    public:
 3593    std::string FinalFile;
 3594 };
 3595 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
 3596       std::string const &DestDir, std::string const &DestFilename) :
 3597    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr())
 3598 {
 3599    Desc.URI = URI(Ver);
 3600    Init(DestDir, DestFilename);
 3601 }
 3602 // some parameters are char* here as they come likely from char* interfaces – which can also return NULL
 3603 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile,
 3604       char const * const Component, char const * const SrcName, char const * const SrcVersion,
 3605       const string &DestDir, const string &DestFilename) :
 3606    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
 3607 {
 3608    Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion);
 3609    Init(DestDir, DestFilename);
 3610 }
 3611 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner,
 3612       std::string const &URI, char const * const SrcName, char const * const SrcVersion,
 3613       const string &DestDir, const string &DestFilename) :
 3614    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
 3615 {
 3616    Desc.URI = URI;
 3617    Init(DestDir, DestFilename);
 3618 }
 3619 void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename)
 3620 {
 3621    if (Desc.URI.empty())
 3622    {
 3623       Status = StatError;
 3624       // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
 3625       strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
 3626       // Let the error message print something sensible rather than "Failed to fetch /"
 3627       if (DestFilename.empty())
 3628      DestFile = SrcName + ".changelog";
 3629       else
 3630      DestFile = DestFilename;
 3631       Desc.URI = "changelog:/" + pkgAcquire::URIEncode(DestFile);
 3632       return;
 3633    }
 3634 
 3635    std::string DestFileName;
 3636    if (DestFilename.empty())
 3637       DestFileName = flCombine(DestFile, SrcName + ".changelog");
 3638    else
 3639       DestFileName = flCombine(DestFile, DestFilename);
 3640 
 3641    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
 3642    std::string const systemTemp = GetTempDir(SandboxUser);
 3643    char tmpname[1000];
 3644    snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str());
 3645    if (NULL == mkdtemp(tmpname))
 3646    {
 3647       _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str());
 3648       Status = StatError;
 3649       return;
 3650    }
 3651    TemporaryDirectory = tmpname;
 3652 
 3653    ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", TemporaryDirectory.c_str(),
 3654      SandboxUser.c_str(), ROOT_GROUP, 0700);
 3655 
 3656    DestFile = flCombine(TemporaryDirectory, DestFileName);
 3657    if (DestDir.empty() == false)
 3658    {
 3659       d->FinalFile = flCombine(DestDir, DestFileName);
 3660       if (RealFileExists(d->FinalFile))
 3661       {
 3662      FileFd file1, file2;
 3663      if (file1.Open(DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive) &&
 3664            file2.Open(d->FinalFile, FileFd::ReadOnly) && CopyFile(file2, file1))
 3665      {
 3666         ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", DestFile.c_str(), "root", ROOT_GROUP, 0644);
 3667         struct timeval times[2];
 3668         times[0].tv_sec = times[1].tv_sec = file2.ModificationTime();
 3669         times[0].tv_usec = times[1].tv_usec = 0;
 3670         utimes(DestFile.c_str(), times);
 3671      }
 3672       }
 3673    }
 3674 
 3675    Desc.ShortDesc = "Changelog";
 3676    strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
 3677    Desc.Owner = this;
 3678    QueueURI(Desc);
 3679 }
 3680                                     /*}}}*/
 3681 std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver)  /*{{{*/
 3682 {
 3683    std::string const confOnline = "Acquire::Changelogs::AlwaysOnline";
 3684    bool AlwaysOnline = _config->FindB(confOnline, false);
 3685    if (AlwaysOnline == false)
 3686       for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
 3687       {
 3688      pkgCache::PkgFileIterator const PF = VF.File();
 3689      if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
 3690         continue;
 3691      pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
 3692      if (RF->Origin != 0 && _config->FindB(confOnline + "::Origin::" + RF.Origin(), false))
 3693      {
 3694         AlwaysOnline = true;
 3695         break;
 3696      }
 3697       }
 3698    if (AlwaysOnline == false)
 3699    {
 3700       pkgCache::PkgIterator const Pkg = Ver.ParentPkg();
 3701       if (Pkg->CurrentVer != 0 && Pkg.CurrentVer() == Ver)
 3702       {
 3703      std::string const root = _config->FindDir("Dir");
 3704      std::string const basename = root + std::string("usr/share/doc/") + Pkg.Name() + "/changelog";
 3705      std::string const debianname = basename + ".Debian";
 3706      if (FileExists(debianname))
 3707         return "copy://" + debianname;
 3708      else if (FileExists(debianname + ".gz"))
 3709         return "store://" + debianname + ".gz";
 3710      else if (FileExists(basename))
 3711         return "copy://" + basename;
 3712      else if (FileExists(basename + ".gz"))
 3713         return "store://" + basename + ".gz";
 3714       }
 3715    }
 3716 
 3717    char const * const SrcName = Ver.SourcePkgName();
 3718    char const * const SrcVersion = Ver.SourceVerStr();
 3719    // find the first source for this version which promises a changelog
 3720    for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
 3721    {
 3722       pkgCache::PkgFileIterator const PF = VF.File();
 3723       if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
 3724      continue;
 3725       pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
 3726       std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion);
 3727       if (uri.empty())
 3728      continue;
 3729       return uri;
 3730    }
 3731    return "";
 3732 }
 3733 std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls)
 3734 {
 3735    if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0))
 3736       return "";
 3737    std::string const serverConfig = "Acquire::Changelogs::URI";
 3738    std::string server;
 3739 #define APT_EMPTY_SERVER \
 3740    if (server.empty() == false) \
 3741    { \
 3742       if (server != "no") \
 3743      return server; \
 3744       return ""; \
 3745    }
 3746 #define APT_CHECK_SERVER(X, Y) \
 3747    if (Rls->X != 0) \
 3748    { \
 3749       std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
 3750       server = _config->Find(specialServerConfig); \
 3751       APT_EMPTY_SERVER \
 3752    }
 3753    // this way e.g. Debian-Security can fallback to Debian
 3754    APT_CHECK_SERVER(Label, "Override::")
 3755    APT_CHECK_SERVER(Origin, "Override::")
 3756 
 3757    if (RealFileExists(Rls.FileName()))
 3758    {
 3759       _error->PushToStack();
 3760       FileFd rf;
 3761       /* This can be costly. A caller wanting to get millions of URIs might
 3762      want to do this on its own once and use Override settings.
 3763      We don't do this here as Origin/Label are not as unique as they
 3764      should be so this could produce request order-dependent anomalies */
 3765       if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
 3766       {
 3767      pkgTagFile TagFile(&rf, rf.Size());
 3768      pkgTagSection Section;
 3769      if (TagFile.Step(Section) == true)
 3770         server = Section.FindS("Changelogs");
 3771       }
 3772       _error->RevertToStack();
 3773       APT_EMPTY_SERVER
 3774    }
 3775 
 3776    APT_CHECK_SERVER(Label, "")
 3777    APT_CHECK_SERVER(Origin, "")
 3778 #undef APT_CHECK_SERVER
 3779 #undef APT_EMPTY_SERVER
 3780    return "";
 3781 }
 3782 std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls,
 3783      char const * const Component, char const * const SrcName,
 3784      char const * const SrcVersion)
 3785 {
 3786    return URI(URITemplate(Rls), Component, SrcName, SrcVersion);
 3787 }
 3788 std::string pkgAcqChangelog::URI(std::string const &Template,
 3789      char const * const Component, char const * const SrcName,
 3790      char const * const SrcVersion)
 3791 {
 3792    if (Template.find("@CHANGEPATH@") == std::string::npos)
 3793       return "";
 3794 
 3795    // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/apt_1.1 or contrib/liba/libapt/libapt_2.0
 3796    std::string const Src{SrcName};
 3797    std::string path = pkgAcquire::URIEncode(APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1));
 3798    path.append("/").append(pkgAcquire::URIEncode(Src)).append("/");
 3799    path.append(pkgAcquire::URIEncode(Src)).append("_").append(pkgAcquire::URIEncode(StripEpoch(SrcVersion)));
 3800    // we omit component for releases without one (= flat-style repositories)
 3801    if (Component != NULL && strlen(Component) != 0)
 3802       path = pkgAcquire::URIEncode(Component) + "/" + path;
 3803 
 3804    return SubstVar(Template, "@CHANGEPATH@", path);
 3805 }
 3806                                     /*}}}*/
 3807 // AcqChangelog::Failed - Failure handler               /*{{{*/
 3808 void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
 3809 {
 3810    Item::Failed(Message,Cnf);
 3811 
 3812    std::string errText;
 3813    // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
 3814    strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
 3815 
 3816    // Error is probably something techy like 404 Not Found
 3817    if (ErrorText.empty())
 3818       ErrorText = errText;
 3819    else
 3820       ErrorText = errText + " (" + ErrorText + ")";
 3821 }
 3822                                     /*}}}*/
 3823 // AcqChangelog::Done - Item downloaded OK              /*{{{*/
 3824 void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes,
 3825               pkgAcquire::MethodConfig const * const Cnf)
 3826 {
 3827    Item::Done(Message,CalcHashes,Cnf);
 3828    if (d->FinalFile.empty() == false)
 3829    {
 3830       if (RemoveFile("pkgAcqChangelog::Done", d->FinalFile) == false ||
 3831         Rename(DestFile, d->FinalFile) == false)
 3832      Status = StatError;
 3833    }
 3834 
 3835    Complete = true;
 3836 }
 3837                                     /*}}}*/
 3838 pkgAcqChangelog::~pkgAcqChangelog()                 /*{{{*/
 3839 {
 3840    if (TemporaryDirectory.empty() == false)
 3841    {
 3842       RemoveFile("~pkgAcqChangelog", DestFile);
 3843       rmdir(TemporaryDirectory.c_str());
 3844    }
 3845    delete d;
 3846 }
 3847                                     /*}}}*/
 3848 
 3849 // AcqFile::pkgAcqFile - Constructor                    /*{{{*/
 3850 pkgAcqFile::pkgAcqFile(pkgAcquire *const Owner, string const &URI, HashStringList const &Hashes,
 3851                unsigned long long const Size, string const &Dsc, string const &ShortDesc,
 3852                const string &DestDir, const string &DestFilename,
 3853                bool const IsIndexFile) : Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
 3854 {
 3855    ::URI url{URI};
 3856    if (url.Path.find(' ') != std::string::npos || url.Path.find('%') == std::string::npos)
 3857       url.Path = pkgAcquire::URIEncode(url.Path);
 3858 
 3859    if(!DestFilename.empty())
 3860       DestFile = DestFilename;
 3861    else if(!DestDir.empty())
 3862       DestFile = DestDir + "/" + DeQuoteString(flNotDir(url.Path));
 3863    else
 3864       DestFile = DeQuoteString(flNotDir(url.Path));
 3865 
 3866    // Create the item
 3867    Desc.URI = std::string(url);
 3868    Desc.Description = Dsc;
 3869    Desc.Owner = this;
 3870 
 3871    // Set the short description to the archive component
 3872    Desc.ShortDesc = ShortDesc;
 3873 
 3874    // Get the transfer sizes
 3875    FileSize = Size;
 3876    struct stat Buf;
 3877    if (stat(DestFile.c_str(),&Buf) == 0)
 3878    {
 3879       // Hmm, the partial file is too big, erase it
 3880       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
 3881      RemoveFile("pkgAcqFile", DestFile);
 3882       else
 3883      PartialSize = Buf.st_size;
 3884    }
 3885 
 3886    QueueURI(Desc);
 3887 }
 3888                                     /*}}}*/
 3889 // AcqFile::Done - Item downloaded OK                   /*{{{*/
 3890 void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
 3891               pkgAcquire::MethodConfig const * const Cnf)
 3892 {
 3893    Item::Done(Message,CalcHashes,Cnf);
 3894 
 3895    std::string const FileName = LookupTag(Message,"Filename");
 3896    Complete = true;
 3897 
 3898    // The files timestamp matches
 3899    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 3900       return;
 3901 
 3902    // We have to copy it into place
 3903    if (RealFileExists(DestFile.c_str()) == false)
 3904    {
 3905       Local = true;
 3906       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
 3907       Cnf->Removable == true)
 3908       {
 3909      Desc.URI = "copy:" + pkgAcquire::URIEncode(FileName);
 3910      QueueURI(Desc);
 3911      return;
 3912       }
 3913 
 3914       // Erase the file if it is a symlink so we can overwrite it
 3915       struct stat St;
 3916       if (lstat(DestFile.c_str(),&St) == 0)
 3917       {
 3918      if (S_ISLNK(St.st_mode) != 0)
 3919         RemoveFile("pkgAcqFile::Done", DestFile);
 3920       }
 3921 
 3922       // Symlink the file
 3923       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
 3924       {
 3925      _error->PushToStack();
 3926      _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str());
 3927      std::stringstream msg;
 3928      _error->DumpErrors(msg, GlobalError::DEBUG, false);
 3929      _error->RevertToStack();
 3930      ErrorText = msg.str();
 3931      Status = StatError;
 3932      Complete = false;
 3933       }
 3934    }
 3935 }
 3936                                     /*}}}*/
 3937 string pkgAcqFile::Custom600Headers() const             /*{{{*/
 3938 {
 3939    string Header = pkgAcquire::Item::Custom600Headers();
 3940    if (not IsIndexFile)
 3941       return Header;
 3942    return Header + "\nIndex-File: true";
 3943 }
 3944                                     /*}}}*/
 3945 pkgAcqFile::~pkgAcqFile() {}
 3946 
 3947 void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/
 3948 {
 3949    pkgAcqFile::Failed(Message, Cnf);
 3950    if (Status == StatIdle)
 3951       return;
 3952    if (RealFileExists(DestFile))
 3953       Rename(DestFile, DestFile + ".FAILED");
 3954    Worker->ReplyAux(Desc);
 3955 }
 3956                                     /*}}}*/
 3957 void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/
 3958              pkgAcquire::MethodConfig const *const Cnf)
 3959 {
 3960    pkgAcqFile::Done(Message, CalcHashes, Cnf);
 3961    if (Status == StatDone)
 3962       Worker->ReplyAux(Desc);
 3963    else if (Status == StatAuthError || Status == StatError)
 3964       Worker->ReplyAux(Desc);
 3965 }
 3966                                     /*}}}*/
 3967 std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/
 3968 {
 3969    if (MaximumSize == 0)
 3970       return pkgAcqFile::Custom600Headers();
 3971    std::string maxsize;
 3972    strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize);
 3973    return pkgAcqFile::Custom600Headers().append(maxsize);
 3974 }
 3975                                     /*}}}*/
 3976 void pkgAcqAuxFile::Finished() /*{{{*/
 3977 {
 3978    auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
 3979    if (APT::String::Startswith(DestFile, dirname))
 3980    {
 3981       // the file is never returned by method requesting it, so fix up the permission now
 3982       if (FileExists(DestFile))
 3983       {
 3984      ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644);
 3985      if (Status == StatDone)
 3986         return;
 3987       }
 3988    }
 3989    else
 3990    {
 3991       dirname = flNotFile(DestFile);
 3992       RemoveFile("pkgAcqAuxFile::Finished", DestFile);
 3993       RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED");
 3994       rmdir(dirname.c_str());
 3995    }
 3996    DestFile.clear();
 3997 }
 3998                                     /*}}}*/
 3999 // GetAuxFileNameFromURI                        /*{{{*/
 4000 static std::string GetAuxFileNameFromURIInLists(std::string const &uri)
 4001 {
 4002    // check if we have write permission for our usual location.
 4003    auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
 4004    char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
 4005    std::string const tmpfile_tpl = flCombine(dirname, filetag);
 4006    std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
 4007    int const fd = mkstemp(tmpfile.get());
 4008    if (fd == -1)
 4009       return "";
 4010    RemoveFile("GetAuxFileNameFromURI", tmpfile.get());
 4011    close(fd);
 4012    return flCombine(dirname, URItoFileName(uri));
 4013 }
 4014 static std::string GetAuxFileNameFromURI(std::string const &uri)
 4015 {
 4016    auto const lists = GetAuxFileNameFromURIInLists(uri);
 4017    if (lists.empty() == false)
 4018       return lists;
 4019 
 4020    std::string tmpdir_tpl;
 4021    strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str());
 4022    std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free };
 4023    if (mkdtemp(tmpdir.get()) == nullptr)
 4024    {
 4025       _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get());
 4026       return flCombine("/nonexistent/auxfiles/", URItoFileName(uri));
 4027    }
 4028    chmod(tmpdir.get(), 0755);
 4029    auto const filename = flCombine(tmpdir.get(), URItoFileName(uri));
 4030    _error->PushToStack();
 4031    FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly);
 4032    if (in.IsOpen())
 4033    {
 4034       FileFd out(filename, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive);
 4035       CopyFile(in, out);
 4036       ChangeOwnerAndPermissionOfFile("GetAuxFileNameFromURI", filename.c_str(), "root", ROOT_GROUP, 0644);
 4037    }
 4038    _error->RevertToStack();
 4039    return filename;
 4040 }
 4041                                     /*}}}*/
 4042 pkgAcqAuxFile::pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker,
 4043                  std::string const &ShortDesc, std::string const &Desc, std::string const &URI,
 4044                  HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false),
 4045                                                    Owner(Owner), Worker(Worker), MaximumSize(MaximumSize)
 4046 {
 4047    /* very bad failures can happen while constructing which causes
 4048       us to hang as the aux request is never answered (e.g. method not available)
 4049       Ideally we catch failures earlier, but a safe guard can't hurt. */
 4050    if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching)
 4051       return;
 4052    Failed(std::string("400 URI Failure\n") +
 4053          "URI: " + URI + "\n" +
 4054          "Filename: " + DestFile,
 4055       nullptr);
 4056 }
 4057 pkgAcqAuxFile::~pkgAcqAuxFile() {}