apt  2.2.4
About: Apt (Advanced Package Tool) is a management system for software packages (Debian/Ubuntu). Release series 2.2.
  Fossies Dox: apt-2.2.4.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

pkgcachegen.cc
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4 
5  Package Cache Generator - Generator for the cache structure.
6 
7  This builds the cache structure from the abstract package list parser.
8 
9  ##################################################################### */
10  /*}}}*/
11 // Include Files /*{{{*/
12 #include <config.h>
13 
14 #include <apt-pkg/configuration.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/indexfile.h>
18 #include <apt-pkg/macros.h>
19 #include <apt-pkg/metaindex.h>
20 #include <apt-pkg/mmap.h>
21 #include <apt-pkg/pkgcache.h>
22 #include <apt-pkg/pkgcachegen.h>
23 #include <apt-pkg/pkgsystem.h>
24 #include <apt-pkg/progress.h>
25 #include <apt-pkg/sourcelist.h>
26 #include <apt-pkg/version.h>
27 
28 #include <algorithm>
29 #include <iostream>
30 #include <memory>
31 #include <string>
32 #include <vector>
33 #include <stddef.h>
34 #include <string.h>
35 #include <sys/stat.h>
36 #include <unistd.h>
37 
38 #include <apti18n.h>
39  /*}}}*/
40 constexpr auto APT_CACHE_START_DEFAULT = 24 * 1024 * 1024;
41 
42 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
43 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
44 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
45 
47  APT::StringView CurMd5, std::string const &CurLang);
48 
49 using std::string;
50 using APT::StringView;
51 
52 // Convert an offset returned from e.g. DynamicMMap or ptr difference to
53 // an uint32_t location without data loss.
54 template <typename T>
55 static inline uint32_t NarrowOffset(T ptr)
56 {
57  uint32_t res = static_cast<uint32_t>(ptr);
58  if (unlikely(ptr < 0))
59  abort();
60  if (unlikely(static_cast<T>(res) != ptr))
61  abort();
62  return res;
63 }
64 
65 // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
66 // ---------------------------------------------------------------------
67 /* We set the dirty flag and make sure that is written to the disk */
69  Map(*pMap), Cache(pMap,false), Progress(Prog),
70  CurrentRlsFile(nullptr), CurrentFile(nullptr), d(nullptr)
71 {
72 }
74 {
75  if (Map.Size() == 0)
76  {
77  // Setup the map interface..
79  _error->PushToStack();
81  bool const newError = _error->PendingError();
82  _error->MergeWithStack();
83  if (newError)
84  return false;
85  if (Map.Size() <= 0)
86  return false;
87 
89 
90  // Starting header
92 
93  // make room for the hashtables for packages and groups
94  if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer<void>))) == 0)
95  return false;
96 
97  map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
98  if (unlikely(idxVerSysName == 0))
99  return false;
100  map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
101  if (unlikely(idxArchitecture == 0))
102  return false;
103  map_stringitem_t idxArchitectures;
104 
105  std::vector<std::string> archs = APT::Configuration::getArchitectures();
106  if (archs.size() > 1)
107  {
108  std::vector<std::string>::const_iterator a = archs.begin();
109  std::string list = *a;
110  for (++a; a != archs.end(); ++a)
111  list.append(",").append(*a);
112  idxArchitectures = WriteStringInMap(list);
113  if (unlikely(idxArchitectures == 0))
114  return false;
115  }
116  else
117  idxArchitectures = idxArchitecture;
118 
120  Cache.HeaderP->VerSysName = idxVerSysName;
121  Cache.HeaderP->Architecture = idxArchitecture;
122  Cache.HeaderP->SetArchitectures(idxArchitectures);
123 
124  // Calculate the hash for the empty map, so ReMap does not fail
126  Cache.ReMap();
127  }
128  else
129  {
130  // Map directly from the existing file
131  Cache.ReMap();
132  Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
133  if (Cache.VS != _system->VS)
134  return _error->Error(_("Cache has an incompatible versioning system"));
135  }
136 
137  Cache.HeaderP->Dirty = true;
138  Map.Sync(0,sizeof(pkgCache::Header));
139  return true;
140 }
141  /*}}}*/
142 // CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
143 // ---------------------------------------------------------------------
144 /* We sync the data then unset the dirty flag in two steps so as to
145  advoid a problem during a crash */
147 {
148  if (_error->PendingError() == true || Map.validData() == false)
149  return;
150  if (Map.Sync() == false)
151  return;
152 
153  Cache.HeaderP->Dirty = false;
155 
156  if (_config->FindB("Debug::pkgCacheGen", false))
157  std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
158  Map.Sync(0,sizeof(pkgCache::Header));
159 }
160  /*}}}*/
161 void pkgCacheGenerator::ReMap(void const * const oldMap, void * const newMap, size_t oldSize) {/*{{{*/
162  if (oldMap == newMap)
163  return;
164 
165  // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
166  // returns (something, true) the iterator was not yet seen and we can
167  // remap it.
168  std::unordered_set<void *> seen;
169 
170  if (_config->FindB("Debug::pkgCacheGen", false))
171  std::clog << "Remapping from " << oldMap << " to " << newMap << std::endl;
172 
173  Cache.ReMap(false);
174 
175  if (CurrentFile != nullptr)
176  CurrentFile = static_cast<pkgCache::PackageFile *>(newMap) + (CurrentFile - static_cast<pkgCache::PackageFile const *>(oldMap));
177  if (CurrentRlsFile != nullptr)
178  CurrentRlsFile = static_cast<pkgCache::ReleaseFile *>(newMap) + (CurrentRlsFile - static_cast<pkgCache::ReleaseFile const *>(oldMap));
179 
180 #define APT_REMAP(TYPE) \
181  for (auto * const i : Dynamic<TYPE>::toReMap) \
182  if (seen.insert(i).second) \
183  i->ReMap(oldMap, newMap)
192 #undef APT_REMAP
193 
195  if (std::get<1>(seen.insert(ViewP)) == false)
196  continue;
197  // Ignore views outside of the cache.
198  if (ViewP->data() < static_cast<const char*>(oldMap)
199  || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
200  continue;
201  const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
202  *ViewP = StringView(data , ViewP->size());
203  }
204 } /*}}}*/
205 // CacheGenerator::WriteStringInMap /*{{{*/
207  const unsigned long &Len) {
208  size_t oldSize = Map.Size();
209  void const * const oldMap = Map.Data();
210  map_stringitem_t const index{NarrowOffset(Map.WriteString(String, Len))};
211  if (index != 0)
212  ReMap(oldMap, Map.Data(), oldSize);
213  return index;
214 }
215  /*}}}*/
216 // CacheGenerator::WriteStringInMap /*{{{*/
218  size_t oldSize = Map.Size();
219  void const * const oldMap = Map.Data();
220  map_stringitem_t const index{NarrowOffset(Map.WriteString(String))};
221  if (index != 0)
222  ReMap(oldMap, Map.Data(), oldSize);
223  return index;
224 }
225  /*}}}*/
226 uint32_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
227  size_t oldSize = Map.Size();
228  void const * const oldMap = Map.Data();
229  auto index = Map.Allocate(size);
230  if (index != 0)
231  ReMap(oldMap, Map.Data(), oldSize);
232  if (index != static_cast<uint32_t>(index))
233  abort(); // Internal error
234  return static_cast<uint32_t>(index);
235 }
236  /*}}}*/
237 // CacheGenerator::MergeList - Merge the package list /*{{{*/
238 // ---------------------------------------------------------------------
239 /* This provides the generation of the entries in the cache. Each loop
240  goes through a single package record from the underlying parse engine. */
242  pkgCache::VerIterator *OutVer)
243 {
244  List.Owner = this;
245 
246  unsigned int Counter = 0;
247  while (List.Step() == true)
248  {
249  string const PackageName = List.Package();
250  if (PackageName.empty() == true)
251  return false;
252 
253  Counter++;
254  if (Counter % 100 == 0 && Progress != 0)
255  Progress->Progress(List.Offset());
256 
257  APT::StringView Arch = List.Architecture();
258  Dynamic<APT::StringView> DynArch(Arch);
259  APT::StringView Version = List.Version();
260  Dynamic<APT::StringView> DynVersion(Version);
261  if (Version.empty() == true && Arch.empty() == true)
262  {
263  // package descriptions
264  if (MergeListGroup(List, PackageName) == false)
265  return false;
266  continue;
267  }
268 
269  // Get a pointer to the package structure
271  Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
272  if (NewPackage(Pkg, PackageName, Arch) == false)
273  // TRANSLATOR: The first placeholder is a package name,
274  // the other two should be copied verbatim as they include debug info
275  return _error->Error(_("Error occurred while processing %s (%s%d)"),
276  PackageName.c_str(), "NewPackage", 1);
277 
278 
279  if (Version.empty() == true)
280  {
281  if (MergeListPackage(List, Pkg) == false)
282  return false;
283  }
284  else
285  {
286  if (MergeListVersion(List, Pkg, Version, OutVer) == false)
287  return false;
288  }
289 
290  if (OutVer != 0)
291  return true;
292  }
293 
294  if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
295  return _error->Error(_("Wow, you exceeded the number of package "
296  "names this APT is capable of."));
297  if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
298  return _error->Error(_("Wow, you exceeded the number of versions "
299  "this APT is capable of."));
300  if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
301  return _error->Error(_("Wow, you exceeded the number of descriptions "
302  "this APT is capable of."));
303  if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
304  return _error->Error(_("Wow, you exceeded the number of dependencies "
305  "this APT is capable of."));
306 
307  return true;
308 }
309 // CacheGenerator::MergeListGroup /*{{{*/
310 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
311 {
312  pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
313  // a group has no data on it's own, only packages have it but these
314  // stanzas like this come from Translation- files to add descriptions,
315  // but without a version we don't need a description for it…
316  if (Grp.end() == true)
317  return true;
318  Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
319 
321  Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
322  for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
323  if (MergeListPackage(List, Pkg) == false)
324  return false;
325 
326  return true;
327 }
328  /*}}}*/
329 // CacheGenerator::MergeListPackage /*{{{*/
331 {
332  // we first process the package, then the descriptions
333  // (for deb this package processing is in fact a no-op)
335  Dynamic<pkgCache::VerIterator> DynVer(Ver);
336  if (List.UsePackage(Pkg, Ver) == false)
337  return _error->Error(_("Error occurred while processing %s (%s%d)"),
338  Pkg.Name(), "UsePackage", 1);
339 
340  // Find the right version to write the description
341  StringView CurMd5 = List.Description_md5();
342  std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
343  for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
344  {
345  pkgCache::DescIterator VerDesc = Ver.DescriptionList();
346 
347  // a version can only have one md5 describing it
348  if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
349  continue;
350 
351  map_stringitem_t md5idx = VerDesc->md5sum;
352  for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
353  {
354  // don't add a new description if we have one for the given
355  // md5 && language
356  if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
357  continue;
358 
359  AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
360  }
361 
362  // we can stop here as all "same" versions will share the description
363  break;
364  }
365 
366  return true;
367 }
368  /*}}}*/
369 // CacheGenerator::MergeListVersion /*{{{*/
371  APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
372 {
373  pkgCache::VerIterator Ver = Pkg.VersionList();
374  Dynamic<pkgCache::VerIterator> DynVer(Ver);
375  map_pointer<pkgCache::Version> *LastVer = &Pkg->VersionList;
376  void const * oldMap = Map.Data();
377 
378  auto Hash = List.VersionHash();
379  if (Ver.end() == false)
380  {
381  /* We know the list is sorted so we use that fact in the search.
382  Insertion of new versions is done with correct sorting */
383  int Res = 1;
384  for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
385  {
386  char const * const VerStr = Ver.VerStr();
387  Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
388  VerStr, VerStr + strlen(VerStr));
389  // Version is higher as current version - insert here
390  if (Res > 0)
391  break;
392  // Versionstrings are equal - is hash also equal?
393  if (Res == 0)
394  {
395  if (List.SameVersion(Hash, Ver) == true)
396  break;
397  // sort (volatile) sources above not-sources like the status file
398  if (CurrentFile == nullptr || (CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
399  {
400  auto VF = Ver.FileList();
401  for (; VF.end() == false; ++VF)
402  if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
403  break;
404  if (VF.end() == true)
405  break;
406  }
407  }
408  // proceed with the next till we have either the right
409  // or we found another version (which will be lower)
410  }
411 
412  /* We already have a version for this item, record that we saw it */
413  if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
414  {
415  if (List.UsePackage(Pkg,Ver) == false)
416  return _error->Error(_("Error occurred while processing %s (%s%d)"),
417  Pkg.Name(), "UsePackage", 2);
418 
419  if (NewFileVer(Ver,List) == false)
420  return _error->Error(_("Error occurred while processing %s (%s%d)"),
421  Pkg.Name(), "NewFileVer", 1);
422 
423  // Read only a single record and return
424  if (OutVer != 0)
425  {
426  *OutVer = Ver;
427  return true;
428  }
429 
430  return true;
431  }
432  }
433 
434  // Add a new version
435  map_pointer<pkgCache::Version> const verindex = NewVersion(Ver, Version, Pkg.MapPointer(), Hash, *LastVer);
436  if (unlikely(verindex == 0))
437  return _error->Error(_("Error occurred while processing %s (%s%d)"),
438  Pkg.Name(), "NewVersion", 1);
439 
440  if (oldMap != Map.Data())
441  LastVer = static_cast<map_pointer<pkgCache::Version> *>(Map.Data()) + (LastVer - static_cast<map_pointer<pkgCache::Version> const *>(oldMap));
442  *LastVer = verindex;
443 
444  if (unlikely(List.NewVersion(Ver) == false))
445  return _error->Error(_("Error occurred while processing %s (%s%d)"),
446  Pkg.Name(), "NewVersion", 2);
447 
448  if (unlikely(List.UsePackage(Pkg,Ver) == false))
449  return _error->Error(_("Error occurred while processing %s (%s%d)"),
450  Pkg.Name(), "UsePackage", 3);
451 
452  if (unlikely(NewFileVer(Ver,List) == false))
453  return _error->Error(_("Error occurred while processing %s (%s%d)"),
454  Pkg.Name(), "NewFileVer", 2);
455 
456  pkgCache::GrpIterator Grp = Pkg.Group();
457  Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
458 
459  /* If it is the first version of this package we need to add implicit
460  Multi-Arch dependencies to all other package versions in the group now -
461  otherwise we just add them for this new version */
462  if (Pkg.VersionList()->NextVer == 0)
463  {
464  pkgCache::PkgIterator P = Grp.PackageList();
466  for (; P.end() != true; P = Grp.NextPkg(P))
467  {
468  if (P->ID == Pkg->ID)
469  continue;
470  pkgCache::VerIterator V = P.VersionList();
472  for (; V.end() != true; ++V)
473  if (unlikely(AddImplicitDepends(V, Pkg) == false))
474  return _error->Error(_("Error occurred while processing %s (%s%d)"),
475  Pkg.Name(), "AddImplicitDepends", 1);
476  }
477  }
478  if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
479  return _error->Error(_("Error occurred while processing %s (%s%d)"),
480  Pkg.Name(), "AddImplicitDepends", 2);
481 
482  // Read only a single record and return
483  if (OutVer != 0)
484  {
485  *OutVer = Ver;
486  return true;
487  }
488 
489  /* Record the Description(s) based on their master md5sum */
490  StringView CurMd5 = List.Description_md5();
491 
492  /* Before we add a new description we first search in the group for
493  a version with a description of the same MD5 - if so we reuse this
494  description group instead of creating our own for this version */
495  for (pkgCache::PkgIterator P = Grp.PackageList();
496  P.end() == false; P = Grp.NextPkg(P))
497  {
498  for (pkgCache::VerIterator V = P.VersionList();
499  V.end() == false; ++V)
500  {
501  if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
502  continue;
503  Ver->DescriptionList = V->DescriptionList;
504  }
505  }
506 
507  // We haven't found reusable descriptions, so add the first description(s)
508  map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
509  std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
510  for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
511  if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
512  return false;
513  return true;
514 }
515  /*}}}*/
516 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
517 {
519  Dynamic<pkgCache::DescIterator> DynDesc(Desc);
520 
521  map_pointer<pkgCache::Description> const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
522  if (unlikely(descindex == 0))
523  return _error->Error(_("Error occurred while processing %s (%s%d)"),
524  Ver.ParentPkg().Name(), "NewDescription", 1);
525 
526  md5idx = Desc->md5sum;
527  Desc->ParentPkg = Ver.ParentPkg().MapPointer();
528 
529  // we add at the end, so that the start is constant as we need
530  // that to be able to efficiently share these lists
531  pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
532  for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
533  map_pointer<pkgCache::Description> * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
534  *LastNextDesc = descindex;
535 
536  if (NewFileDesc(Desc,List) == false)
537  return _error->Error(_("Error occurred while processing %s (%s%d)"),
538  Ver.ParentPkg().Name(), "NewFileDesc", 1);
539 
540  return true;
541 }
542  /*}}}*/
543  /*}}}*/
544 // CacheGenerator::NewGroup - Add a new group /*{{{*/
545 // ---------------------------------------------------------------------
546 /* This creates a new group structure and adds it to the hash table */
548 {
549  Dynamic<StringView> DName(Name);
550  Grp = Cache.FindGrp(Name);
551  if (Grp.end() == false)
552  return true;
553 
554  // Get a structure
555  auto const Group = AllocateInMap<pkgCache::Group>();
556  if (unlikely(Group == 0))
557  return false;
558 
559  map_stringitem_t const idxName = WriteStringInMap(Name);
560  if (unlikely(idxName == 0))
561  return false;
562 
563  Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
564  Grp->Name = idxName;
565 
566  // Insert it into the hash table
567  unsigned long const Hash = Cache.Hash(Name);
568  map_pointer<pkgCache::Group> *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
569 
570  while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
571  insertAt = &(Cache.GrpP + *insertAt)->Next;
572  Grp->Next = *insertAt;
573  *insertAt = Group;
574 
575  Grp->ID = Cache.HeaderP->GroupCount++;
576  return true;
577 }
578  /*}}}*/
579 // CacheGenerator::NewPackage - Add a new package /*{{{*/
580 // ---------------------------------------------------------------------
581 /* This creates a new package structure and adds it to the hash table */
583  StringView Arch) {
585  Dynamic<StringView> DName(Name);
586  Dynamic<StringView> DArch(Arch);
587  Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
588  if (unlikely(NewGroup(Grp, Name) == false))
589  return false;
590 
591  Pkg = Grp.FindPkg(Arch);
592  if (Pkg.end() == false)
593  return true;
594 
595  // Get a structure
596  auto const Package = AllocateInMap<pkgCache::Package>();
597  if (unlikely(Package == 0))
598  return false;
599  Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
600 
601  // Set the name, arch and the ID
602  Pkg->Group = Grp.MapPointer();
603  // all is mapped to the native architecture
604  map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
605  if (unlikely(idxArch == 0))
606  return false;
607  Pkg->Arch = idxArch;
608  Pkg->ID = Cache.HeaderP->PackageCount++;
609 
610  // Insert the package into our package list
611  if (Grp->FirstPackage == 0) // the group is new
612  {
613  Grp->FirstPackage = Package;
614  // Insert it into the hash table
615  map_id_t const Hash = Cache.Hash(Name);
616  map_pointer<pkgCache::Package> *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
617  while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
618  insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
619  Pkg->NextPackage = *insertAt;
620  *insertAt = Package;
621  }
622  else // Group the Packages together
623  {
624  // if sibling is provided by another package, this one is too
625  {
626  pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
627  if (M.end() == false) {
629  Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
630  for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
631  {
632  if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
633  continue;
634  pkgCache::VerIterator Ver = Prv.OwnerVer();
635  Dynamic<pkgCache::VerIterator> DynVer(Ver);
636  if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
638  (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
639  {
640  if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
641  continue;
642  if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
643  return false;
644  }
645  }
646  }
647  }
648  // let M-A:foreign package siblings provide this package
649  {
653  Dynamic<pkgCache::VerIterator> DynVer(Ver);
654 
655  for (P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
656  {
657  if (APT::Configuration::checkArchitecture(P.Arch()) == false)
658  continue;
659  for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
660  if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
661  if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
662  return false;
663  }
664  }
665  // and negative dependencies, don't forget negative dependencies
666  {
667  pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
668  if (M.end() == false) {
670  Dynamic<pkgCache::DepIterator> DynDep(Dep);
671  for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
672  {
673  if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
674  continue;
675  if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
676  Dep->Type != pkgCache::Dep::Replaces)
677  continue;
678  pkgCache::VerIterator Ver = Dep.ParentVer();
679  Dynamic<pkgCache::VerIterator> DynVer(Ver);
680  map_pointer<pkgCache::Dependency> * unused = NULL;
681  if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
682  return false;
683  }
684  }
685  }
686 
687  // this package is the new last package
688  pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
689  Pkg->NextPackage = LastPkg->NextPackage;
690  LastPkg->NextPackage = Package;
691  }
692  Grp->LastPackage = Package;
693 
694  // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
695  if (Arch == "any")
696  {
697  size_t const found = Name.rfind(':');
698  StringView ArchA = Name.substr(found + 1);
699  if (ArchA != "any")
700  {
701  // ArchA is used inside the loop which might remap (NameA is not used)
702  Dynamic<StringView> DynArchA(ArchA);
703  StringView NameA = Name.substr(0, found);
704  pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
705  Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
706  if (PkgA.end())
707  {
708  Dynamic<StringView> DynNameA(NameA);
709  if (NewPackage(PkgA, NameA, ArchA) == false)
710  return false;
711  }
712  if (unlikely(PkgA.end()))
713  return _error->Fatal("NewPackage was successful for %s:%s,"
714  "but the package doesn't exist anyhow!",
715  NameA.to_string().c_str(), ArchA.to_string().c_str());
716  else
717  {
718  pkgCache::PrvIterator Prv = PkgA.ProvidesList();
719  for (; Prv.end() == false; ++Prv)
720  {
721  if (Prv.IsMultiArchImplicit())
722  continue;
723  pkgCache::VerIterator V = Prv.OwnerVer();
724  if (ArchA != V.ParentPkg().Arch())
725  continue;
727  return false;
728  }
729  pkgCache::VerIterator V = PkgA.VersionList();
731  for (; V.end() == false; ++V)
732  {
734  return false;
735  }
736  }
737  }
738  }
739  return true;
740 }
741  /*}}}*/
742 // CacheGenerator::AddImplicitDepends /*{{{*/
746 {
747  APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
748  Dynamic<APT::StringView> DynArch(Arch);
749  map_pointer<pkgCache::Dependency> *OldDepLast = NULL;
750  /* MultiArch handling introduces a lot of implicit Dependencies:
751  - MultiArch: same → Co-Installable if they have the same version
752  - All others conflict with all other group members */
753  bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
754  pkgCache::PkgIterator D = G.PackageList();
756  map_stringitem_t const VerStrIdx = V->VerStr;
757  for (; D.end() != true; D = G.NextPkg(D))
758  {
759  if (Arch == D.Arch() || D->VersionList == 0)
760  continue;
761  /* We allow only one installed arch at the time
762  per group, therefore each group member conflicts
763  with all other group members */
764  if (coInstall == true)
765  {
766  // Replaces: ${self}:other ( << ${binary:Version})
767  NewDepends(D, V, VerStrIdx,
769  OldDepLast);
770  // Breaks: ${self}:other (!= ${binary:Version})
771  NewDepends(D, V, VerStrIdx,
773  OldDepLast);
774  } else {
775  // Conflicts: ${self}:other
776  NewDepends(D, V, 0,
778  OldDepLast);
779  }
780  }
781  return true;
782 }
785 {
786  /* MultiArch handling introduces a lot of implicit Dependencies:
787  - MultiArch: same → Co-Installable if they have the same version
788  - All others conflict with all other group members */
789  map_pointer<pkgCache::Dependency> *OldDepLast = NULL;
790  bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
791  if (coInstall == true)
792  {
793  map_stringitem_t const VerStrIdx = V->VerStr;
794  // Replaces: ${self}:other ( << ${binary:Version})
795  NewDepends(D, V, VerStrIdx,
797  OldDepLast);
798  // Breaks: ${self}:other (!= ${binary:Version})
799  NewDepends(D, V, VerStrIdx,
801  OldDepLast);
802  } else {
803  // Conflicts: ${self}:other
804  NewDepends(D, V, 0,
806  OldDepLast);
807  }
808  return true;
809 }
810 
811  /*}}}*/
812 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
813 // ---------------------------------------------------------------------
814 /* */
816  ListParser &List)
817 {
818  if (CurrentFile == nullptr)
819  return true;
820 
821  // Get a structure
822  auto const VerFile = AllocateInMap<pkgCache::VerFile>();
823  if (VerFile == 0)
824  return false;
825 
828 
829  // Link it to the end of the list
830  map_pointer<pkgCache::VerFile> *Last = &Ver->FileList;
831  for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
832  Last = &V->NextFile;
833  VF->NextFile = *Last;
834  *Last = VF.MapPointer();
835 
836  VF->Offset = List.Offset();
837  VF->Size = List.Size();
838  if (Cache.HeaderP->MaxVerFileSize < VF->Size)
839  Cache.HeaderP->MaxVerFileSize = VF->Size;
841 
842  return true;
843 }
844  /*}}}*/
845 // CacheGenerator::NewVersion - Create a new Version /*{{{*/
846 // ---------------------------------------------------------------------
847 /* This puts a version structure in the linked list */
849  APT::StringView const &VerStr,
850  map_pointer<pkgCache::Package> const ParentPkg,
851  uint32_t Hash,
853 {
854  // Get a structure
855  auto const Version = AllocateInMap<pkgCache::Version>();
856  if (Version == 0)
857  return 0;
858 
859  // Fill it in
860  Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
861  auto d = AllocateInMap<pkgCache::Version::Extra>(); // sequence point so Ver can be moved if needed
862  Ver->d = d;
863  if (not Ver.PhasedUpdatePercentage(100))
864  abort();
865 
866  //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
867  Ver->NextVer = Next;
868  Ver->ParentPkg = ParentPkg;
869  Ver->Hash = Hash;
870  Ver->ID = Cache.HeaderP->VersionCount++;
871 
872  // try to find the version string in the group for reuse
873  pkgCache::PkgIterator Pkg = Ver.ParentPkg();
874  pkgCache::GrpIterator Grp = Pkg.Group();
875  if (Pkg.end() == false && Grp.end() == false)
876  {
877  for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
878  {
879  if (Pkg == P)
880  continue;
881  for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
882  {
883  int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
884  if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
885  {
886  Ver->VerStr = V->VerStr;
887  return Version;
888  }
889  else if (cmp < 0)
890  break;
891  }
892  }
893  }
894  // haven't found the version string, so create
895  map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
896  if (unlikely(idxVerStr == 0))
897  return 0;
898  Ver->VerStr = idxVerStr;
899  return Version;
900 }
901  /*}}}*/
902 // CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
903 // ---------------------------------------------------------------------
904 /* */
906  ListParser &List)
907 {
908  if (CurrentFile == nullptr)
909  return true;
910 
911  // Get a structure
912  auto const DescFile = AllocateInMap<pkgCache::DescFile>();
913  if (DescFile == 0)
914  return false;
915 
918 
919  // Link it to the end of the list
920  map_pointer<pkgCache::DescFile> *Last = &Desc->FileList;
921  for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
922  Last = &D->NextFile;
923 
924  DF->NextFile = *Last;
925  *Last = DF.MapPointer();
926 
927  DF->Offset = List.Offset();
928  DF->Size = List.Size();
929  if (Cache.HeaderP->MaxDescFileSize < DF->Size)
930  Cache.HeaderP->MaxDescFileSize = DF->Size;
932 
933  return true;
934 }
935  /*}}}*/
936 // CacheGenerator::NewDescription - Create a new Description /*{{{*/
937 // ---------------------------------------------------------------------
938 /* This puts a description structure in the linked list */
940  const string &Lang,
941  APT::StringView md5sum,
942  map_stringitem_t const idxmd5str)
943 {
944  // Get a structure
945  auto const Description = AllocateInMap<pkgCache::Description>();
946  if (Description == 0)
947  return 0;
948 
949  // Fill it in
950  Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
951  Desc->ID = Cache.HeaderP->DescriptionCount++;
952  map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
953  if (unlikely(idxlanguage_code == 0))
954  return 0;
955  Desc->language_code = idxlanguage_code;
956 
957  if (idxmd5str != 0)
958  Desc->md5sum = idxmd5str;
959  else
960  {
961  map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
962  if (unlikely(idxmd5sum == 0))
963  return 0;
964  Desc->md5sum = idxmd5sum;
965  }
966 
967  return Description;
968 }
969  /*}}}*/
970 // CacheGenerator::NewDepends - Create a dependency element /*{{{*/
971 // ---------------------------------------------------------------------
972 /* This creates a dependency element in the tree. It is linked to the
973  version and to the package that it is pointing to. */
976  map_stringitem_t const Version,
977  uint8_t const Op,
978  uint8_t const Type,
979  map_pointer<pkgCache::Dependency> * &OldDepLast)
980 {
981  void const * const oldMap = Map.Data();
982  // Get a structure
983  auto const Dependency = AllocateInMap<pkgCache::Dependency>();
984  if (unlikely(Dependency == 0))
985  return false;
986 
987  bool isDuplicate = false;
988  map_pointer<pkgCache::DependencyData> DependencyData = 0;
989  map_pointer<pkgCache::DependencyData> PreviousData = 0;
990  if (Pkg->RevDepends != 0)
991  {
992  pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
993  DependencyData = L->DependencyData;
994  do {
995  pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
996  if (Version > D->Version)
997  break;
998  if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
999  {
1000  isDuplicate = true;
1001  break;
1002  }
1003  PreviousData = DependencyData;
1004  DependencyData = D->NextData;
1005  } while (DependencyData != 0);
1006  }
1007 
1008  if (isDuplicate == false)
1009  {
1010  DependencyData = AllocateInMap<pkgCache::DependencyData>();
1011  if (unlikely(DependencyData == 0))
1012  return false;
1013  }
1014 
1015  pkgCache::Dependency * Link = Cache.DepP + Dependency;
1016  Link->ParentVer = Ver.MapPointer();
1017  Link->DependencyData = DependencyData;
1018  Link->ID = Cache.HeaderP->DependsCount++;
1019 
1020  pkgCache::DepIterator Dep(Cache, Link);
1021  if (isDuplicate == false)
1022  {
1023  Dep->Type = Type;
1024  Dep->CompareOp = Op;
1025  Dep->Version = Version;
1026  Dep->Package = Pkg.MapPointer();
1028  if (PreviousData != 0)
1029  {
1030  pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1031  Dep->NextData = D->NextData;
1032  D->NextData = DependencyData;
1033  }
1034  else if (Pkg->RevDepends != 0)
1035  {
1036  pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1037  Dep->NextData = D->DependencyData;
1038  }
1039  }
1040 
1041  if (isDuplicate == true || PreviousData != 0)
1042  {
1043  pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1044  Link->NextRevDepends = L->NextRevDepends;
1045  L->NextRevDepends = Dependency;
1046  }
1047  else
1048  {
1049  Link->NextRevDepends = Pkg->RevDepends;
1050  Pkg->RevDepends = Dependency;
1051  }
1052 
1053 
1054  // Do we know where to link the Dependency to?
1055  if (OldDepLast == NULL)
1056  {
1057  OldDepLast = &Ver->DependsList;
1058  for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1059  OldDepLast = &D->NextDepends;
1060  } else if (oldMap != Map.Data())
1061  OldDepLast = static_cast<map_pointer<pkgCache::Dependency> *>(Map.Data()) + (OldDepLast - static_cast<map_pointer<pkgCache::Dependency> const *>(oldMap));
1062 
1063  Dep->NextDepends = *OldDepLast;
1064  *OldDepLast = Dependency;
1065  OldDepLast = &Dep->NextDepends;
1066  return true;
1067 }
1068  /*}}}*/
1069 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1070 // ---------------------------------------------------------------------
1071 /* This creates a Group and the Package to link this dependency to if
1072  needed and handles also the caching of the old endpoint */
1074  StringView PackageName,
1075  StringView Arch,
1076  StringView Version,
1077  uint8_t const Op,
1078  uint8_t const Type)
1079 {
1081  Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1082  Dynamic<StringView> DynPackageName(PackageName);
1083  Dynamic<StringView> DynArch(Arch);
1084  Dynamic<StringView> DynVersion(Version);
1085  if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1086  return false;
1087 
1088  map_stringitem_t idxVersion = 0;
1089  if (Version.empty() == false)
1090  {
1091  int const CmpOp = Op & 0x0F;
1092  // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1093  if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1094  idxVersion = Ver->VerStr;
1095 
1096  if (idxVersion == 0)
1097  {
1099  if (unlikely(idxVersion == 0))
1100  return false;
1101  }
1102  }
1103 
1104  bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1105  Type == pkgCache::Dep::Conflicts ||
1106  Type == pkgCache::Dep::Replaces);
1107 
1109  Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1110  if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1111  {
1112  // Locate the target package
1113  Pkg = Grp.FindPkg(Arch);
1114  if (Pkg.end() == true) {
1115  if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1116  return false;
1117  }
1118 
1119  /* Caching the old end point speeds up generation substantially */
1120  if (OldDepVer != Ver) {
1121  OldDepLast = NULL;
1122  OldDepVer = Ver;
1123  }
1124 
1125  return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1126  }
1127  else
1128  {
1129  /* Caching the old end point speeds up generation substantially */
1130  if (OldDepVer != Ver) {
1131  OldDepLast = NULL;
1132  OldDepVer = Ver;
1133  }
1134 
1135  for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1136  {
1137  if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1138  return false;
1139  }
1140  }
1141  return true;
1142 }
1143  /*}}}*/
1144 // ListParser::NewProvides - Create a Provides element /*{{{*/
1147  StringView PkgArch,
1148  StringView Version,
1149  uint8_t const Flags)
1150 {
1151  pkgCache const &Cache = Owner->Cache;
1152  Dynamic<StringView> DynPkgName(PkgName);
1153  Dynamic<StringView> DynArch(PkgArch);
1154  Dynamic<StringView> DynVersion(Version);
1155 
1156  // We do not add self referencing provides
1157  if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1158  (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1159  (Version.empty() || Version == Ver.VerStr()))
1160  return true;
1161 
1162  // Locate the target package
1164  Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1165  if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1166  return false;
1167 
1168  map_stringitem_t idxProvideVersion = 0;
1169  if (Version.empty() == false) {
1171  if (unlikely(idxProvideVersion == 0))
1172  return false;
1173  }
1174  return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1175 }
1177  pkgCache::PkgIterator &Pkg,
1178  map_stringitem_t const ProvideVersion,
1179  uint8_t const Flags)
1180 {
1181  // Get a structure
1182  auto const Provides = AllocateInMap<pkgCache::Provides>();
1183  if (unlikely(Provides == 0))
1184  return false;
1186 
1187  // Fill it in
1189  Prv->Version = Ver.MapPointer();
1190  Prv->ProvideVersion = ProvideVersion;
1191  Prv->Flags = Flags;
1192  Prv->NextPkgProv = Ver->ProvidesList;
1193  Ver->ProvidesList = Prv.MapPointer();
1194 
1195  // Link it to the package
1196  Prv->ParentPkg = Pkg.MapPointer();
1197  Prv->NextProvides = Pkg->ProvidesList;
1198  Pkg->ProvidesList = Prv.MapPointer();
1199  return true;
1200 }
1201  /*}}}*/
1202 // ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/
1204  StringView Version, uint8_t const Flags) {
1205  pkgCache &Cache = Owner->Cache;
1206  pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1207  Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1208  Dynamic<StringView> DynPackage(Package);
1209  Dynamic<StringView> DynVersion(Version);
1210 
1211  if (Grp.end() == true || Grp->FirstPackage == 0)
1212  return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1213  else
1214  {
1215  map_stringitem_t idxProvideVersion = 0;
1216  if (Version.empty() == false) {
1218  if (unlikely(idxProvideVersion == 0))
1219  return false;
1220  }
1221 
1223  bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1224  pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1225  Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1227  Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1228  for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1229  {
1230  if (isImplicit && OwnerPkg == Pkg)
1231  continue;
1232  if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1233  continue;
1234  if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1235  return false;
1236  }
1237  }
1238  return true;
1239 }
1240  /*}}}*/
1241 bool pkgCacheListParser::SameVersion(uint32_t Hash, /*{{{*/
1242  pkgCache::VerIterator const &Ver)
1243 {
1244  return Hash == Ver->Hash;
1245 }
1246  /*}}}*/
1247 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to /*{{{*/
1248 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1249  unsigned long Flags)
1250 {
1251  CurrentRlsFile = nullptr;
1252  if (File.empty() && Site.empty())
1253  return true;
1254 
1255  // Get some space for the structure
1256  auto const idxFile = AllocateInMap<pkgCache::ReleaseFile>();
1257  if (unlikely(idxFile == 0))
1258  return false;
1259  CurrentRlsFile = Cache.RlsFileP + idxFile;
1260 
1261  // Fill it in
1262  map_stringitem_t const idxFileName = WriteStringInMap(File);
1263  map_stringitem_t const idxSite = StoreString(MIXED, Site);
1264  if (unlikely(idxFileName == 0 || idxSite == 0))
1265  return false;
1266  CurrentRlsFile->FileName = idxFileName;
1267  CurrentRlsFile->Site = idxSite;
1271  RlsFileName = File;
1274 
1275  return true;
1276 }
1277  /*}}}*/
1278 // CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
1279 // ---------------------------------------------------------------------
1280 /* This is used to select which file is to be associated with all newly
1281  added versions. The caller is responsible for setting the IMS fields. */
1282 bool pkgCacheGenerator::SelectFile(std::string const &File,
1283  pkgIndexFile const &Index,
1284  std::string const &Architecture,
1285  std::string const &Component,
1286  unsigned long const Flags)
1287 {
1288  CurrentFile = nullptr;
1289  // Get some space for the structure
1290  auto const idxFile = AllocateInMap<pkgCache::PackageFile>();
1291  if (unlikely(idxFile == 0))
1292  return false;
1293  CurrentFile = Cache.PkgFileP + idxFile;
1294 
1295  // Fill it in
1296  map_stringitem_t const idxFileName = WriteStringInMap(File);
1297  if (unlikely(idxFileName == 0))
1298  return false;
1299  CurrentFile->FileName = idxFileName;
1302  map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1303  if (unlikely(idxIndexType == 0))
1304  return false;
1305  CurrentFile->IndexType = idxIndexType;
1306  if (Architecture.empty())
1308  else
1309  {
1310  map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1311  if (unlikely(arch == 0))
1312  return false;
1313  CurrentFile->Architecture = arch;
1314  }
1315  map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1316  if (unlikely(component == 0))
1317  return false;
1318  CurrentFile->Component = component;
1319  CurrentFile->Flags = Flags;
1320  if (CurrentRlsFile != nullptr)
1322  else
1323  CurrentFile->Release = 0;
1324  PkgFileName = File;
1327 
1328  if (Progress != 0)
1329  Progress->SubProgress(Index.Size());
1330  return true;
1331 }
1332  /*}}}*/
1333 // CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
1334 // ---------------------------------------------------------------------
1335 /* This is used to create handles to strings. Given the same text it
1336  always returns the same number */
1338  unsigned int Size)
1339 {
1340  auto strings = &strMixed;
1341  switch(type) {
1342  case MIXED: strings = &strMixed; break;
1343  case VERSIONNUMBER: strings = &strVersions; break;
1344  case SECTION: strings = &strSections; break;
1345  default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1346  }
1347 
1348  auto const item = strings->find({S, Size, nullptr, 0});
1349  if (item != strings->end())
1350  return item->item;
1351 
1352  map_stringitem_t const idxString = WriteStringInMap(S,Size);
1353  strings->insert({nullptr, Size, this, idxString});
1354  return idxString;
1355 }
1356  /*}}}*/
1357 // CheckValidity - Check that a cache is up-to-date /*{{{*/
1358 // ---------------------------------------------------------------------
1359 /* This just verifies that each file in the list of index files exists,
1360  has matching attributes with the cache and the cache does not have
1361  any extra files. */
1363 public:
1364  ScopedErrorRevert() { _error->PushToStack(); }
1365  ~ScopedErrorRevert() { _error->RevertToStack(); }
1366 };
1367 
1368 static bool CheckValidity(FileFd &CacheFile, std::string const &CacheFileName,
1369  pkgSourceList &List,
1370  FileIterator const Start,
1371  FileIterator const End,
1372  MMap **OutMap = 0,
1373  pkgCache **OutCache = 0)
1374 {
1375  if (CacheFileName.empty())
1376  return false;
1377  ScopedErrorRevert ser;
1378 
1379  bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1380  // No file, certainly invalid
1381  if (CacheFile.Open(CacheFileName, FileFd::ReadOnly, FileFd::None) == false)
1382  {
1383  if (Debug == true)
1384  std::clog << "CacheFile " << CacheFileName << " doesn't exist" << std::endl;
1385  return false;
1386  }
1387 
1388  if (_config->FindI("APT::Cache-Start", 0) == 0)
1389  {
1390  auto const size = CacheFile.FileSize();
1391  if (std::numeric_limits<int>::max() >= size && size > APT_CACHE_START_DEFAULT)
1392  _config->Set("APT::Cache-Start", size);
1393  }
1394 
1395  if (List.GetLastModifiedTime() > CacheFile.ModificationTime())
1396  {
1397  if (Debug == true)
1398  std::clog << "sources.list is newer than the cache" << std::endl;
1399  return false;
1400  }
1401 
1402  // Map it
1403  std::unique_ptr<MMap> Map(new MMap(CacheFile,0));
1404  if (unlikely(Map->validData()) == false)
1405  return false;
1406  std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1407  pkgCache &Cache = *CacheP.get();
1408  if (_error->PendingError() || Map->Size() == 0)
1409  {
1410  if (Debug == true)
1411  std::clog << "Errors are pending or Map is empty() for " << CacheFileName << std::endl;
1412  return false;
1413  }
1414 
1415  std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1416  memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1417  std::vector<pkgIndexFile *> Files;
1418  for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1419  {
1420  if (Debug == true)
1421  std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1422  pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1423  if (RlsFile.end() == true)
1424  {
1425  if (Debug == true)
1426  std::clog << "FindInCache returned end-Pointer" << std::endl;
1427  return false;
1428  }
1429 
1430  RlsVisited[RlsFile->ID] = true;
1431  if (Debug == true)
1432  std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1433 
1434  std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1435  std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1436  [](pkgIndexFile const * const I) { return I->HasPackages(); });
1437  }
1438  for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1439  if (RlsVisited[I] == false)
1440  {
1441  if (Debug == true)
1442  std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1443  return false;
1444  }
1445 
1446  std::copy(Start, End, std::back_inserter(Files));
1447 
1448  /* Now we check every index file, see if it is in the cache,
1449  verify the IMS data and check that it is on the disk too.. */
1450  std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1451  memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1452  for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1453  {
1454  if (Debug == true)
1455  std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1456  if ((*PkgFile)->Exists() == false)
1457  {
1458  if (Debug == true)
1459  std::clog << "file doesn't exist" << std::endl;
1460  continue;
1461  }
1462 
1463  // FindInCache is also expected to do an IMS check.
1464  pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1465  if (File.end() == true)
1466  {
1467  if (Debug == true)
1468  std::clog << "FindInCache returned end-Pointer" << std::endl;
1469  return false;
1470  }
1471 
1472  Visited[File->ID] = true;
1473  if (Debug == true)
1474  std::clog << "with ID " << File->ID << " is valid" << std::endl;
1475  }
1476 
1477  for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1478  if (Visited[I] == false)
1479  {
1480  if (Debug == true)
1481  std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1482  return false;
1483  }
1484 
1485  if (_error->PendingError() == true)
1486  {
1487  if (Debug == true)
1488  {
1489  std::clog << "Validity failed because of pending errors:" << std::endl;
1490  _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1491  }
1492  return false;
1493  }
1494 
1495  if (OutMap != 0)
1496  *OutMap = Map.release();
1497  if (OutCache != 0)
1498  *OutCache = CacheP.release();
1499  return true;
1500 }
1501  /*}}}*/
1502 // ComputeSize - Compute the total size of a bunch of files /*{{{*/
1503 // ---------------------------------------------------------------------
1504 /* Size is kind of an abstract notion that is only used for the progress
1505  meter */
1507 {
1508  map_filesize_t TotalSize = 0;
1509  if (List != NULL)
1510  {
1511  for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1512  {
1513  std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1514  for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1515  if ((*j)->HasPackages() == true)
1516  TotalSize += (*j)->Size();
1517  }
1518  }
1519 
1520  for (; Start < End; ++Start)
1521  {
1522  if ((*Start)->HasPackages() == false)
1523  continue;
1524  TotalSize += (*Start)->Size();
1525  }
1526  return TotalSize;
1527 }
1528  /*}}}*/
1529 // BuildCache - Merge the list of index files into the cache /*{{{*/
1531  OpProgress * const Progress,
1532  map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1533  pkgSourceList const * const List,
1534  FileIterator const Start, FileIterator const End)
1535 {
1536  bool mergeFailure = false;
1537 
1538  auto const indexFileMerge = [&](pkgIndexFile * const I) {
1539  if (I->HasPackages() == false || mergeFailure)
1540  return;
1541 
1542  if (I->Exists() == false)
1543  return;
1544 
1545  if (I->FindInCache(Gen.GetCache()).end() == false)
1546  {
1547  _error->Warning("Duplicate sources.list entry %s",
1548  I->Describe().c_str());
1549  return;
1550  }
1551 
1552  map_filesize_t const Size = I->Size();
1553  if (Progress != NULL)
1554  Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1555  CurrentSize += Size;
1556 
1557  if (I->Merge(Gen,Progress) == false)
1558  mergeFailure = true;
1559  };
1560 
1561  if (List != NULL)
1562  {
1563  for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1564  {
1565  if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1566  {
1567  _error->Warning("Duplicate sources.list entry %s",
1568  (*i)->Describe().c_str());
1569  continue;
1570  }
1571 
1572  if ((*i)->Merge(Gen, Progress) == false)
1573  return false;
1574 
1575  std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1576  if (Indexes != NULL)
1577  std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1578  if (mergeFailure)
1579  return false;
1580  }
1581  }
1582 
1583  if (Start != End)
1584  {
1585  Gen.SelectReleaseFile("", "");
1586  std::for_each(Start, End, indexFileMerge);
1587  if (mergeFailure)
1588  return false;
1589  }
1590  return true;
1591 }
1592  /*}}}*/
1593 // CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
1594 // ---------------------------------------------------------------------
1595 /* This makes sure that the status cache (the cache that has all
1596  index files from the sources list and all local ones) is ready
1597  to be mmaped. If OutMap is not zero then a MMap object representing
1598  the cache will be stored there. This is pretty much mandatory if you
1599  are using AllowMem. AllowMem lets the function be run as non-root
1600  where it builds the cache 'fast' into a memory buffer. */
1601 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1602 {
1603  map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", APT_CACHE_START_DEFAULT);
1604  map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1605  map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1606  Flags |= MMap::Moveable;
1607  if (_config->FindB("APT::Cache-Fallback", false) == true)
1608  Flags |= MMap::Fallback;
1609  if (CacheF != NULL)
1610  return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1611  else
1612  return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1613 }
1614 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1615  std::string const &FileName)
1616 {
1617  FileFd SCacheF(FileName, FileFd::WriteAtomic);
1618  if (SCacheF.IsOpen() == false || SCacheF.Failed())
1619  return false;
1620 
1621  fchmod(SCacheF.Fd(),0644);
1622 
1623  // Write out the main data
1624  if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1625  return _error->Error(_("IO Error saving source cache"));
1626 
1627  // Write out the proper header
1628  Gen->GetCache().HeaderP->Dirty = false;
1629  Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1630  if (SCacheF.Seek(0) == false ||
1631  SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1632  return _error->Error(_("IO Error saving source cache"));
1633  Gen->GetCache().HeaderP->Dirty = true;
1634  return true;
1635 }
1636 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1637  std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, FileFd &CacheF)
1638 {
1639  Map.reset(CreateDynamicMMap(NULL, 0));
1640  if (unlikely(Map->validData()) == false)
1641  return false;
1642  if (CacheF.IsOpen() == false || CacheF.Seek(0) == false || CacheF.Failed())
1643  return false;
1644  _error->PushToStack();
1645  uint32_t const alloc = Map->RawAllocate(CacheF.Size());
1646  bool const newError = _error->PendingError();
1647  _error->MergeWithStack();
1648  if (alloc == 0 && newError)
1649  return false;
1650  if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1651  return false;
1652  Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1653  return Gen->Start();
1654 }
1656  MMap **OutMap,bool)
1657 {
1658  return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1659 }
1661  MMap **OutMap,pkgCache **OutCache, bool)
1662 {
1663  // FIXME: deprecate the ignored AllowMem parameter
1664  bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1665 
1666  std::vector<pkgIndexFile *> Files;
1667  if (_system->AddStatusFiles(Files) == false)
1668  return false;
1669 
1670  // Decide if we can write to the files..
1671  string const CacheFileName = _config->FindFile("Dir::Cache::pkgcache");
1672  string const SrcCacheFileName = _config->FindFile("Dir::Cache::srcpkgcache");
1673 
1674  // ensure the cache directory exists
1675  if (CacheFileName.empty() == false || SrcCacheFileName.empty() == false)
1676  {
1677  string dir = _config->FindDir("Dir::Cache");
1678  size_t const len = dir.size();
1679  if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1680  dir = dir.substr(0, len - 5);
1681  if (CacheFileName.empty() == false)
1682  CreateDirectory(dir, flNotFile(CacheFileName));
1683  if (SrcCacheFileName.empty() == false)
1684  CreateDirectory(dir, flNotFile(SrcCacheFileName));
1685  }
1686 
1687  if (Progress != NULL)
1688  Progress->OverallProgress(0,1,1,_("Reading package lists"));
1689 
1690  bool pkgcache_fine = false;
1691  bool srcpkgcache_fine = false;
1692  bool volatile_fine = List.GetVolatileFiles().empty();
1693  FileFd CacheFile;
1694  if (CheckValidity(CacheFile, CacheFileName, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1695  volatile_fine ? OutCache : NULL) == true)
1696  {
1697  if (Debug == true)
1698  std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1699  pkgcache_fine = true;
1700  srcpkgcache_fine = true;
1701  }
1702 
1703  FileFd SrcCacheFile;
1704  if (pkgcache_fine == false)
1705  {
1706  if (CheckValidity(SrcCacheFile, SrcCacheFileName, List, Files.end(), Files.end()) == true)
1707  {
1708  if (Debug == true)
1709  std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1710  srcpkgcache_fine = true;
1711  }
1712  }
1713 
1714  if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1715  {
1716  if (Progress != NULL)
1717  Progress->OverallProgress(1,1,1,_("Reading package lists"));
1718  return true;
1719  }
1720 
1721  bool Writeable = false;
1722  if (srcpkgcache_fine == false || pkgcache_fine == false)
1723  {
1724  if (CacheFileName.empty() == false)
1725  Writeable = access(flNotFile(CacheFileName).c_str(),W_OK) == 0;
1726  else if (SrcCacheFileName.empty() == false)
1727  Writeable = access(flNotFile(SrcCacheFileName).c_str(),W_OK) == 0;
1728 
1729  if (Debug == true)
1730  std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1731  }
1732 
1733  // At this point we know we need to construct something, so get storage ready
1734  std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1735  if (unlikely(Map->validData()) == false)
1736  return false;
1737  if (Debug == true)
1738  std::clog << "Open memory Map (not filebased)" << std::endl;
1739 
1740  std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1741  map_filesize_t CurrentSize = 0;
1742  std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1743  map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1744  if (srcpkgcache_fine == true && pkgcache_fine == false)
1745  {
1746  if (Debug == true)
1747  std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1748  if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1749  return false;
1750  srcpkgcache_fine = true;
1751  TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1752  }
1753  else if (srcpkgcache_fine == false)
1754  {
1755  if (Debug == true)
1756  std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1757  Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1758  if (Gen->Start() == false)
1759  return false;
1760 
1761  TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1762  if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1763  Files.end(),Files.end()) == false)
1764  return false;
1765 
1766  if (Writeable == true && SrcCacheFileName.empty() == false)
1767  if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFileName) == false)
1768  return false;
1769  }
1770 
1771  if (pkgcache_fine == false)
1772  {
1773  if (Debug == true)
1774  std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1775  if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1776  Files.begin(), Files.end()) == false)
1777  return false;
1778 
1779  if (Writeable == true && CacheFileName.empty() == false)
1780  if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFileName) == false)
1781  return false;
1782  }
1783 
1784  if (Debug == true)
1785  std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1786 
1787  if (volatile_fine == false)
1788  {
1789  if (Gen == nullptr)
1790  {
1791  if (Debug == true)
1792  std::clog << "Populate new MMap with cachefile contents" << std::endl;
1793  if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1794  return false;
1795  }
1796 
1797  Files = List.GetVolatileFiles();
1798  if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1799  Files.begin(), Files.end()) == false)
1800  return false;
1801  }
1802 
1803  if (OutMap != nullptr)
1804  *OutMap = Map.release();
1805 
1806  if (Debug == true)
1807  std::clog << "Everything is ready for shipping" << std::endl;
1808  return true;
1809 }
1810  /*}}}*/
1811 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1813 public:
1814  ScopedErrorMerge() { _error->PushToStack(); }
1815  ~ScopedErrorMerge() { _error->MergeWithStack(); }
1816 };
1818 {
1819  std::vector<pkgIndexFile *> Files;
1820  if (_system->AddStatusFiles(Files) == false)
1821  return false;
1822 
1823  ScopedErrorMerge sem;
1824  std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1825  if (unlikely(Map->validData()) == false)
1826  return false;
1827  map_filesize_t CurrentSize = 0;
1828  map_filesize_t TotalSize = 0;
1829  TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1830 
1831  // Build the status cache
1832  if (Progress != NULL)
1833  Progress->OverallProgress(0,1,1,_("Reading package lists"));
1834  pkgCacheGenerator Gen(Map.get(),Progress);
1835  if (Gen.Start() == false || _error->PendingError() == true)
1836  return false;
1837  if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1838  Files.begin(), Files.end()) == false)
1839  return false;
1840 
1841  if (_error->PendingError() == true)
1842  return false;
1843  *OutMap = Map.release();
1844 
1845  return true;
1846 }
1847  /*}}}*/
1848 // IsDuplicateDescription /*{{{*/
1850  APT::StringView CurMd5, std::string const &CurLang)
1851 {
1852  // Descriptions in the same link-list have all the same md5
1853  if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
1854  return false;
1855  for (; Desc.end() == false; ++Desc)
1856  if (Desc.LanguageCode() == CurLang)
1857  return true;
1858  return false;
1859 }
1860  /*}}}*/
1861 
1862 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
static bool std::string const metaIndex const *const pkgAcqMetaClearSig *const pkgAcquire::Item *const I
return false
if(isError)
Simple subset of std::string_view from C++17.
Definition: string_view.h:27
constexpr StringView substr(size_t pos, size_t n=npos) const
Definition: string_view.h:48
constexpr size_t length() const
Definition: string_view.h:138
constexpr const char * data() const
Definition: string_view.h:133
size_t rfind(int c, size_t pos) const
Definition: string_view.h:69
std::string to_string() const
Definition: string_view.h:106
constexpr bool empty() const
Definition: string_view.h:132
bool Open(bool WithLock=true)
int FindI(const char *Name, int const &Default=0) const
void Set(const std::string &Name, const std::string &Value)
Definition: configuration.h:92
std::string Find(const char *Name, const char *Default=0) const
bool FindB(const char *Name, bool const &Default=false) const
std::string FindDir(const char *Name, const char *Default=0) const
std::string FindFile(const char *Name, const char *Default=0) const
unsigned long RawAllocate(unsigned long long Size, unsigned long Aln=0)
Definition: mmap.cc:325
unsigned long WriteString(const char *String, unsigned long Len=std::numeric_limits< unsigned long >::max())
Definition: mmap.cc:414
unsigned long Allocate(unsigned long ItemSize)
Definition: mmap.cc:350
void UsePools(Pool &P, unsigned int Count)
Definition: mmap.h:107
Definition: fileutl.h:39
bool IsOpen()
Definition: fileutl.h:150
@ None
Definition: fileutl.h:79
@ WriteAtomic
Definition: fileutl.h:74
@ ReadOnly
Definition: fileutl.h:59
bool Write(const void *From, unsigned long long Size)
Definition: fileutl.cc:2819
bool Seek(unsigned long long To)
Definition: fileutl.cc:2875
bool Failed()
Definition: fileutl.h:151
int Fd()
Definition: fileutl.h:147
bool Read(void *To, unsigned long long Size, bool AllowEof)
Definition: fileutl.h:89
unsigned long long Size()
Definition: fileutl.cc:2967
@ DEBUG
for developers only in areas it is hard to print something directly
Definition: error.h:66
Definition: mmap.h:41
bool Sync()
Definition: mmap.cc:161
bool validData() const
Definition: mmap.h:66
@ Fallback
Definition: mmap.h:59
@ Moveable
Definition: mmap.h:59
void * Data()
Definition: mmap.h:63
unsigned long long Size()
Definition: mmap.h:64
void OverallProgress(unsigned long long Current, unsigned long long Total, unsigned long long Size, const std::string &Op)
Definition: progress.cc:55
void Progress(unsigned long long Current)
Definition: progress.cc:43
void SubProgress(unsigned long long SubTotal, const std::string &Op="", float const Percent=-1)
Definition: progress.cc:73
virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const
Definition: metaindex.cc:22
virtual std::string Describe() const
Definition: metaindex.cc:17
bool Exists(std::string const &MetaKey) const
tests if a checksum for this file is available
Definition: metaindex.cc:100
virtual bool Merge(pkgCacheGenerator &Gen, OpProgress *Prog) const
Definition: metaindex.cc:27
std::string PkgFileName
Definition: pkgcachegen.h:109
pkgCache::PackageFile * CurrentFile
Definition: pkgcachegen.h:110
APT_HIDDEN bool AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx)
Definition: pkgcachegen.cc:516
APT_HIDDEN bool AddImplicitDepends(pkgCache::GrpIterator &G, pkgCache::PkgIterator &P, pkgCache::VerIterator &V)
Definition: pkgcachegen.cc:743
APT_HIDDEN map_stringitem_t WriteStringInMap(APT::StringView String)
Definition: pkgcachegen.h:44
map_stringitem_t StoreString(StringType const type, const char *S, unsigned int const Size)
bool NewPackage(pkgCache::PkgIterator &Pkg, APT::StringView Name, APT::StringView Arch)
Definition: pkgcachegen.cc:582
bool MergeList(ListParser &List, pkgCache::VerIterator *Ver=0)
Definition: pkgcachegen.cc:241
static APT_PUBLIC bool MakeStatusCache(pkgSourceList &List, OpProgress *Progress, MMap **OutMap=0, bool AllowMem=false)
APT_HIDDEN bool MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
Definition: pkgcachegen.cc:330
virtual ~pkgCacheGenerator()
Definition: pkgcachegen.cc:146
bool SelectReleaseFile(const std::string &File, const std::string &Site, unsigned long Flags=0)
APT_HIDDEN bool MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg, APT::StringView const &Version, pkgCache::VerIterator *&OutVer)
Definition: pkgcachegen.cc:370
static APT_PUBLIC bool MakeOnlyStatusCache(OpProgress *Progress, DynamicMMap **OutMap)
bool NewGroup(pkgCache::GrpIterator &Grp, APT::StringView Name)
Definition: pkgcachegen.cc:547
map_pointer< T > AllocateInMap()
Definition: pkgcachegen.h:48
void ReMap(void const *const oldMap, void *const newMap, size_t oldSize)
Definition: pkgcachegen.cc:161
map_pointer< pkgCache::Version > NewVersion(pkgCache::VerIterator &Ver, APT::StringView const &VerStr, map_pointer< pkgCache::Package > const ParentPkg, uint32_t Hash, map_pointer< pkgCache::Version > const Next)
Definition: pkgcachegen.cc:848
bool SelectFile(const std::string &File, pkgIndexFile const &Index, std::string const &Architecture, std::string const &Component, unsigned long Flags=0)
OpProgress * Progress
Definition: pkgcachegen.h:105
bool NewDepends(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver, map_stringitem_t const Version, uint8_t const Op, uint8_t const Type, map_pointer< pkgCache::Dependency > *&OldDepLast)
Definition: pkgcachegen.cc:974
pkgCache & GetCache()
Definition: pkgcachegen.h:137
bool NewFileVer(pkgCache::VerIterator &Ver, ListParser &List)
Definition: pkgcachegen.cc:815
map_pointer< pkgCache::Description > NewDescription(pkgCache::DescIterator &Desc, const std::string &Lang, APT::StringView md5sum, map_stringitem_t const idxmd5str)
Definition: pkgcachegen.cc:939
pkgCache::ReleaseFile * CurrentRlsFile
Definition: pkgcachegen.h:108
bool NewProvides(pkgCache::VerIterator &Ver, pkgCache::PkgIterator &Pkg, map_stringitem_t const ProvidesVersion, uint8_t const Flags)
pkgCacheGenerator(DynamicMMap *Map, OpProgress *Progress)
Definition: pkgcachegen.cc:68
DynamicMMap & Map
Definition: pkgcachegen.h:103
APT_HIDDEN bool MergeListGroup(ListParser &List, std::string const &GrpName)
Definition: pkgcachegen.cc:310
void *const d
Definition: pkgcachegen.h:156
bool NewFileDesc(pkgCache::DescIterator &Desc, ListParser &List)
Definition: pkgcachegen.cc:905
std::string RlsFileName
Definition: pkgcachegen.h:107
virtual bool UsePackage(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver)=0
virtual bool SameVersion(uint32_t Hash, pkgCache::VerIterator const &Ver)
virtual map_filesize_t Offset()=0
virtual std::string Package()=0
virtual std::vector< std::string > AvailableDescriptionLanguages()=0
virtual APT::StringView Description_md5()=0
virtual uint32_t VersionHash()=0
map_stringitem_t StoreString(pkgCacheGenerator::StringType const type, const char *S, unsigned int Size)
Definition: pkgcachegen.h:184
pkgCacheGenerator * Owner
Definition: pkgcachegen.h:173
virtual map_filesize_t Size()=0
virtual bool Step()=0
bool NewDepends(pkgCache::VerIterator &Ver, APT::StringView Package, APT::StringView Arch, APT::StringView Version, uint8_t const Op, uint8_t const Type)
bool NewProvidesAllArch(pkgCache::VerIterator &Ver, APT::StringView Package, APT::StringView Version, uint8_t const Flags)
virtual APT::StringView Version()=0
virtual bool NewVersion(pkgCache::VerIterator &Ver)=0
virtual ~pkgCacheListParser()
virtual APT::StringView Architecture()=0
pkgCache::VerIterator OldDepVer
Definition: pkgcachegen.h:177
map_pointer< pkgCache::Dependency > * OldDepLast
Definition: pkgcachegen.h:178
bool NewProvides(pkgCache::VerIterator &Ver, APT::StringView PkgName, APT::StringView PkgArch, APT::StringView Version, uint8_t const Flags)
map_id_t Hash(APT::StringView S) const
Definition: pkgcache.h:242
Provides * ProvideP
Definition: pkgcache.h:230
ReleaseFile * RlsFileP
Definition: pkgcache.h:226
char const * NativeArch()
Definition: pkgcache.h:798
void ReMap(void const *const oldMap, void *const newMap)
Dependency * DepP
Definition: pkgcache.h:231
PkgIterator FindPkg(APT::StringView Arch=APT::StringView("any", 3)) const
DependencyData * DepDataP
Definition: pkgcache.h:232
Package * PkgP
Definition: pkgcache.h:223
pkgVersioningSystem * VS
Definition: pkgcache.h:275
Version * VerP
Definition: pkgcache.h:228
Description * DescP
Definition: pkgcache.h:229
Group * GrpP
Definition: pkgcache.h:222
DescFile * DescFileP
Definition: pkgcache.h:225
Header * HeaderP
Definition: pkgcache.h:221
char * StrP
Definition: pkgcache.h:233
PackageFile * PkgFileP
Definition: pkgcache.h:227
APT::StringView ViewString(map_stringitem_t idx) const
Definition: pkgcache.h:254
GrpIterator FindGrp(APT::StringView Name)
Definition: pkgcache.cc:296
VerFile * VerFileP
Definition: pkgcache.h:224
APT_HIDDEN uint32_t CacheHash()
Definition: pkgcache.cc:230
const char * Label
Definition: indexfile.h:118
virtual const Type * GetType() const =0
virtual unsigned long Size() const =0
std::vector< pkgIndexFile * > GetVolatileFiles() const
Definition: sourcelist.cc:530
const_iterator begin() const
Definition: sourcelist.h:97
time_t GetLastModifiedTime()
Definition: sourcelist.cc:510
const_iterator end() const
Definition: sourcelist.h:98
std::vector< metaIndex * >::const_iterator const_iterator
Definition: sourcelist.h:76
virtual bool AddStatusFiles(std::vector< pkgIndexFile * > &List)=0
pkgVersioningSystem *const VS
Definition: pkgsystem.h:62
const char * Label
Definition: version.h:34
virtual int DoCmpVersion(const char *A, const char *Aend, const char *B, const char *Bend)=0
Configuration * _config
string flNotFile(string File)
Definition: fileutl.cc:676
bool CreateDirectory(string const &Parent, string const &Path)
Definition: fileutl.cc:364
#define APT_HIDDEN
Definition: macros.h:78
APT_PUBLIC std::vector< std::string > const getArchitectures(bool const &Cached=true)
Returns a vector of Architectures we support.
APT_PUBLIC bool checkArchitecture(std::string const &Arch)
Are we interested in the given Architecture?
static int StringViewCompareFast(StringView a, StringView b)
Faster comparison for string views (compare size before data)
Definition: string_view.h:145
pkgCache - Structure definitions for the cache file
uint32_t map_id_t
Definition: pkgcache.h:93
uint64_t map_filesize_t
Definition: pkgcache.h:89
static bool loadBackMMapFromFile(std::unique_ptr< pkgCacheGenerator > &Gen, std::unique_ptr< DynamicMMap > &Map, OpProgress *const Progress, FileFd &CacheF)
static map_filesize_t ComputeSize(pkgSourceList const *const List, FileIterator Start, FileIterator End)
static bool CheckValidity(FileFd &CacheFile, std::string const &CacheFileName, pkgSourceList &List, FileIterator const Start, FileIterator const End, MMap **OutMap=0, pkgCache **OutCache=0)
std::vector< pkgIndexFile * >::iterator FileIterator
Definition: pkgcachegen.cc:43
static bool writeBackMMapToFile(pkgCacheGenerator *const Gen, DynamicMMap *const Map, std::string const &FileName)
#define APT_REMAP(TYPE)
static bool BuildCache(pkgCacheGenerator &Gen, OpProgress *const Progress, map_filesize_t &CurrentSize, map_filesize_t TotalSize, pkgSourceList const *const List, FileIterator const Start, FileIterator const End)
static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc, APT::StringView CurMd5, std::string const &CurLang)
static uint32_t NarrowOffset(T ptr)
Definition: pkgcachegen.cc:55
constexpr auto APT_CACHE_START_DEFAULT
Definition: pkgcachegen.cc:40
static DynamicMMap * CreateDynamicMMap(FileFd *const CacheF, unsigned long Flags)
pkgSystem * _system
Definition: pkgsystem.cc:24
@ MultiArchImplicit
Definition: pkgcache.h:175
information for a single dependency record
Definition: pkgcache.h:738
map_flags_t CompareOp
comparison operator specified on the depends line
Definition: pkgcache.h:752
map_stringitem_t Version
string of the version the dependency is applied against
Definition: pkgcache.h:740
map_pointer< DependencyData > NextData
Definition: pkgcache.h:754
map_number_t Type
Dependency type - Depends, Recommends, Conflicts, etc.
Definition: pkgcache.h:748
map_pointer< Dependency > NextRevDepends
next reverse dependency of this package
Definition: pkgcache.h:762
map_pointer< pkgCache::DependencyData > DependencyData
Definition: pkgcache.h:758
map_pointer< Version > ParentVer
version of the package which has the depends
Definition: pkgcache.h:760
map_id_t ID
unique sequel ID
Definition: pkgcache.h:767
uint32_t GetHashTableSize() const
Definition: pkgcache.h:386
map_id_t ProvidesCount
Definition: pkgcache.h:344
map_id_t DependsDataCount
Definition: pkgcache.h:339
map_fileid_t DescFileCount
Definition: pkgcache.h:343
DynamicMMap::Pool Pools[2 *12]
The Pool structures manage the allocation pools that the generator uses.
Definition: pkgcache.h:373
map_fileid_t ReleaseFileCount
Definition: pkgcache.h:340
map_filesize_small_t CacheFileSize
Hash of the file (TODO: Rename)
Definition: pkgcache.h:397
map_id_t DescriptionCount
Definition: pkgcache.h:337
map_stringitem_t VerSysName
String representing the version system used.
Definition: pkgcache.h:355
map_filesize_t MaxVerFileSize
The maximum size of a raw entry from the original Package file.
Definition: pkgcache.h:361
map_id_t VersionCount
Definition: pkgcache.h:336
void SetArchitectures(map_stringitem_t const idx)
Definition: pkgcache.h:389
map_filesize_t MaxDescFileSize
The maximum size of a raw entry from the original Translation file.
Definition: pkgcache.h:363
map_pointer< ReleaseFile > RlsFileList
index of the first ReleaseFile structure
Definition: pkgcache.h:352
map_id_t PackageCount
Definition: pkgcache.h:335
map_fileid_t VerFileCount
Definition: pkgcache.h:342
map_stringitem_t Architecture
native architecture the cache was built against
Definition: pkgcache.h:357
map_fileid_t PackageFileCount
Definition: pkgcache.h:341
map_id_t DependsCount
Definition: pkgcache.h:338
map_id_t GroupCount
Structure counts.
Definition: pkgcache.h:334
map_pointer< PackageFile > FileList
index of the first PackageFile structure
Definition: pkgcache.h:350
bool Dirty
indicates if the cache should be erased
Definition: pkgcache.h:307
stores information about the files used to generate the cache
Definition: pkgcache.h:552
map_stringitem_t Component
Definition: pkgcache.h:558
map_pointer< PackageFile > NextFile
Link to the next PackageFile in the Cache.
Definition: pkgcache.h:579
map_fileid_t ID
unique sequel ID
Definition: pkgcache.h:581
map_stringitem_t FileName
physical disk file that this PackageFile represents
Definition: pkgcache.h:554
map_flags_t Flags
Definition: pkgcache.h:575
map_stringitem_t IndexType
indicates what sort of index file this is
Definition: pkgcache.h:564
map_stringitem_t Architecture
Definition: pkgcache.h:559
map_pointer< ReleaseFile > Release
the release information
Definition: pkgcache.h:556
stores information about the release files used to generate the cache
Definition: pkgcache.h:508
map_fileid_t ID
unique sequel ID
Definition: pkgcache.h:539
map_stringitem_t Site
The site the index file was fetched from.
Definition: pkgcache.h:521
map_pointer< ReleaseFile > NextFile
Link to the next ReleaseFile in the Cache.
Definition: pkgcache.h:537
map_stringitem_t FileName
physical disk file that this ReleaseFile represents
Definition: pkgcache.h:510
map_flags_t Flags
Definition: pkgcache.h:533