apt  2.2.4
About: Apt (Advanced Package Tool) is a management system for software packages (Debian/Ubuntu). Release series 2.2.
  Fossies Dox: apt-2.2.4.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

mirror.cc
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4 
5  Mirror URI – This method helps avoiding hardcoding of mirrors in the
6  sources.lists by looking up a list of mirrors first to which the
7  following requests are redirected.
8 
9  ##################################################################### */
10  /*}}}*/
11 // Include Files /*{{{*/
12 #include <config.h>
13 
14 #include "aptmethod.h"
15 #include <apt-pkg/configuration.h>
16 #include <apt-pkg/error.h>
17 #include <apt-pkg/fileutl.h>
18 #include <apt-pkg/metaindex.h>
19 #include <apt-pkg/sourcelist.h>
20 #include <apt-pkg/strutl.h>
21 
22 #include <functional>
23 #include <random>
24 #include <string>
25 #include <unordered_map>
26 
27 #include <sys/utsname.h>
28 
29 #include <apti18n.h>
30  /*}}}*/
31 constexpr char const *const disallowLocal[] = {"ftp", "http", "https"};
32 
33 static void sortByLength(std::vector<std::string> &vec) /*{{{*/
34 {
35  // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
36  // by checking for the longest matches first
37  std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
38  return a.length() > b.length();
39  });
40 }
41  /*}}}*/
42 class MirrorMethod : public aptMethod /*{{{*/
43 {
44  std::mt19937 genrng;
45  std::vector<std::string> sourceslist;
46  std::unordered_map<std::string, std::string> msgCache;
48  {
51  AVAILABLE
52  };
53  struct MirrorInfo
54  {
55  std::string uri;
56  unsigned long priority = std::numeric_limits<decltype(priority)>::max();
57  decltype(genrng)::result_type seed = 0;
58  std::unordered_map<std::string, std::vector<std::string>> tags;
59  explicit MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u)
60  {
61  for (auto &&tag : ptags)
62  {
63  auto const colonfound = tag.find(':');
64  if (unlikely(colonfound == std::string::npos))
65  continue;
66  auto name = tag.substr(0, colonfound);
67  auto value = tag.substr(colonfound + 1);
68  if (name == "arch")
69  tags["Architecture"].emplace_back(std::move(value));
70  else if (name == "lang")
71  tags["Language"].emplace_back(std::move(value));
72  else if (name == "priority")
73  priority = std::strtoul(value.c_str(), nullptr, 10);
74  else if (likely(name.empty() == false))
75  {
76  if (name == "codename" || name == "suite")
77  tags["Release"].push_back(value);
78  name[0] = std::toupper(name[0]);
79  tags[std::move(name)].emplace_back(std::move(value));
80  }
81  }
82  }
83  };
85  {
87  std::string baseuri;
88  std::vector<MirrorInfo> list;
89  };
90  std::unordered_map<std::string, MirrorListInfo> mirrorfilestate;
91 
92  virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
93 
94  void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message);
95  bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm);
96  std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm);
97  void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler);
98 
99  public:
100  explicit MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests | SendURIEncoded), genrng(clock())
101  {
103  }
104 };
105  /*}}}*/
106 void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/
107 {
108  std::unordered_map<std::string, std::string> matchers;
109  matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture"));
110  matchers.emplace("Codename", LookupTag(Message, "Target-Codename"));
111  matchers.emplace("Component", LookupTag(Message, "Target-Component"));
112  matchers.emplace("Language", LookupTag(Message, "Target-Language"));
113  matchers.emplace("Release", LookupTag(Message, "Target-Release"));
114  matchers.emplace("Suite", LookupTag(Message, "Target-Suite"));
115  matchers.emplace("Type", LookupTag(Message, "Target-Type"));
116  decltype(info.list) possMirrors;
117  for (auto const &mirror : info.list)
118  {
119  bool failedMatch = false;
120  for (auto const &m : matchers)
121  {
122  if (m.second.empty())
123  continue;
124  auto const tagsetiter = mirror.tags.find(m.first);
125  if (tagsetiter == mirror.tags.end())
126  continue;
127  auto const tagset = tagsetiter->second;
128  if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end())
129  {
130  failedMatch = true;
131  break;
132  }
133  }
134  if (failedMatch)
135  continue;
136  possMirrors.push_back(mirror);
137  }
138  for (auto &&mirror : possMirrors)
139  mirror.seed = genrng();
140  std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) {
141  if (a.priority != b.priority)
142  return a.priority < b.priority;
143  return a.seed < b.seed;
144  });
145  std::string const path = Itm->Uri.substr(info.baseuri.length());
146  std::string altMirrors;
147  std::unordered_map<std::string, std::string> fields;
148  fields.emplace("URI", Queue->Uri);
149  for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror)
150  {
151  std::string mirror = curMirror->uri;
152  if (APT::String::Endswith(mirror, "/") == false)
153  mirror.append("/");
154  mirror.append(path);
155  if (curMirror == possMirrors.cbegin())
156  fields.emplace("New-URI", mirror);
157  else if (altMirrors.empty())
158  altMirrors.append(mirror);
159  else
160  altMirrors.append("\n").append(mirror);
161  }
162  fields.emplace("Alternate-URIs", altMirrors);
163  SendMessage("103 Redirect", std::move(fields));
164  Dequeue();
165 }
166  /*}}}*/
167 void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
168  MirrorListInfo const &info, FetchItem *const Itm,
169  std::function<void()> handler)
170 {
171  FetchItem **LastItm = &Itm->Next;
172  while (*LastItm != nullptr)
173  LastItm = &((*LastItm)->Next);
174  while (Queue != Itm)
175  {
176  if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
177  std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
178  {
179  // move the item behind the aux file not related to it
180  *LastItm = Queue;
181  Queue = QueueBack = Queue->Next;
182  (*LastItm)->Next = nullptr;
183  LastItm = &((*LastItm)->Next);
184  }
185  else
186  {
187  handler();
188  }
189  }
190  // now remove out trigger
191  QueueBack = Queue = Queue->Next;
192  delete Itm;
193 }
194  /*}}}*/
196 {
197  std::vector<std::string> baseuris;
198  for (auto const &i : mirrorfilestate)
199  if (info.baseuri.length() < i.second.baseuri.length() &&
200  i.second.state == REQUESTED &&
201  APT::String::Startswith(i.second.baseuri, info.baseuri))
202  baseuris.push_back(i.second.baseuri);
203  sortByLength(baseuris);
204 
205  FileFd mirrorlist;
206  if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
207  {
208  auto const accessColon = info.baseuri.find(':');
209  auto access = info.baseuri.substr(0, accessColon);
210  std::string prefixAccess;
211  if (APT::String::Startswith(access, "mirror") == false)
212  {
213  auto const plus = info.baseuri.find('+');
214  prefixAccess = info.baseuri.substr(0, plus);
215  access.erase(0, plus + 1);
216  }
217  std::vector<std::string> limitAccess;
218  // If the mirror file comes from an online source, allow only other online
219  // sources, not e.g. file:///. If the mirrorlist comes from there we can assume
220  // the admin knows what (s)he is doing through and not limit the options.
221  if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal),
222  [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) ||
223  access == "mirror")
224  {
225  std::copy(std::begin(disallowLocal), std::end(disallowLocal), std::back_inserter(limitAccess));
226  }
227  std::string line;
228  while (mirrorlist.ReadLine(line))
229  {
230  if (line.empty() || line[0] == '#')
231  continue;
232  auto const access = line.substr(0, line.find(':'));
233  if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end())
234  continue;
235  auto const tab = line.find('\t');
236  if (tab == std::string::npos)
237  {
238  if (prefixAccess.empty())
239  info.list.emplace_back(std::move(line));
240  else
241  info.list.emplace_back(prefixAccess + '+' + line);
242  }
243  else
244  {
245  auto uri = line.substr(0, tab);
246  if (prefixAccess.empty() == false)
247  uri = prefixAccess + '+' + uri;
248  auto tagline = line.substr(tab + 1);
249  std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' ');
250  auto tags = VectorizeString(tagline, ' ');
251  tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end());
252  info.list.emplace_back(std::move(uri), std::move(tags));
253  }
254  }
255  mirrorlist.Close();
256 
257  if (info.list.empty())
258  {
259  info.state = FAILED;
260  DealWithPendingItems(baseuris, info, Itm, [&]() {
261  std::string msg;
262  strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
263  Fail(msg, false);
264  });
265  }
266  else
267  {
268  info.state = AVAILABLE;
269  DealWithPendingItems(baseuris, info, Itm, [&]() {
270  RedirectItem(info, Queue, msgCache[Queue->Uri]);
271  });
272  msgCache.clear();
273  }
274  }
275  else
276  {
277  info.state = FAILED;
278  DealWithPendingItems(baseuris, info, Itm, [&]() {
279  std::string msg;
280  strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
281  Fail(msg, false);
282  });
283  }
284  return true;
285 }
286  /*}}}*/
287 std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
288 {
290  {
291  std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
292  if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
293  sourceslist.push_back(repouri);
294  }
295  if (sourceslist.empty())
296  {
297  // read sources.list and find the matching base uri
298  pkgSourceList sl;
299  if (sl.ReadMainList() == false)
300  {
301  _error->Error(_("The list of sources could not be read."));
302  return "";
303  }
304  std::string const needle = Binary + ":";
305  for (auto const &SL : sl)
306  {
307  std::string uristr = SL->GetURI();
308  if (APT::String::Startswith(uristr, needle))
309  sourceslist.push_back(uristr);
310  }
312  }
313  for (auto uristr : sourceslist)
314  {
315  if (APT::String::Startswith(Itm->Uri, uristr))
316  {
317  uristr.erase(uristr.length() - 1); // remove the ending '/'
318  auto const colon = uristr.find(':');
319  if (unlikely(colon == std::string::npos))
320  continue;
321  auto const plus = uristr.find("+");
322  if (plus < colon)
323  {
324  // started as tor+mirror+http we want to get the file via tor+http
325  auto const access = uristr.substr(0, colon);
326  if (APT::String::Startswith(access, "mirror") == false)
327  {
328  uristr.erase(plus, strlen("mirror") + 1);
329  return uristr;
330  }
331  else
332  return uristr.substr(plus + 1);
333  }
334  else
335  {
336  uristr.replace(0, strlen("mirror"), "http");
337  return uristr;
338  }
339  }
340  }
341  return "";
342 }
343  /*}}}*/
344 bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
345 {
346  auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
347  if (mirrorinfo != mirrorfilestate.end())
348  return MirrorListFileRecieved(mirrorinfo->second, Itm);
349 
350  std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
351  if (mirrorfileuri.empty())
352  {
353  _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
354  return false;
355  }
356  if (DebugEnabled())
357  std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;
358 
359  // have we requested this mirror file already?
360  auto const state = mirrorfilestate.find(mirrorfileuri);
361  if (state == mirrorfilestate.end())
362  {
363  msgCache[Itm->Uri] = Message;
364  MirrorListInfo info;
365  info.state = REQUESTED;
366  info.baseuri = mirrorfileuri + '/';
367  auto const colon = info.baseuri.find(':');
368  if (unlikely(colon == std::string::npos))
369  return false;
370  info.baseuri.replace(0, colon, Binary);
371  mirrorfilestate[mirrorfileuri] = info;
372  std::unordered_map<std::string, std::string> fields;
373  fields.emplace("URI", Itm->Uri);
374  fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
375  fields.emplace("Aux-ShortDesc", "Mirrorlist");
376  fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
377  fields.emplace("Aux-Uri", mirrorfileuri);
378  SendMessage("351 Aux Request", std::move(fields));
379  return true;
380  }
381 
382  switch (state->second.state)
383  {
384  case REQUESTED:
385  // lets wait for the requested mirror file
386  msgCache[Itm->Uri] = Message;
387  return true;
388  case FAILED:
389  Fail("Downloading mirror file failed", false);
390  return true;
391  case AVAILABLE:
392  RedirectItem(state->second, Itm, Message);
393  return true;
394  }
395  return false;
396 }
397  /*}}}*/
398 
399 int main(int, const char *argv[])
400 {
401  return MirrorMethod(flNotDir(argv[0])).Run();
402 }
strprintf(m, msg, repo.c_str())
static char const *const msg
Definition: fileutl.h:39
@ Extension
Definition: fileutl.h:80
@ ReadOnly
Definition: fileutl.h:59
char * ReadLine(char *To, unsigned long long const Size)
Definition: fileutl.cc:2776
bool Open(std::string FileName, unsigned int const Mode, CompressMode Compress, unsigned long const AccessMode=0666)
Definition: fileutl.cc:2415
bool Close()
Definition: fileutl.cc:2977
void DealWithPendingItems(std::vector< std::string > const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function< void()> handler)
Definition: mirror.cc:167
std::unordered_map< std::string, std::string > msgCache
Definition: mirror.cc:46
std::mt19937 genrng
Definition: mirror.cc:44
void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message)
Definition: mirror.cc:106
MirrorMethod(std::string &&pProg)
Definition: mirror.cc:100
bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm)
Definition: mirror.cc:195
virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE
Definition: mirror.cc:344
std::unordered_map< std::string, MirrorListInfo > mirrorfilestate
Definition: mirror.cc:90
std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm)
Definition: mirror.cc:287
std::vector< std::string > sourceslist
Definition: mirror.cc:45
bool DebugEnabled() const
Definition: aptmethod.h:409
unsigned long SeccompFlags
Definition: aptmethod.h:50
void Dequeue()
Definition: aptmethod.h:479
std::string const Binary
Definition: aptmethod.h:49
@ DIRECTORY
Definition: aptmethod.h:55
FetchItem * QueueBack
FetchItem * Queue
void SendMessage(std::string const &header, std::unordered_map< std::string, std::string > &&fields)
int Run(bool Single=false)
void Fail(bool Transient=false)
bool ReadMainList()
Definition: sourcelist.cc:307
string flNotDir(string File)
Definition: fileutl.cc:664
bool FileExists(string File)
Definition: fileutl.cc:326
#define APT_OVERRIDE
Definition: macros.h:111
int main(int, const char *argv[])
Definition: mirror.cc:399
constexpr char const *const disallowLocal[]
Definition: mirror.cc:31
static void sortByLength(std::vector< std::string > &vec)
Definition: mirror.cc:33
bool Endswith(const std::string &s, const std::string &end)
Definition: strutl.cc:77
bool Startswith(const std::string &s, const std::string &start)
Definition: strutl.cc:84
unsigned long priority
Definition: mirror.cc:56
MirrorInfo(std::string const &u, std::vector< std::string > &&ptags={})
Definition: mirror.cc:59
decltype(genrng) ::result_type seed
Definition: mirror.cc:57
std::unordered_map< std::string, std::vector< std::string > > tags
Definition: mirror.cc:58
std::vector< MirrorInfo > list
Definition: mirror.cc:88
MirrorFileState state
Definition: mirror.cc:86
int isspace_ascii(int const c) APT_PURE APT_COLD
Definition: strutl.cc:1517
vector< string > VectorizeString(string const &haystack, char const &split)
Definition: strutl.cc:1308
std::string LookupTag(const std::string &Message, const char *TagC, const char *Default)
Definition: strutl.cc:742