apt  2.2.4
About: Apt (Advanced Package Tool) is a management system for software packages (Debian/Ubuntu). Release series 2.2.
  Fossies Dox: apt-2.2.4.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

basehttp.cc
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4 
5  HTTP and HTTPS share a lot of common code and these classes are
6  exactly the dumping ground for this common code
7 
8  ##################################################################### */
9  /*}}}*/
10 // Include Files /*{{{*/
11 #include <config.h>
12 
13 #include <apt-pkg/configuration.h>
14 #include <apt-pkg/error.h>
15 #include <apt-pkg/fileutl.h>
16 #include <apt-pkg/strutl.h>
17 
18 #include <iostream>
19 #include <limits>
20 #include <map>
21 #include <string>
22 #include <vector>
23 #include <ctype.h>
24 #include <signal.h>
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <sys/stat.h>
28 #include <sys/time.h>
29 #include <time.h>
30 #include <unistd.h>
31 
32 #include "basehttp.h"
33 
34 #include <apti18n.h>
35  /*}}}*/
36 using namespace std;
37 
39 int BaseHttpMethod::FailFd = -1;
40 time_t BaseHttpMethod::FailTime = 0;
41 
42 // Number of successful requests in a pipeline needed to continue
43 // pipelining after a connection reset.
45 
46 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
47 // ---------------------------------------------------------------------
48 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
49  parse error occurred */
51  const std::string &Uri)
52 {
53  Owner->Status(_("Waiting for headers"));
54  do
55  {
56  string Data;
57  if (ReadHeaderLines(Data) == false)
58  continue;
59 
60  if (Owner->Debug == true)
61  clog << "Answer for: " << Uri << endl << Data;
62 
63  for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
64  {
65  string::const_iterator J = I;
66  for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
67  if (Req.HeaderLine(string(I,J)) == false)
69  I = J;
70  }
71 
72  // 100 Continue is a Nop...
73  if (Req.Result == 100)
74  continue;
75 
76  // Tidy up the connection persistence state.
77  if (Req.Encoding == RequestState::Closes && Req.HaveContent == true)
78  Persistent = false;
79 
80  return RUN_HEADERS_OK;
81  } while (LoadNextResponse(false, Req) == ResultState::SUCCESSFUL);
82 
83  return RUN_HEADERS_IO_ERROR;
84 }
85  /*}}}*/
86 bool RequestState::HeaderLine(string const &Line) /*{{{*/
87 {
88  if (Line.empty() == true)
89  return true;
90 
91  if (Result == 0 && Line.size() > 4 && stringcasecmp(Line.data(), Line.data() + 4, "HTTP") == 0)
92  {
93  // Evil servers return no version
94  if (Line[4] == '/')
95  {
96  int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
97  if (elements == 3)
98  {
99  Code[0] = '\0';
100  if (Owner != NULL && Owner->Debug == true)
101  clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl;
102  }
103  else if (elements != 4)
104  return _error->Error(_("The HTTP server sent an invalid reply header"));
105  }
106  else
107  {
108  Major = 0;
109  Minor = 9;
110  if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
111  return _error->Error(_("The HTTP server sent an invalid reply header"));
112  }
113  auto const CodeLen = strlen(Code);
114  auto const CodeEnd = std::remove_if(Code, Code + CodeLen, [](char c) { return isprint(c) == 0; });
115  *CodeEnd = '\0';
116 
117  /* Check the HTTP response header to get the default persistence
118  state. */
119  if (Major < 1)
120  Server->Persistent = false;
121  else
122  {
123  if (Major == 1 && Minor == 0)
124  {
125  Server->Persistent = false;
126  }
127  else
128  {
129  Server->Persistent = true;
130  if (Server->PipelineAllowed)
131  Server->Pipeline = true;
132  }
133  }
134 
135  return true;
136  }
137 
138  // Blah, some servers use "connection:closes", evil.
139  // and some even send empty header fields…
140  string::size_type Pos = Line.find(':');
141  if (Pos == string::npos)
142  return _error->Error(_("Bad header line"));
143  ++Pos;
144 
145  // Parse off any trailing spaces between the : and the next word.
146  string::size_type Pos2 = Pos;
147  while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
148  Pos2++;
149 
150  string const Tag(Line,0,Pos);
151  string const Val(Line,Pos2);
152 
153  if (stringcasecmp(Tag,"Content-Length:") == 0)
154  {
155  auto ContentLength = strtoull(Val.c_str(), NULL, 10);
156  if (ContentLength == 0)
157  return true;
158  if (Encoding == Closes)
159  Encoding = Stream;
160  HaveContent = true;
161 
162  unsigned long long * DownloadSizePtr = &DownloadSize;
163  if (Result == 416 || (Result >= 300 && Result < 400))
164  DownloadSizePtr = &JunkSize;
165 
166  *DownloadSizePtr = ContentLength;
167  if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
168  return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
169  else if (*DownloadSizePtr == 0)
170  HaveContent = false;
171 
172  // On partial content (206) the Content-Length less than the real
173  // size, so do not set it here but leave that to the Content-Range
174  // header instead
175  if(Result != 206 && TotalFileSize == 0)
177 
178  return true;
179  }
180 
181  if (stringcasecmp(Tag,"Content-Type:") == 0)
182  {
183  HaveContent = true;
184  return true;
185  }
186 
187  // The Content-Range field only has a meaning in HTTP/1.1 for the
188  // 206 (Partial Content) and 416 (Range Not Satisfiable) responses
189  // according to RFC7233 "Range Requests", §4.2, so only consider it
190  // for such responses.
191  if ((Result == 416 || Result == 206) && stringcasecmp(Tag,"Content-Range:") == 0)
192  {
193  HaveContent = true;
194 
195  // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
196  if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
197  ; // we got the expected filesize which is all we wanted
198  else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
199  return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
200  if (StartPos > TotalFileSize)
201  return _error->Error(_("This HTTP server has broken range support"));
202 
203  // figure out what we will download
205  return true;
206  }
207 
208  if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
209  {
210  HaveContent = true;
211  if (stringcasecmp(Val,"chunked") == 0)
212  Encoding = Chunked;
213  return true;
214  }
215 
216  if (stringcasecmp(Tag,"Connection:") == 0)
217  {
218  if (stringcasecmp(Val,"close") == 0)
219  {
220  Server->Persistent = false;
221  Server->Pipeline = false;
222  /* Some servers send error pages (as they are dynamically generated)
223  for simplicity via a connection close instead of e.g. chunked,
224  so assuming an always closing server only if we get a file + close */
225  if (Result >= 200 && Result < 300 && Server->PipelineAnswersReceived < PIPELINE_MIN_SUCCESSFUL_ANSWERS_TO_CONTINUE)
226  {
227  Server->PipelineAllowed = false;
229  }
230  }
231  else if (stringcasecmp(Val,"keep-alive") == 0)
232  Server->Persistent = true;
233  return true;
234  }
235 
236  if (stringcasecmp(Tag,"Last-Modified:") == 0)
237  {
238  if (RFC1123StrToTime(Val, Date) == false)
239  return _error->Error(_("Unknown date format"));
240  return true;
241  }
242 
243  if (stringcasecmp(Tag,"Location:") == 0)
244  {
245  Location = Val;
246  return true;
247  }
248 
249  if (stringcasecmp(Tag, "Accept-Ranges:") == 0)
250  {
251  std::string ranges = ',' + Val + ',';
252  ranges.erase(std::remove(ranges.begin(), ranges.end(), ' '), ranges.end());
253  if (ranges.find(",bytes,") == std::string::npos)
254  Server->RangesAllowed = false;
255  return true;
256  }
257 
258  return true;
259 }
260  /*}}}*/
261 // ServerState::ServerState - Constructor /*{{{*/
263  ServerName(Srv), TimeOut(30), Owner(Owner)
264 {
265  Reset();
266 }
267  /*}}}*/
269 {
271  return Server->GetHashes()->AddFD(File, StartPos);
272 }
273  /*}}}*/
274 void ServerState::Reset() /*{{{*/
275 {
276  Persistent = false;
277  Pipeline = false;
278  PipelineAllowed = true;
279  RangesAllowed = true;
281 }
282  /*}}}*/
283 
284 // BaseHttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
285 // ---------------------------------------------------------------------
286 /* We look at the header data we got back from the server and decide what
287  to do. Returns DealWithHeadersResult (see http.h for details).
288  */
289 static std::string fixURIEncoding(std::string const &part)
290 {
291  // if the server sends a space this is not an encoded URI
292  // so other clients seem to encode it and we do it as well
293  if (part.find_first_of(" ") != std::string::npos)
294  return aptMethod::URIEncode(part);
295  return part;
296 }
299 {
300  // Not Modified
301  if (Req.Result == 304)
302  {
303  RemoveFile("server", Queue->DestFile);
304  Res.IMSHit = true;
306  Res.Size = 0;
307  return IMS_HIT;
308  }
309 
310  /* Note that it is only OK for us to treat all redirection the same
311  because we *always* use GET, not other HTTP methods.
312  Codes not mentioned are handled as errors later as required by the
313  HTTP spec to handle unknown codes the same as the x00 code. */
314  constexpr unsigned int RedirectCodes[] = {
315  301, // Moved Permanently
316  302, // Found
317  303, // See Other
318  307, // Temporary Redirect
319  308, // Permanent Redirect
320  };
321  if (AllowRedirect && std::find(std::begin(RedirectCodes), std::end(RedirectCodes), Req.Result) != std::end(RedirectCodes))
322  {
323  if (Req.Location.empty() == true)
324  ;
325  else if (Req.Location[0] == '/' && Queue->Uri.empty() == false)
326  {
327  URI Uri(Queue->Uri);
328  if (Uri.Host.empty() == false)
329  NextURI = URI::SiteOnly(Uri);
330  else
331  NextURI.clear();
332  if (_config->FindB("Acquire::Send-URI-Encoded", false))
333  NextURI.append(fixURIEncoding(Req.Location));
334  else
335  NextURI.append(DeQuoteString(Req.Location));
336  if (Queue->Uri == NextURI)
337  {
338  SetFailReason("RedirectionLoop");
339  _error->Error("Redirection loop encountered");
340  if (Req.HaveContent == true)
342  return ERROR_UNRECOVERABLE;
343  }
344  return TRY_AGAIN_OR_REDIRECT;
345  }
346  else
347  {
348  bool const SendURIEncoded = _config->FindB("Acquire::Send-URI-Encoded", false);
349  if (not SendURIEncoded)
350  Req.Location = DeQuoteString(Req.Location);
351  URI tmpURI(Req.Location);
352  if (SendURIEncoded)
353  tmpURI.Path = fixURIEncoding(tmpURI.Path);
354  if (tmpURI.Access.find('+') != std::string::npos)
355  {
356  _error->Error("Server tried to trick us into using a specific implementation: %s", tmpURI.Access.c_str());
357  if (Req.HaveContent == true)
359  return ERROR_UNRECOVERABLE;
360  }
361  NextURI = tmpURI;
362  URI Uri(Queue->Uri);
363  if (Binary.find('+') != std::string::npos)
364  {
365  auto base = Binary.substr(0, Binary.find('+'));
366  if (base != tmpURI.Access)
367  {
368  tmpURI.Access = base + '+' + tmpURI.Access;
369  if (tmpURI.Access == Binary)
370  {
371  std::swap(tmpURI.Access, Uri.Access);
372  NextURI = tmpURI;
373  std::swap(tmpURI.Access, Uri.Access);
374  }
375  else
376  NextURI = tmpURI;
377  }
378  }
379  if (Queue->Uri == NextURI)
380  {
381  SetFailReason("RedirectionLoop");
382  _error->Error("Redirection loop encountered");
383  if (Req.HaveContent == true)
385  return ERROR_UNRECOVERABLE;
386  }
387  Uri.Access = Binary;
388  // same protocol redirects are okay
389  if (tmpURI.Access == Uri.Access)
390  return TRY_AGAIN_OR_REDIRECT;
391  // as well as http to https
392  else if ((Uri.Access == "http" || Uri.Access == "https+http") && tmpURI.Access == "https")
393  return TRY_AGAIN_OR_REDIRECT;
394  else
395  {
396  auto const tmpplus = tmpURI.Access.find('+');
397  if (tmpplus != std::string::npos && tmpURI.Access.substr(tmpplus + 1) == "https")
398  {
399  auto const uriplus = Uri.Access.find('+');
400  if (uriplus == std::string::npos)
401  {
402  if (Uri.Access == tmpURI.Access.substr(0, tmpplus)) // foo -> foo+https
403  return TRY_AGAIN_OR_REDIRECT;
404  }
405  else if (Uri.Access.substr(uriplus + 1) == "http" &&
406  Uri.Access.substr(0, uriplus) == tmpURI.Access.substr(0, tmpplus)) // foo+http -> foo+https
407  return TRY_AGAIN_OR_REDIRECT;
408  }
409  }
410  _error->Error("Redirection from %s to '%s' is forbidden", Uri.Access.c_str(), NextURI.c_str());
411  }
412  /* else pass through for error message */
413  }
414  // retry after an invalid range response without partial data
415  else if (Req.Result == 416)
416  {
417  struct stat SBuf;
418  if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
419  {
420  bool partialHit = false;
421  if (Queue->ExpectedHashes.usable() == true)
422  {
423  Hashes resultHashes(Queue->ExpectedHashes);
425  Req.TotalFileSize = file.FileSize();
426  Req.Date = file.ModificationTime();
427  resultHashes.AddFD(file);
428  HashStringList const hashList = resultHashes.GetHashStringList();
429  partialHit = (Queue->ExpectedHashes == hashList);
430  }
431  else if ((unsigned long long)SBuf.st_size == Req.TotalFileSize)
432  partialHit = true;
433  if (partialHit == true)
434  {
435  // the file is completely downloaded, but was not moved
436  if (Req.HaveContent == true)
437  {
438  // nuke the sent error page
439  Server->RunDataToDevNull(Req);
440  Req.HaveContent = false;
441  }
442  Req.StartPos = Req.TotalFileSize;
443  Req.Result = 200;
444  }
445  else if (RemoveFile("server", Queue->DestFile))
446  {
447  NextURI = Queue->Uri;
448  return TRY_AGAIN_OR_REDIRECT;
449  }
450  }
451  }
452 
453  /* We have a reply we don't handle. This should indicate a perm server
454  failure */
455  if (Req.Result < 200 || Req.Result >= 300)
456  {
457  if (_error->PendingError() == false)
458  {
459  std::string err;
460  strprintf(err, "HttpError%u", Req.Result);
461  SetFailReason(err);
462  _error->Error("%u %s", Req.Result, Req.Code);
463  }
464  if (Req.HaveContent == true)
466  return ERROR_UNRECOVERABLE;
467  }
468 
469  // This is some sort of 2xx 'data follows' reply
470  Res.LastModified = Req.Date;
471  Res.Size = Req.TotalFileSize;
472  return FILE_IS_OPEN;
473 }
474  /*}}}*/
475 // BaseHttpMethod::SigTerm - Handle a fatal signal /*{{{*/
476 // ---------------------------------------------------------------------
477 /* This closes and timestamps the open file. This is necessary to get
478  resume behavior on user abort */
480 {
481  if (FailFd == -1)
482  _exit(100);
483 
484  struct timeval times[2];
485  times[0].tv_sec = FailTime;
486  times[1].tv_sec = FailTime;
487  times[0].tv_usec = times[1].tv_usec = 0;
488  utimes(FailFile.c_str(), times);
489  close(FailFd);
490 
491  _exit(100);
492 }
493  /*}}}*/
494 // BaseHttpMethod::Fetch - Fetch an item /*{{{*/
495 // ---------------------------------------------------------------------
496 /* This adds an item to the pipeline. We keep the pipeline at a fixed
497  depth. */
499 {
500  if (Server == nullptr || QueueBack == nullptr)
501  return true;
502 
503  // If pipelining is disabled, we only queue 1 request
504  auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0;
505  // how deep is our pipeline currently?
506  decltype(PipelineDepth) CurrentDepth = 0;
507  for (FetchItem const *I = Queue; I != QueueBack; I = I->Next)
508  ++CurrentDepth;
509  if (CurrentDepth > AllowedDepth)
510  return true;
511 
512  do {
513  // Make sure we stick with the same server
514  if (Server->Comp(URI(QueueBack->Uri)) == false)
515  break;
516 
517  bool const UsableHashes = QueueBack->ExpectedHashes.usable();
518  // if we have no hashes, do at most one such request
519  // as we can't fixup pipeling misbehaviors otherwise
520  if (CurrentDepth != 0 && UsableHashes == false)
521  break;
522 
523  if (UsableHashes && FileExists(QueueBack->DestFile))
524  {
527  if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize())
528  {
529  if (wehave.AddFD(partial) &&
531  {
532  FetchResult Res;
533  Res.Filename = QueueBack->DestFile;
535  URIStart(Res);
536  // move item to the start of the queue as URIDone will
537  // always dequeued the first item in the queue
538  if (Queue != QueueBack)
539  {
540  FetchItem *Prev = Queue;
541  for (; Prev->Next != QueueBack; Prev = Prev->Next)
542  /* look for the previous queue item */;
543  Prev->Next = QueueBack->Next;
544  QueueBack->Next = Queue;
545  Queue = QueueBack;
546  QueueBack = Prev->Next;
547  }
548  Res.TakeHashes(wehave);
549  URIDone(Res);
550  continue;
551  }
552  else
553  RemoveFile("Fetch-Partial", QueueBack->DestFile);
554  }
555  }
556  auto const Tmp = QueueBack;
558  SendReq(Tmp);
559  ++CurrentDepth;
560  } while (CurrentDepth <= AllowedDepth && QueueBack != nullptr);
561 
562  return true;
563 }
564  /*}}}*/
565 // BaseHttpMethod::Loop - Main loop /*{{{*/
567 {
568  signal(SIGTERM,SigTerm);
569  signal(SIGINT,SigTerm);
570 
571  Server = 0;
572 
573  int FailCounter = 0;
574  while (1)
575  {
576  // We have no commands, wait for some to arrive
577  if (Queue == 0)
578  {
579  if (WaitFd(STDIN_FILENO) == false)
580  return 0;
581  }
582 
583  /* Run messages, we can accept 0 (no message) if we didn't
584  do a WaitFd above.. Otherwise the FD is closed. */
585  int Result = Run(true);
586  if (Result != -1 && (Result != 0 || Queue == 0))
587  {
588  if(FailReason.empty() == false ||
589  ConfigFindB("DependOnSTDIN", true) == true)
590  return 100;
591  else
592  return 0;
593  }
594 
595  if (Queue == 0)
596  continue;
597 
598  // Connect to the server
599  if (Server == 0 || Server->Comp(URI(Queue->Uri)) == false)
600  {
601  if (!Queue->Proxy().empty())
602  {
603  URI uri(Queue->Uri);
604  _config->Set("Acquire::" + uri.Access + "::proxy::" + uri.Host, Queue->Proxy());
605  }
608  AllowRedirect = ConfigFindB("AllowRedirect", true);
609  PipelineDepth = ConfigFindI("Pipeline-Depth", 10);
610  Debug = DebugEnabled();
611  }
612 
613  /* If the server has explicitly said this is the last connection
614  then we pre-emptively shut down the pipeline and tear down
615  the connection. This will speed up HTTP/1.0 servers a tad
616  since we don't have to wait for the close sequence to
617  complete */
618  if (Server->Persistent == false)
619  Server->Close();
620 
621  // Reset the pipeline
622  if (Server->IsOpen() == false) {
623  QueueBack = Queue;
624  Server->PipelineAnswersReceived = 0;
625  }
626 
627  // Connect to the host
628  switch (Server->Open())
629  {
631  Fail(false);
632  Server = nullptr;
633  continue;
635  Fail(true);
636  Server = nullptr;
637  continue;
639  break;
640  }
641 
642  // Fill the pipeline.
643  Fetch(0);
644 
645  RequestState Req(this, Server.get());
646  // Fetch the next URL header data from the server.
647  switch (Server->RunHeaders(Req, Queue->Uri))
648  {
650  break;
651 
652  // The header data is bad
654  {
655  _error->Error(_("Bad header data"));
656  Fail(true);
657  Server->Close();
658  RotateDNS();
659  continue;
660  }
661 
662  // The server closed a connection during the header get..
663  default:
665  {
666  FailCounter++;
667  _error->Discard();
668  Server->Close();
669  Server->Pipeline = false;
670  Server->PipelineAllowed = false;
671 
672  if (FailCounter >= 2)
673  {
674  Fail(_("Connection failed"),true);
675  FailCounter = 0;
676  }
677 
678  RotateDNS();
679  continue;
680  }
681  };
682 
683  // Decide what to do.
684  FetchResult Res;
685  Res.Filename = Queue->DestFile;
686  switch (DealWithHeaders(Res, Req))
687  {
688  // Ok, the file is Open
689  case FILE_IS_OPEN:
690  {
691  URIStart(Res);
692 
693  // Run the data
695 
696  // ensure we don't fetch too much
697  // we could do "Server->MaximumSize = Queue->MaximumSize" here
698  // but that would break the clever pipeline messup detection
699  // so instead we use the size of the biggest item in the queue
701 
702  if (Req.HaveContent)
703  {
704  /* If the server provides Content-Length we can figure out with it if
705  this satisfies any request we have made so far (in the pipeline).
706  If not we can kill the connection as whatever file the server is trying
707  to send to us would be rejected with a hashsum mismatch later or triggers
708  a maximum size error. We don't run the data to /dev/null as this can be MBs
709  of junk data we would waste bandwidth on and instead just close the connection
710  to reopen a fresh one which should be more cost/time efficient */
711  if (Req.DownloadSize > 0)
712  {
713  decltype(Queue->ExpectedHashes.FileSize()) const filesize = Req.StartPos + Req.DownloadSize;
714  bool found = false;
715  for (FetchItem const *I = Queue; I != 0 && I != QueueBack; I = I->Next)
716  {
717  auto const fs = I->ExpectedHashes.FileSize();
718  if (fs == 0 || fs == filesize)
719  {
720  found = true;
721  break;
722  }
723  }
724  if (found == false)
725  {
726  SetFailReason("MaximumSizeExceeded");
727  _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
728  filesize, Queue->ExpectedHashes.FileSize());
729  Result = ResultState::FATAL_ERROR;
730  }
731  }
732  if (Result == ResultState::SUCCESSFUL)
733  Result = Server->RunData(Req);
734  }
735 
736  /* If the server is sending back sizeless responses then fill in
737  the size now */
738  if (Res.Size == 0)
739  Res.Size = Req.File.Size();
740 
741  // Close the file, destroy the FD object and timestamp it
742  FailFd = -1;
743  Req.File.Close();
744 
745  // Timestamp
746  struct timeval times[2];
747  times[0].tv_sec = times[1].tv_sec = Req.Date;
748  times[0].tv_usec = times[1].tv_usec = 0;
749  utimes(Queue->DestFile.c_str(), times);
750 
751  // Send status to APT
752  if (Result == ResultState::SUCCESSFUL)
753  {
754  Hashes * const resultHashes = Server->GetHashes();
755  HashStringList const hashList = resultHashes->GetHashStringList();
756  if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
757  {
758  // we did not get the expected hash… mhhh:
759  // could it be that server/proxy messed up pipelining?
760  FetchItem * BeforeI = Queue;
761  for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
762  {
763  if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
764  {
765  // yes, he did! Disable pipelining and rewrite queue
766  if (Server->Pipeline == true)
767  {
768  std::string msg;
769  strprintf(msg, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
770  Warning(std::move(msg));
771  Server->Pipeline = false;
772  Server->PipelineAllowed = false;
773  // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
774  }
775  Rename(Res.Filename, I->DestFile);
776  Res.Filename = I->DestFile;
777  BeforeI->Next = I->Next;
778  I->Next = Queue;
779  Queue = I;
780  break;
781  }
782  BeforeI = I;
783  }
784  }
785  if (Server->Pipeline == true)
786  {
787  Server->PipelineAnswersReceived++;
788  }
789  Res.TakeHashes(*resultHashes);
790  URIDone(Res);
791  }
792  else
793  {
794  if (not Server->IsOpen())
795  {
796  // Reset the pipeline
797  QueueBack = Queue;
798  Server->PipelineAnswersReceived = 0;
799  }
800 
801  Server->Close();
802  FailCounter = 0;
803  switch (Result)
804  {
806  Fail(true);
807  break;
810  Fail(false);
811  break;
812  }
813  }
814  break;
815  }
816 
817  // IMS hit
818  case IMS_HIT:
819  {
820  URIDone(Res);
821  break;
822  }
823 
824  // Hard server error, not found or something
825  case ERROR_UNRECOVERABLE:
826  {
827  Fail();
828  break;
829  }
830 
831  // Hard internal error, kill the connection and fail
833  {
834  Fail();
835  RotateDNS();
836  Server->Close();
837  break;
838  }
839 
840  // We need to flush the data, the header is like a 404 w/ error text
842  {
843  Server->RunDataToDevNull(Req);
844  constexpr unsigned int TransientCodes[] = {
845  408, // Request Timeout
846  429, // Too Many Requests
847  500, // Internal Server Error
848  502, // Bad Gateway
849  503, // Service Unavailable
850  504, // Gateway Timeout
851  599, // Network Connect Timeout Error
852  };
853  if (std::find(std::begin(TransientCodes), std::end(TransientCodes), Req.Result) != std::end(TransientCodes))
854  Fail(true);
855  else
856  Fail();
857  break;
858  }
859 
860  // Try again with a new URL
862  {
863  // Clear rest of response if there is content
864  if (Req.HaveContent)
865  Server->RunDataToDevNull(Req);
866  Redirect(NextURI);
867  break;
868  }
869 
870  default:
871  Fail(_("Internal error"));
872  break;
873  }
874 
875  FailCounter = 0;
876  }
877 
878  return 0;
879 }
880  /*}}}*/
881 unsigned long long BaseHttpMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
882 {
883  unsigned long long MaxSizeInQueue = 0;
884  for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
885  {
886  if (I->MaximumSize == 0)
887  return 0;
888  MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
889  }
890  return MaxSizeInQueue;
891 }
892  /*}}}*/
893 BaseHttpMethod::BaseHttpMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) /*{{{*/
894  : aptAuthConfMethod(std::move(Binary), Ver, Flags), Server(nullptr),
895  AllowRedirect(false), Debug(false), PipelineDepth(10)
896 {
897 }
898  /*}}}*/
899 bool BaseHttpMethod::Configuration(std::string Message) /*{{{*/
900 {
901  if (aptAuthConfMethod::Configuration(Message) == false)
902  return false;
903 
904  _config->CndSet("Acquire::tor::Proxy",
905  "socks5h://apt-transport-tor@127.0.0.1:9050");
906  return true;
907 }
908  /*}}}*/
909 bool BaseHttpMethod::AddProxyAuth(URI &Proxy, URI const &Server) /*{{{*/
910 {
912  if (std::find(methodNames.begin(), methodNames.end(), "tor") != methodNames.end() &&
913  Proxy.User == "apt-transport-tor" && Proxy.Password.empty())
914  {
915  std::string pass = Server.Host;
916  pass.erase(std::remove_if(pass.begin(), pass.end(), [](char const c) { return std::isalnum(c) == 0; }), pass.end());
917  if (pass.length() > 255)
918  Proxy.Password = pass.substr(0, 255);
919  else
920  Proxy.Password = std::move(pass);
921  }
922  return true;
923 }
924  /*}}}*/
strprintf(m, msg, repo.c_str())
static bool std::string const metaIndex const *const pkgAcqMetaClearSig *const pkgAcquire::Item *const I
return false
static char const *const msg
ResultState
Definition: aptmethod.h:35
static std::string fixURIEncoding(std::string const &part)
Definition: basehttp.cc:289
constexpr int PIPELINE_MIN_SUCCESSFUL_ANSWERS_TO_CONTINUE
Definition: basehttp.cc:44
BaseHttpMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags)
Definition: basehttp.cc:893
std::string NextURI
Definition: basehttp.h:124
static time_t FailTime
Definition: basehttp.h:157
static APT_NORETURN void SigTerm(int)
Definition: basehttp.cc:479
std::unique_ptr< ServerState > Server
Definition: basehttp.h:123
virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req)
Handle the retrieved header data.
Definition: basehttp.cc:298
DealWithHeadersResult
Result of the header parsing.
Definition: basehttp.h:137
@ ERROR_UNRECOVERABLE
The server reported a unrecoverable error.
Definition: basehttp.h:143
@ FILE_IS_OPEN
The file is open and ready.
Definition: basehttp.h:139
@ ERROR_WITH_CONTENT_PAGE
The server reported a error with a error content page.
Definition: basehttp.h:145
@ ERROR_NOT_FROM_SERVER
An error on the client side.
Definition: basehttp.h:147
@ IMS_HIT
We got a IMS hit, the file has not changed.
Definition: basehttp.h:141
@ TRY_AGAIN_OR_REDIRECT
A redirect or retry request.
Definition: basehttp.h:149
virtual bool Configuration(std::string Message) APT_OVERRIDE
Definition: basehttp.cc:899
virtual void RotateDNS()=0
static int FailFd
Definition: basehttp.h:156
virtual std::unique_ptr< ServerState > CreateServerState(URI const &uri)=0
virtual bool Fetch(FetchItem *) APT_OVERRIDE
Definition: basehttp.cc:498
static std::string FailFile
Definition: basehttp.h:155
bool AddProxyAuth(URI &Proxy, URI const &Server)
Definition: basehttp.cc:909
unsigned long PipelineDepth
Definition: basehttp.h:134
unsigned long long FindMaximumObjectSizeInQueue() const APT_PURE
Definition: basehttp.cc:881
virtual void SendReq(FetchItem *Itm)=0
bool AllowRedirect
Definition: basehttp.h:126
Definition: fileutl.h:39
@ ReadOnly
Definition: fileutl.h:59
unsigned long long FileSize()
Definition: fileutl.cc:2943
bool Truncate(unsigned long long To)
Definition: fileutl.cc:2892
time_t ModificationTime()
Definition: fileutl.cc:2955
unsigned long long Size()
Definition: fileutl.cc:2967
bool Close()
Definition: fileutl.cc:2977
bool usable() const
Definition: hashes.cc:178
unsigned long long FileSize() const
Definition: hashes.cc:210
Definition: hashes.h:170
HashStringList GetHashStringList()
Definition: hashes.cc:431
bool AddFD(int const Fd, unsigned long long Size=0)
Definition: hashes.cc:362
Definition: strutl.h:193
std::string Access
Definition: strutl.h:198
std::string Path
Definition: strutl.h:202
std::string User
Definition: strutl.h:199
std::string Host
Definition: strutl.h:201
std::string Password
Definition: strutl.h:200
static std::string SiteOnly(const std::string &URI)
Definition: strutl.cc:1788
virtual bool Configuration(std::string Message) APT_OVERRIDE
Definition: aptmethod.h:515
bool MaybeAddAuthTo(URI &uri)
Definition: aptmethod.h:551
void Warning(std::string &&msg)
Definition: aptmethod.h:387
bool DebugEnabled() const
Definition: aptmethod.h:409
std::vector< std::string > methodNames
Definition: aptmethod.h:400
bool ConfigFindB(char const *const postfix, bool const defValue) const APT_NONNULL(2)
Definition: aptmethod.h:441
static std::string URIEncode(std::string const &part)
Definition: aptmethod.h:487
void setPostfixForMethodNames(char const *const postfix) APT_NONNULL(2)
Definition: aptmethod.h:401
std::string const Binary
Definition: aptmethod.h:49
struct timeval times[2]
Definition: aptmethod.h:469
int ConfigFindI(char const *const postfix, int const defValue) const APT_NONNULL(2)
Definition: aptmethod.h:445
FetchItem * QueueBack
void Status(const char *Format,...)
void SetFailReason(std::string Msg)
virtual void URIStart(FetchResult &Res)
std::string FailReason
void Redirect(const std::string &NewURI)
FetchItem * Queue
int Run(bool Single=false)
void Fail(bool Transient=false)
virtual void URIDone(FetchResult &Res, FetchResult *Alt=0)
Configuration * _config
bool WaitFd(int Fd, bool write, unsigned long timeout)
Definition: fileutl.cc:819
bool FileExists(string File)
Definition: fileutl.cc:326
bool RemoveFile(char const *const Function, std::string const &FileName)
Definition: fileutl.cc:198
bool Rename(std::string From, std::string To)
Definition: fileutl.cc:3187
unsigned long TimeOut
Definition: ftp.cc:65
URI Proxy
Definition: ftp.cc:66
enum RequestState::@6 Encoding
unsigned int Result
Definition: basehttp.h:34
unsigned long long StartPos
Definition: basehttp.h:44
char Code[360]
Definition: basehttp.h:35
unsigned long long MaximumSize
Definition: basehttp.h:46
unsigned long long TotalFileSize
Definition: basehttp.h:38
FileFd File
Definition: basehttp.h:54
unsigned long long DownloadSize
Definition: basehttp.h:40
unsigned long long JunkSize
Definition: basehttp.h:42
ServerState *const Server
Definition: basehttp.h:57
BaseHttpMethod *const Owner
Definition: basehttp.h:56
bool AddPartialFileToHashes(FileFd &File)
Definition: basehttp.cc:268
time_t Date
Definition: basehttp.h:48
bool HeaderLine(std::string const &Line)
Definition: basehttp.cc:86
unsigned int Major
Definition: basehttp.h:32
std::string Location
Definition: basehttp.h:52
unsigned int Minor
Definition: basehttp.h:33
bool HaveContent
Definition: basehttp.h:49
RunHeadersResult
Result of the header acquire.
Definition: basehttp.h:86
@ RUN_HEADERS_PARSE_ERROR
Parse error after retrieving.
Definition: basehttp.h:92
@ RUN_HEADERS_IO_ERROR
IO error while retrieving.
Definition: basehttp.h:90
@ RUN_HEADERS_OK
Header ok.
Definition: basehttp.h:88
bool Pipeline
Definition: basehttp.h:72
RunHeadersResult RunHeaders(RequestState &Req, const std::string &Uri)
Get the headers before the data.
Definition: basehttp.cc:50
virtual bool ReadHeaderLines(std::string &Data)=0
ServerState(URI Srv, BaseHttpMethod *Owner)
Definition: basehttp.cc:262
virtual void Reset()
Definition: basehttp.cc:274
BaseHttpMethod * Owner
Definition: basehttp.h:78
bool PipelineAllowed
Definition: basehttp.h:68
virtual ResultState LoadNextResponse(bool const ToFile, RequestState &Req)=0
bool RangesAllowed
Definition: basehttp.h:69
bool Persistent
Definition: basehttp.h:67
virtual Hashes * GetHashes()=0
unsigned long PipelineAnswersReceived
Definition: basehttp.h:70
HashStringList ExpectedHashes
unsigned long long Size
void TakeHashes(class Hashes &Hash)
unsigned long long ResumePoint
int stringcasecmp(const char *A, const char *AEnd, const char *B, const char *BEnd)
Definition: strutl.cc:685
int isspace_ascii(int const c) APT_PURE APT_COLD
Definition: strutl.cc:1517
bool RFC1123StrToTime(std::string const &str, time_t &time)
Definition: strutl.cc:1040
string DeQuoteString(const string &Str)
Definition: strutl.cc:404