apt  2.2.4
About: Apt (Advanced Package Tool) is a management system for software packages (Debian/Ubuntu). Release series 2.2.
  Fossies Dox: apt-2.2.4.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

http.cc
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4 
5  HTTP Acquire Method - This is the HTTP acquire method for APT.
6 
7  It uses HTTP/1.1 and many of the fancy options there-in, such as
8  pipelining, range, if-range and so on.
9 
10  It is based on a doubly buffered select loop. A groupe of requests are
11  fed into a single output buffer that is constantly fed out the
12  socket. This provides ideal pipelining as in many cases all of the
13  requests will fit into a single packet. The input socket is buffered
14  the same way and fed into the fd for the file (may be a pipe in future).
15 
16  ##################################################################### */
17  /*}}}*/
18 // Include Files /*{{{*/
19 #include <config.h>
20 
21 #include <apt-pkg/configuration.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/fileutl.h>
24 #include <apt-pkg/hashes.h>
25 #include <apt-pkg/proxy.h>
26 #include <apt-pkg/string_view.h>
27 #include <apt-pkg/strutl.h>
28 
29 #include <chrono>
30 #include <cstring>
31 #include <iostream>
32 #include <sstream>
33 #include <arpa/inet.h>
34 #include <errno.h>
35 #include <signal.h>
36 #include <stddef.h>
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <sys/select.h>
40 #include <sys/stat.h>
41 #include <sys/time.h>
42 #include <unistd.h>
43 
44 #include "config.h"
45 #include "connect.h"
46 #include "http.h"
47 
48 #include <apti18n.h>
49 
50 #ifdef HAVE_SYSTEMD
51 #include <systemd/sd-login.h>
52 #endif
53  /*}}}*/
54 using namespace std;
55 
56 unsigned long long CircleBuf::BwReadLimit=0;
57 unsigned long long CircleBuf::BwTickReadData=0;
58 std::chrono::steady_clock::duration CircleBuf::BwReadTick{0};
59 const unsigned int CircleBuf::BW_HZ=10;
60 
61 // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
62 // ---------------------------------------------------------------------
63 /* */
64 CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
65  : Size(Size), Hash(NULL), TotalWriten(0)
66 {
67  Buf = new unsigned char[Size];
68  Reset();
69 
70  CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
71 }
72  /*}}}*/
73 // CircleBuf::Reset - Reset to the default state /*{{{*/
74 // ---------------------------------------------------------------------
75 /* */
77 {
78  InP = 0;
79  OutP = 0;
80  StrPos = 0;
81  TotalWriten = 0;
82  MaxGet = (unsigned long long)-1;
83  OutQueue = string();
84  if (Hash != NULL)
85  {
86  delete Hash;
87  Hash = NULL;
88  }
89 }
90  /*}}}*/
91 // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
92 // ---------------------------------------------------------------------
93 /* This fills up the buffer with as much data as is in the FD, assuming it
94  is non-blocking.. */
95 bool CircleBuf::Read(std::unique_ptr<MethodFd> const &Fd)
96 {
97  size_t ReadThisCycle = 0;
98  while (1)
99  {
100  // Woops, buffer is full
101  if (InP - OutP == Size)
102  return true;
103 
104  // what's left to read in this tick
105  unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
106 
108  auto const now = std::chrono::steady_clock::now().time_since_epoch();
109  auto const d = now - CircleBuf::BwReadTick;
110 
111  auto const tickLen = std::chrono::microseconds(std::chrono::seconds(1)) / BW_HZ;
112  if(d > tickLen) {
113  CircleBuf::BwReadTick = now;
115  }
116 
117  if(CircleBuf::BwTickReadData >= BwReadMax) {
118  usleep(tickLen.count());
119  return true;
120  }
121  }
122 
123  // Write the buffer segment
124  ssize_t Res;
126  Res = Fd->Read(Buf + (InP % Size),
127  BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
128  } else
129  Res = Fd->Read(Buf + (InP % Size), LeftRead());
130 
131  if(Res > 0 && BwReadLimit > 0)
133 
134  if (Res == 0)
135  return ReadThisCycle != 0;
136  if (Res < 0)
137  {
138  if (errno == EAGAIN)
139  return true;
140  return false;
141  }
142 
143  InP += Res;
144  ReadThisCycle += Res;
145  }
146 }
147  /*}}}*/
148 // CircleBuf::Read - Put the string into the buffer /*{{{*/
149 // ---------------------------------------------------------------------
150 /* This will hold the string in and fill the buffer with it as it empties */
151 bool CircleBuf::Read(string const &Data)
152 {
153  OutQueue.append(Data);
154  FillOut();
155  return true;
156 }
157  /*}}}*/
158 // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
159 // ---------------------------------------------------------------------
160 /* */
162 {
163  if (OutQueue.empty() == true)
164  return;
165  while (1)
166  {
167  // Woops, buffer is full
168  if (InP - OutP == Size)
169  return;
170 
171  // Write the buffer segment
172  unsigned long long Sz = LeftRead();
173  if (OutQueue.length() - StrPos < Sz)
174  Sz = OutQueue.length() - StrPos;
175  memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
176 
177  // Advance
178  StrPos += Sz;
179  InP += Sz;
180  if (OutQueue.length() == StrPos)
181  {
182  StrPos = 0;
183  OutQueue = "";
184  return;
185  }
186  }
187 }
188  /*}}}*/
189 // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
190 // ---------------------------------------------------------------------
191 /* This empties the buffer into the FD. */
192 bool CircleBuf::Write(std::unique_ptr<MethodFd> const &Fd)
193 {
194  while (1)
195  {
196  FillOut();
197 
198  // Woops, buffer is empty
199  if (OutP == InP)
200  return true;
201 
202  if (OutP == MaxGet)
203  return true;
204 
205  // Write the buffer segment
206  ssize_t Res;
207  Res = Fd->Write(Buf + (OutP % Size), LeftWrite());
208 
209  if (Res < 0)
210  {
211  if (errno == EAGAIN)
212  return true;
213 
214  return false;
215  }
216 
217  TotalWriten += Res;
218 
219  if (Hash != NULL)
220  Hash->Add(Buf + (OutP%Size),Res);
221 
222  OutP += Res;
223  }
224 }
225  /*}}}*/
226 // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
227 // ---------------------------------------------------------------------
228 /* This copies till the first empty line */
229 bool CircleBuf::WriteTillEl(string &Data,bool Single)
230 {
231  // We cheat and assume it is unneeded to have more than one buffer load
232  for (unsigned long long I = OutP; I < InP; I++)
233  {
234  if (Buf[I%Size] != '\n')
235  continue;
236  ++I;
237 
238  if (Single == false)
239  {
240  if (I < InP && Buf[I%Size] == '\r')
241  ++I;
242  if (I >= InP || Buf[I%Size] != '\n')
243  continue;
244  ++I;
245  }
246 
247  Data = "";
248  while (OutP < I)
249  {
250  unsigned long long Sz = LeftWrite();
251  if (Sz == 0)
252  return false;
253  if (I - OutP < Sz)
254  Sz = I - OutP;
255  Data += string((char *)(Buf + (OutP%Size)),Sz);
256  OutP += Sz;
257  }
258  return true;
259  }
260  return false;
261 }
262  /*}}}*/
263 // CircleBuf::Write - Write from the buffer to a string /*{{{*/
264 // ---------------------------------------------------------------------
265 /* This copies everything */
266 bool CircleBuf::Write(string &Data)
267 {
268  Data = std::string((char *)Buf + (OutP % Size), LeftWrite());
269  OutP += LeftWrite();
270  return true;
271 }
272  /*}}}*/
274 {
275  delete [] Buf;
276  delete Hash;
277 }
278  /*}}}*/
279 
280 // UnwrapHTTPConnect - Does the HTTP CONNECT handshake /*{{{*/
281 // ---------------------------------------------------------------------
282 /* Performs a TLS handshake on the socket */
283 struct HttpConnectFd : public MethodFd
284 {
285  std::unique_ptr<MethodFd> UnderlyingFd;
286  std::string Buffer;
287 
288  int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); }
289 
290  ssize_t Read(void *buf, size_t count) APT_OVERRIDE
291  {
292  if (!Buffer.empty())
293  {
294  auto read = count < Buffer.size() ? count : Buffer.size();
295 
296  memcpy(buf, Buffer.data(), read);
297  Buffer.erase(Buffer.begin(), Buffer.begin() + read);
298  return read;
299  }
300 
301  return UnderlyingFd->Read(buf, count);
302  }
303  ssize_t Write(void *buf, size_t count) APT_OVERRIDE
304  {
305  return UnderlyingFd->Write(buf, count);
306  }
307 
309  {
310  return UnderlyingFd->Close();
311  }
312 
314  {
315  return !Buffer.empty();
316  }
317 };
318 
319 static ResultState UnwrapHTTPConnect(std::string Host, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd,
320  unsigned long Timeout, aptAuthConfMethod *Owner)
321 {
322  Owner->Status(_("Connecting to %s (%s)"), "HTTP proxy", URI::SiteOnly(Proxy).c_str());
323  // The HTTP server expects a hostname with a trailing :port
324  std::stringstream Req;
325  std::string ProperHost;
326 
327  if (Host.find(':') != std::string::npos)
328  ProperHost = '[' + Host + ']';
329  else
330  ProperHost = Host;
331 
332  // Build the connect
333  Req << "CONNECT " << Host << ":" << std::to_string(Port) << " HTTP/1.1\r\n";
334  if (Proxy.Port != 0)
335  Req << "Host: " << ProperHost << ":" << std::to_string(Port) << "\r\n";
336  else
337  Req << "Host: " << ProperHost << "\r\n";
338 
339  Owner->MaybeAddAuthTo(Proxy);
340  if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
341  Req << "Proxy-Authorization: Basic "
342  << Base64Encode(Proxy.User + ":" + Proxy.Password) << "\r\n";
343 
344  Req << "User-Agent: " << Owner->ConfigFind("User-Agent", "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
345 
346  Req << "\r\n";
347 
348  CircleBuf In(dynamic_cast<HttpMethod *>(Owner), 4096);
349  CircleBuf Out(dynamic_cast<HttpMethod *>(Owner), 4096);
350  std::string Headers;
351 
352  if (Owner->DebugEnabled() == true)
353  cerr << Req.str() << endl;
354  Out.Read(Req.str());
355 
356  // Writing from proxy
357  while (Out.WriteSpace())
358  {
359  if (WaitFd(Fd->Fd(), true, Timeout) == false)
360  {
361  _error->Errno("select", "Writing to proxy failed");
363  }
364  if (Out.Write(Fd) == false)
365  {
366  _error->Errno("write", "Writing to proxy failed");
368  }
369  }
370 
371  while (In.ReadSpace())
372  {
373  if (WaitFd(Fd->Fd(), false, Timeout) == false)
374  {
375  _error->Errno("select", "Reading from proxy failed");
377  }
378  if (In.Read(Fd) == false)
379  {
380  _error->Errno("read", "Reading from proxy failed");
382  }
383 
384  if (In.WriteTillEl(Headers))
385  break;
386  }
387 
388  if (Owner->DebugEnabled() == true)
389  cerr << Headers << endl;
390 
391  if (!(APT::String::Startswith(Headers, "HTTP/1.0 200") || APT::String::Startswith(Headers, "HTTP/1.1 200")))
392  {
393  _error->Error("Invalid response from proxy: %s", Headers.c_str());
395  }
396 
397  if (In.WriteSpace())
398  {
399  // Maybe there is actual data already read, if so we need to buffer it
400  std::unique_ptr<HttpConnectFd> NewFd(new HttpConnectFd());
401  In.Write(NewFd->Buffer);
402  NewFd->UnderlyingFd = std::move(Fd);
403  Fd = std::move(NewFd);
404  }
405 
407 }
408  /*}}}*/
409 
410 // HttpServerState::HttpServerState - Constructor /*{{{*/
411 HttpServerState::HttpServerState(URI Srv, HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, APT_BUFFER_SIZE), Out(Owner, 4 * 1024)
412 {
413  TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
415  Reset();
416 }
417  /*}}}*/
418 // HttpServerState::Open - Open a connection to the server /*{{{*/
419 // ---------------------------------------------------------------------
420 /* This opens a connection to the server. */
422 {
423  // Use the already open connection if possible.
424  if (ServerFd->Fd() != -1)
426 
427  Close();
428  In.Reset();
429  Out.Reset();
430  Persistent = true;
431 
432  bool tls = (ServerName.Access == "https" || APT::String::Endswith(ServerName.Access, "+https"));
433 
434  // Determine the proxy setting
435  // Used to run AutoDetectProxy(ServerName) here, but we now send a Proxy
436  // header in the URI Acquire request and set "Acquire::"+uri.Access+"::proxy::"+uri.Host
437  // to it in BaseHttpMethod::Loop()
438  string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
439  if (!SpecificProxy.empty())
440  {
441  if (SpecificProxy == "DIRECT")
442  Proxy = "";
443  else
444  Proxy = SpecificProxy;
445  }
446  else
447  {
448  string DefProxy = Owner->ConfigFind("Proxy", "");
449  if (!DefProxy.empty())
450  {
451  Proxy = DefProxy;
452  }
453  else
454  {
455  char *result = getenv("http_proxy");
456  Proxy = result ? result : "";
457  if (tls == true)
458  {
459  char *result = getenv("https_proxy");
460  if (result != nullptr)
461  {
462  Proxy = result;
463  }
464  }
465  }
466  }
467 
468  // Parse no_proxy, a , separated list of domains
469  if (getenv("no_proxy") != 0)
470  {
471  if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
472  Proxy = "";
473  }
474 
475  if (Proxy.empty() == false)
477 
478  auto const DefaultService = tls ? "https" : "http";
479  auto const DefaultPort = tls ? 443 : 80;
480  if (Proxy.Access == "socks5h")
481  {
482  auto result = Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner);
483  if (result != ResultState::SUCCESSFUL)
484  return result;
485 
486  result = UnwrapSocks(ServerName.Host, ServerName.Port == 0 ? DefaultPort : ServerName.Port,
487  Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 30), Owner);
488  if (result != ResultState::SUCCESSFUL)
489  return result;
490  }
491  else
492  {
493  // Determine what host and port to use based on the proxy settings
494  int Port = 0;
495  string Host;
496  if (Proxy.empty() == true || Proxy.Host.empty() == true)
497  {
498  if (ServerName.Port != 0)
499  Port = ServerName.Port;
500  Host = ServerName.Host;
501  }
502  else if (Proxy.Access != "http" && Proxy.Access != "https")
503  {
504  _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
506  }
507  else
508  {
509  if (Proxy.Port != 0)
510  Port = Proxy.Port;
511  Host = Proxy.Host;
512 
513  if (Proxy.Access == "https" && Port == 0)
514  Port = 443;
515  }
516  auto result = Connect(Host, Port, DefaultService, DefaultPort, ServerFd, TimeOut, Owner);
517  if (result != ResultState::SUCCESSFUL)
518  return result;
519  if (Host == Proxy.Host && Proxy.Access == "https")
520  {
521  result = UnwrapTLS(Proxy.Host, ServerFd, TimeOut, Owner);
522  if (result != ResultState::SUCCESSFUL)
523  return result;
524  }
525  if (Host == Proxy.Host && tls)
526  {
527  result = UnwrapHTTPConnect(ServerName.Host, ServerName.Port == 0 ? DefaultPort : ServerName.Port, Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 30), Owner);
528  if (result != ResultState::SUCCESSFUL)
529  return result;
530  }
531  }
532 
533  if (tls)
535 
537 }
538  /*}}}*/
539 // HttpServerState::Close - Close a connection to the server /*{{{*/
540 // ---------------------------------------------------------------------
541 /* */
543 {
544  ServerFd->Close();
545  return true;
546 }
547  /*}}}*/
548 // HttpServerState::RunData - Transfer the data from the socket /*{{{*/
550 {
552 
553  // Chunked transfer encoding is fun..
554  if (Req.Encoding == RequestState::Chunked)
555  {
556  while (1)
557  {
558  // Grab the block size
560  string Data;
561  In.Limit(-1);
562  do
563  {
564  if (In.WriteTillEl(Data,true) == true)
565  break;
566  } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL);
567 
568  if (Last != ResultState::SUCCESSFUL)
569  return Last;
570 
571  // See if we are done
572  unsigned long long Len = strtoull(Data.c_str(),0,16);
573  if (Len == 0)
574  {
575  In.Limit(-1);
576 
577  // We have to remove the entity trailer
579  do
580  {
581  if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
582  break;
583  } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL);
584  return Last;
585  }
586 
587  // Transfer the block
588  In.Limit(Len);
589  while (Go(true, Req) == ResultState::SUCCESSFUL)
590  if (In.IsLimit() == true)
591  break;
592 
593  // Error
594  if (In.IsLimit() == false)
596 
597  // The server sends an extra new line before the next block specifier..
598  In.Limit(-1);
600  do
601  {
602  if (In.WriteTillEl(Data,true) == true)
603  break;
604  } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL);
605  if (Last != ResultState::SUCCESSFUL)
606  return Last;
607  }
608  }
609  else
610  {
611  /* Closes encoding is used when the server did not specify a size, the
612  loss of the connection means we are done */
613  if (Req.JunkSize != 0)
614  In.Limit(Req.JunkSize);
615  else if (Req.DownloadSize != 0)
616  {
617  if (Req.MaximumSize != 0 && Req.DownloadSize > Req.MaximumSize)
618  {
619  Owner->SetFailReason("MaximumSizeExceeded");
620  _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
621  Req.DownloadSize, Req.MaximumSize);
623  }
624  In.Limit(Req.DownloadSize);
625  }
626  else if (Persistent == false)
627  In.Limit(-1);
628 
629  // Just transfer the whole block.
630  while (true)
631  {
632  if (In.IsLimit() == false)
633  {
634  auto const result = Go(true, Req);
635  if (result == ResultState::SUCCESSFUL)
636  continue;
637  return result;
638  }
639 
640  In.Limit(-1);
641  return _error->PendingError() ? ResultState::FATAL_ERROR : ResultState::SUCCESSFUL;
642  }
643  }
644 
645  if (Flush(&Req.File) == false)
648 }
649  /*}}}*/
651 {
652  // no need to clean up if we discard the connection anyhow
653  if (Persistent == false)
655  Req.File.Open("/dev/null", FileFd::WriteOnly);
656  return RunData(Req);
657 }
658  /*}}}*/
659 bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
660 {
661  return In.WriteTillEl(Data);
662 }
663  /*}}}*/
665 {
666  return Go(ToFile, Req);
667 }
668  /*}}}*/
669 bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
670 {
671  return Out.Read(Data);
672 }
673  /*}}}*/
675 {
676  return (ServerFd->Fd() != -1);
677 }
678  /*}}}*/
679 bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
680 {
681  delete In.Hash;
682  In.Hash = new Hashes(ExpectedHashes);
683  return true;
684 }
685  /*}}}*/
687 {
689  ServerFd->Close();
690 }
691  /*}}}*/
692 
694 {
695  return In.Hash;
696 }
697  /*}}}*/
698 // HttpServerState::Die - The server has closed the connection. /*{{{*/
700 {
701  unsigned int LErrno = errno;
702 
703  Close();
704 
705  switch (Req.State)
706  {
707  case RequestState::Data:
708  // We have read all data we could, or the connection is not persistent
709  if (In.IsLimit() == true || Persistent == false)
711  break;
713  In.Limit(-1);
714  // We have read some headers, but we might also have read the content
715  // and an EOF and hence reached this point. This is fine.
716  if (In.WriteSpace())
718  break;
719  }
720 
721  // We have reached an actual error, tell the user about it.
722  if (LErrno == 0)
723  {
724  _error->Error(_("Error reading from server. Remote end closed connection"));
726  }
727  errno = LErrno;
728  _error->Errno("read", _("Error reading from server"));
729 
731 }
732  /*}}}*/
733 // HttpServerState::Flush - Dump the buffer into the file /*{{{*/
734 // ---------------------------------------------------------------------
735 /* This takes the current input buffer from the Server FD and writes it
736  into the file */
737 bool HttpServerState::Flush(FileFd *const File, bool MustComplete)
738 {
739  if (File != nullptr)
740  {
741  if (In.WriteSpace() == false)
742  return true;
743 
744  while (In.WriteSpace() == true)
745  {
746  if (In.Write(MethodFd::FromFd(File->Fd())) == false)
747  return _error->Errno("write",_("Error writing to file"));
748  if (In.IsLimit() == true)
749  return true;
750  }
751 
752  if (In.IsLimit() == true || Persistent == false || not MustComplete)
753  return true;
754  }
755  return false;
756 }
757  /*}}}*/
758 // HttpServerState::Go - Run a single loop /*{{{*/
759 // ---------------------------------------------------------------------
760 /* This runs the select loop over the server FDs, Output file FDs and
761  stdin. */
763 {
764  // Server has closed the connection
765  if (ServerFd->Fd() == -1 && (In.WriteSpace() == false ||
766  ToFile == false))
768 
769  // Record if we have data pending to read in the server, so that we can
770  // skip the wait in select(). This can happen if data has already been
771  // read into a methodfd's buffer - the TCP queue might be empty at that
772  // point.
773  bool ServerPending = ServerFd->HasPending();
774 
775  fd_set rfds,wfds;
776  FD_ZERO(&rfds);
777  FD_ZERO(&wfds);
778 
779  /* Add the server. We only send more requests if the connection will
780  be persisting */
781  if (Out.WriteSpace() == true && ServerFd->Fd() != -1 && Persistent == true)
782  FD_SET(ServerFd->Fd(), &wfds);
783  if (In.ReadSpace() == true && ServerFd->Fd() != -1)
784  FD_SET(ServerFd->Fd(), &rfds);
785 
786  // Add the file. Note that we need to add the file to the select and
787  // then write before we read from the server so we do not have content
788  // left to write if the server closes the connection when we read from it.
789  //
790  // An alternative would be to just flush the file in those circumstances
791  // and then return. Because otherwise we might end up blocking indefinitely
792  // in the select() call if we were to continue but all that was left to do
793  // was write to the local file.
794  if (In.WriteSpace() == true && ToFile == true && Req.File.IsOpen())
795  FD_SET(Req.File.Fd(), &wfds);
796 
797  // Add stdin
798  if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
799  FD_SET(STDIN_FILENO,&rfds);
800 
801  // Figure out the max fd
802  int MaxFd = Req.File.Fd();
803  if (MaxFd < ServerFd->Fd())
804  MaxFd = ServerFd->Fd();
805 
806  // Select
807  struct timeval tv;
808  tv.tv_sec = ServerPending ? 0 : TimeOut;
809  tv.tv_usec = 0;
810  int Res = 0;
811  if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
812  {
813  if (errno == EINTR)
815  _error->Errno("select", _("Select failed"));
817  }
818 
819  if (Res == 0 && not ServerPending)
820  {
821  _error->Error(_("Connection timed out"));
823  }
824 
825  // Flush any data before talking to the server, in case the server
826  // closed the connection, we want to be done writing.
827  if (Req.File.IsOpen() && FD_ISSET(Req.File.Fd(), &wfds))
828  {
829  if (not Flush(&Req.File, false))
831  }
832 
833  // Handle server IO
834  if (ServerPending || (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &rfds)))
835  {
836  errno = 0;
837  if (In.Read(ServerFd) == false)
838  return Die(Req);
839  }
840 
841  // Send data to the file
842  if (In.WriteSpace() == true && ToFile == true && Req.File.IsOpen())
843  {
844  if (not Flush(&Req.File, false))
846  }
847 
848  if (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &wfds))
849  {
850  errno = 0;
851  if (Out.Write(ServerFd) == false)
852  return Die(Req);
853  }
854 
855  if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize)
856  {
857  Owner->SetFailReason("MaximumSizeExceeded");
858  _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
859  Req.File.Tell(), Req.MaximumSize);
861  }
862 
863  // Handle commands from APT
864  if (FD_ISSET(STDIN_FILENO,&rfds))
865  {
866  if (Owner->Run(true) != -1)
867  exit(100);
868  }
869 
871 }
872  /*}}}*/
873 
874 // HttpMethod::SendReq - Send the HTTP request /*{{{*/
875 // ---------------------------------------------------------------------
876 /* This places the http request in the outbound buffer */
878 {
879  URI Uri(Itm->Uri);
880  {
881  auto const plus = Binary.find('+');
882  if (plus != std::string::npos)
883  Uri.Access = Binary.substr(plus + 1);
884  }
885 
886  // The HTTP server expects a hostname with a trailing :port
887  std::stringstream Req;
888  string ProperHost;
889 
890  if (Uri.Host.find(':') != string::npos)
891  ProperHost = '[' + Uri.Host + ']';
892  else
893  ProperHost = Uri.Host;
894 
895  /* RFC 2616 ยง5.1.2 requires absolute URIs for requests to proxies,
896  but while its a must for all servers to accept absolute URIs,
897  it is assumed clients will sent an absolute path for non-proxies */
898  std::string requesturi;
899  if ((Server->Proxy.Access != "http" && Server->Proxy.Access != "https") || APT::String::Endswith(Uri.Access, "https") || Server->Proxy.empty() == true || Server->Proxy.Host.empty())
900  requesturi = Uri.Path;
901  else
902  requesturi = Uri;
903 
904  if (not _config->FindB("Acquire::Send-URI-Encoded", false))
905  requesturi = URIEncode(requesturi);
906 
907  /* Build the request. No keep-alive is included as it is the default
908  in 1.1, can cause problems with proxies, and we are an HTTP/1.1
909  client anyway.
910  C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
911  Req << "GET " << requesturi << " HTTP/1.1\r\n";
912  if (Uri.Port != 0)
913  Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
914  else
915  Req << "Host: " << ProperHost << "\r\n";
916 
917  // generate a cache control header (if needed)
918  if (ConfigFindB("No-Cache",false) == true)
919  Req << "Cache-Control: no-cache\r\n"
920  << "Pragma: no-cache\r\n";
921  else if (Itm->IndexFile == true)
922  Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
923  else if (ConfigFindB("No-Store", false) == true)
924  Req << "Cache-Control: no-store\r\n";
925 
926  // If we ask for uncompressed files servers might respond with content-
927  // negotiation which lets us end up with compressed files we do not support,
928  // see 657029, 657560 and co, so if we have no extension on the request
929  // ask for text only. As a sidenote: If there is nothing to negotate servers
930  // seem to be nice and ignore it.
931  if (ConfigFindB("SendAccept", true) == true)
932  {
933  size_t const filepos = Itm->Uri.find_last_of('/');
934  string const file = Itm->Uri.substr(filepos + 1);
935  if (flExtension(file) == file)
936  Req << "Accept: text/*\r\n";
937  }
938 
939  // Check for a partial file and send if-queries accordingly
940  struct stat SBuf;
941  if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
942  Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
943  << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
944  else if (Itm->LastModified != 0)
945  Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
946 
947  if ((Server->Proxy.Access == "http" || Server->Proxy.Access == "https") &&
948  (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false))
949  Req << "Proxy-Authorization: Basic "
950  << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
951 
952  MaybeAddAuthTo(Uri);
953  if (Uri.User.empty() == false || Uri.Password.empty() == false)
954  Req << "Authorization: Basic "
955  << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
956 
957  Req << "User-Agent: " << ConfigFind("User-Agent",
958  "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")");
959 
960 #ifdef HAVE_SYSTEMD
961  if (ConfigFindB("User-Agent-Non-Interactive", false))
962  {
963  using APT::operator""_sv;
964  char *unit = nullptr;
965  sd_pid_get_unit(getpid(), &unit);
966  if (unit != nullptr && *unit != '\0' && not APT::String::Startswith(unit, "user@") // user@ _is_ interactive
967  && "packagekit.service"_sv != unit // packagekit likely is interactive
968  && "dbus.service"_sv != unit) // aptdaemon and qapt don't have systemd services
969  Req << " non-interactive";
970 
971  free(unit);
972  }
973 #endif
974 
975  Req << "\r\n";
976 
977  // the famously typoed HTTP header field
978  auto const referrer = ConfigFind("Referer", "");
979  if (referrer.empty() == false)
980  Req << "Referer: " << referrer << "\r\n";
981 
982  Req << "\r\n";
983 
984  if (Debug == true)
985  cerr << Req.str() << endl;
986 
987  Server->WriteResponse(Req.str());
988 }
989  /*}}}*/
990 std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
991 {
992  return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
993 }
994  /*}}}*/
995 void HttpMethod::RotateDNS() /*{{{*/
996 {
997  ::RotateDNS();
998 }
999  /*}}}*/
1001 {
1002  auto ret = BaseHttpMethod::DealWithHeaders(Res, Req);
1003  if (ret != BaseHttpMethod::FILE_IS_OPEN)
1004  return ret;
1005  if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false)
1006  return ERROR_NOT_FROM_SERVER;
1007 
1008  FailFile = Queue->DestFile;
1009  FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
1010  FailFd = Req.File.Fd();
1011  FailTime = Req.Date;
1012 
1013  if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false)
1014  {
1015  _error->Errno("read",_("Problem hashing file"));
1016  return ERROR_NOT_FROM_SERVER;
1017  }
1018  if (Req.StartPos > 0)
1019  Res.ResumePoint = Req.StartPos;
1020 
1021  return FILE_IS_OPEN;
1022 }
1023  /*}}}*/
1024 HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig | SendURIEncoded) /*{{{*/
1025 {
1027 
1028  auto addName = std::inserter(methodNames, methodNames.begin());
1029  if (Binary != "http")
1030  addName = "http";
1031  auto const plus = Binary.find('+');
1032  if (plus != std::string::npos)
1033  {
1034  auto name2 = Binary.substr(plus + 1);
1035  if (std::find(methodNames.begin(), methodNames.end(), name2) == methodNames.end())
1036  addName = std::move(name2);
1037  addName = Binary.substr(0, plus);
1038  }
1039 }
1040  /*}}}*/
1041 
1042 int main(int, const char *argv[])
1043 {
1044  // ignore SIGPIPE, this can happen on write() if the socket
1045  // closes the connection (this is dealt with via ServerDie())
1046  signal(SIGPIPE, SIG_IGN);
1047  std::string Binary = flNotDir(argv[0]);
1048  if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http")
1049  Binary.append("+http");
1050  return HttpMethod(std::move(Binary)).Loop();
1051 }
static bool std::string const metaIndex const *const pkgAcqMetaClearSig *const pkgAcquire::Item *const I
ResultState
Definition: aptmethod.h:35
static time_t FailTime
Definition: basehttp.h:157
std::unique_ptr< ServerState > Server
Definition: basehttp.h:123
virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req)
Handle the retrieved header data.
Definition: basehttp.cc:298
DealWithHeadersResult
Result of the header parsing.
Definition: basehttp.h:137
@ FILE_IS_OPEN
The file is open and ready.
Definition: basehttp.h:139
@ ERROR_NOT_FROM_SERVER
An error on the client side.
Definition: basehttp.h:147
static int FailFd
Definition: basehttp.h:156
static std::string FailFile
Definition: basehttp.h:155
bool AddProxyAuth(URI &Proxy, URI const &Server)
Definition: basehttp.cc:909
Definition: http.h:32
unsigned long long StrPos
Definition: http.h:38
void FillOut()
Definition: http.cc:161
bool Read(std::unique_ptr< MethodFd > const &Fd)
Definition: http.cc:95
static std::chrono::steady_clock::duration BwReadTick
Definition: http.h:43
unsigned long long OutP
Definition: http.h:36
void Limit(long long Max)
Definition: http.h:79
CircleBuf(HttpMethod const *const Owner, unsigned long long Size)
Definition: http.cc:64
unsigned long long Size
Definition: http.h:34
std::string OutQueue
Definition: http.h:37
unsigned long long TotalWriten
Definition: http.h:67
void Reset()
Definition: http.cc:76
unsigned char * Buf
Definition: http.h:33
~CircleBuf()
Definition: http.cc:273
bool ReadSpace() const
Definition: http.h:84
unsigned long long LeftRead() const
Definition: http.h:46
Hashes * Hash
Definition: http.h:65
static unsigned long long BwReadLimit
Definition: http.h:41
bool WriteTillEl(std::string &Data, bool Single=false)
Definition: http.cc:229
bool IsLimit() const
Definition: http.h:80
static const unsigned int BW_HZ
Definition: http.h:44
bool WriteSpace() const
Definition: http.h:85
unsigned long long LeftWrite() const
Definition: http.h:53
static unsigned long long BwTickReadData
Definition: http.h:42
unsigned long long InP
Definition: http.h:35
unsigned long long MaxGet
Definition: http.h:39
bool Write(std::unique_ptr< MethodFd > const &Fd)
Definition: http.cc:192
Definition: fileutl.h:39
bool IsOpen()
Definition: fileutl.h:150
@ WriteOnly
Definition: fileutl.h:60
@ WriteAny
Definition: fileutl.h:71
unsigned long long Tell()
Definition: fileutl.cc:2905
bool Failed()
Definition: fileutl.h:151
bool Open(std::string FileName, unsigned int const Mode, CompressMode Compress, unsigned long const AccessMode=0666)
Definition: fileutl.cc:2415
int Fd()
Definition: fileutl.h:147
Definition: hashes.h:170
bool Add(const unsigned char *const Data, unsigned long long const Size) APT_NONNULL(2)
Definition: hashes.cc:353
virtual void SendReq(FetchItem *Itm) APT_OVERRIDE
Definition: http.cc:877
HttpMethod(std::string &&pProg)
Definition: http.cc:1024
virtual void RotateDNS() APT_OVERRIDE
Definition: http.cc:995
virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req) APT_OVERRIDE
Handle the retrieved header data.
Definition: http.cc:1000
friend struct HttpServerState
Definition: http.h:137
virtual std::unique_ptr< ServerState > CreateServerState(URI const &uri) APT_OVERRIDE
Definition: http.cc:990
Definition: strutl.h:193
std::string Access
Definition: strutl.h:198
std::string Path
Definition: strutl.h:202
bool empty()
Definition: strutl.h:207
unsigned int Port
Definition: strutl.h:203
std::string User
Definition: strutl.h:199
std::string Host
Definition: strutl.h:201
std::string Password
Definition: strutl.h:200
static std::string SiteOnly(const std::string &URI)
Definition: strutl.cc:1788
bool MaybeAddAuthTo(URI &uri)
Definition: aptmethod.h:551
std::string ConfigFind(char const *const postfix, std::string const &defValue) const APT_NONNULL(2)
Definition: aptmethod.h:425
bool DebugEnabled() const
Definition: aptmethod.h:409
unsigned long SeccompFlags
Definition: aptmethod.h:50
std::vector< std::string > methodNames
Definition: aptmethod.h:400
bool ConfigFindB(char const *const postfix, bool const defValue) const APT_NONNULL(2)
Definition: aptmethod.h:441
static std::string URIEncode(std::string const &part)
Definition: aptmethod.h:487
std::string const Binary
Definition: aptmethod.h:49
int ConfigFindI(char const *const postfix, int const defValue) const APT_NONNULL(2)
Definition: aptmethod.h:445
void Status(const char *Format,...)
void SetFailReason(std::string Msg)
FetchItem * Queue
int Run(bool Single=false)
Configuration * _config
ResultState UnwrapSocks(std::string Host, int Port, URI Proxy, std::unique_ptr< MethodFd > &Fd, unsigned long Timeout, aptMethod *Owner)
Definition: connect.cc:569
ResultState Connect(std::string Host, int Port, const char *Service, int DefPort, std::unique_ptr< MethodFd > &Fd, unsigned long TimeOut, aptMethod *Owner)
Definition: connect.cc:476
ResultState UnwrapTLS(std::string const &Host, std::unique_ptr< MethodFd > &Fd, unsigned long Timeout, aptMethod *Owner)
Definition: connect.cc:896
string flExtension(string File)
Definition: fileutl.cc:688
string flNotDir(string File)
Definition: fileutl.cc:664
bool WaitFd(int Fd, bool write, unsigned long timeout)
Definition: fileutl.cc:819
URI Proxy
Definition: ftp.cc:66
int main(int, const char *argv[])
Definition: http.cc:1042
static ResultState UnwrapHTTPConnect(std::string Host, int Port, URI Proxy, std::unique_ptr< MethodFd > &Fd, unsigned long Timeout, aptAuthConfMethod *Owner)
Definition: http.cc:319
#define APT_OVERRIDE
Definition: macros.h:111
#define APT_PURE
Definition: macros.h:56
static constexpr unsigned long long APT_BUFFER_SIZE
Definition: macros.h:125
bool Endswith(const std::string &s, const std::string &end)
Definition: strutl.cc:77
bool Startswith(const std::string &s, const std::string &start)
Definition: strutl.cc:84
ssize_t Write(void *buf, size_t count) APT_OVERRIDE
Should behave like write(2)
Definition: http.cc:303
int Fd() APT_OVERRIDE
Returns -1 for unusable, or an fd to select() on otherwise.
Definition: http.cc:288
int Close() APT_OVERRIDE
Closes the file descriptor. Can be called multiple times.
Definition: http.cc:308
bool HasPending() APT_OVERRIDE
If there is pending data.
Definition: http.cc:313
std::unique_ptr< MethodFd > UnderlyingFd
Definition: http.cc:285
std::string Buffer
Definition: http.cc:286
ssize_t Read(void *buf, size_t count) APT_OVERRIDE
Should behave like read(2)
Definition: http.cc:290
virtual Hashes * GetHashes() APT_OVERRIDE
Definition: http.cc:693
CircleBuf Out
Definition: http.h:97
virtual ResultState RunData(RequestState &Req) APT_OVERRIDE
Transfer the data from the socket.
Definition: http.cc:549
virtual ResultState Die(RequestState &Req) APT_OVERRIDE
Definition: http.cc:699
virtual ResultState Open() APT_OVERRIDE
Definition: http.cc:421
virtual bool Close() APT_OVERRIDE
Definition: http.cc:542
std::unique_ptr< MethodFd > ServerFd
Definition: http.h:98
virtual bool ReadHeaderLines(std::string &Data) APT_OVERRIDE
Definition: http.cc:659
CircleBuf In
Definition: http.h:96
virtual void Reset() APT_OVERRIDE
Definition: http.cc:686
virtual bool Flush(FileFd *const File, bool MustComplete=true) APT_OVERRIDE
Definition: http.cc:737
HttpServerState(URI Srv, HttpMethod *Owner)
Definition: http.cc:411
virtual bool IsOpen() APT_OVERRIDE
Definition: http.cc:674
virtual bool WriteResponse(std::string const &Data) APT_OVERRIDE
Definition: http.cc:669
virtual ResultState LoadNextResponse(bool const ToFile, RequestState &Req) APT_OVERRIDE
Definition: http.cc:664
virtual bool InitHashes(HashStringList const &ExpectedHashes) APT_OVERRIDE
Definition: http.cc:679
virtual ResultState RunDataToDevNull(RequestState &Req) APT_OVERRIDE
Definition: http.cc:650
virtual ResultState Go(bool ToFile, RequestState &Req) APT_OVERRIDE
Definition: http.cc:762
Small representation of a file descriptor for network traffic.
Definition: connect.h:24
static std::unique_ptr< MethodFd > FromFd(int iFd)
Construct a MethodFd from a UNIX file descriptor.
Definition: connect.cc:106
enum RequestState::@6 Encoding
unsigned long long StartPos
Definition: basehttp.h:44
unsigned long long MaximumSize
Definition: basehttp.h:46
FileFd File
Definition: basehttp.h:54
unsigned long long DownloadSize
Definition: basehttp.h:40
unsigned long long JunkSize
Definition: basehttp.h:42
bool AddPartialFileToHashes(FileFd &File)
Definition: basehttp.cc:268
enum RequestState::@7 State
time_t Date
Definition: basehttp.h:48
virtual void Reset()
Definition: basehttp.cc:274
URI ServerName
Definition: basehttp.h:73
BaseHttpMethod * Owner
Definition: basehttp.h:78
unsigned long TimeOut
Definition: basehttp.h:75
URI Proxy
Definition: basehttp.h:74
bool Persistent
Definition: basehttp.h:67
HashStringList ExpectedHashes
unsigned long long ResumePoint
string Base64Encode(const string &S)
Definition: strutl.cc:573
bool CheckDomainList(const string &Host, const string &List)
Definition: strutl.cc:1527
string TimeRFC1123(time_t Date, bool const NumericTimezone)
Definition: strutl.cc:853