apt  2.2.4
About: Apt (Advanced Package Tool) is a management system for software packages (Debian/Ubuntu). Release series 2.2.
  Fossies Dox: apt-2.2.4.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

multicompress.cc
Go to the documentation of this file.
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4 
5  MultiCompressor
6 
7  This class is very complicated in order to optimize for the common
8  case of its use, writing a large set of compressed files that are
9  different from the old set. It spawns off compressors in parallel
10  to maximize compression throughput and has a separate task managing
11  the data going into the compressors.
12 
13  ##################################################################### */
14  /*}}}*/
15 // Include Files /*{{{*/
16 #include <config.h>
17 
19 #include <apt-pkg/error.h>
20 #include <apt-pkg/fileutl.h>
21 #include <apt-pkg/hashes.h>
22 #include <apt-pkg/strutl.h>
23 
24 #include <ctype.h>
25 #include <sys/stat.h>
26 #include <sys/time.h>
27 #include <sys/types.h>
28 #include <unistd.h>
29 
30 #include <algorithm>
31 #include <vector>
32 
33 #include "multicompress.h"
34 #include <apti18n.h>
35  /*}}}*/
36 
37 using namespace std;
38 
39 static std::vector<APT::Configuration::Compressor>::const_iterator findMatchingCompressor(std::string::const_iterator &I,
40  std::string::const_iterator const &End, std::vector<APT::Configuration::Compressor> const &Compressors)
41 {
42  // Grab a word (aka: a compressor name)
43  for (; I != End && isspace(*I); ++I);
44  string::const_iterator Start = I;
45  for (; I != End && !isspace(*I); ++I);
46 
47  auto const Comp = std::find_if(Compressors.begin(), Compressors.end(),
48  [&](APT::Configuration::Compressor const &C) { return stringcmp(Start, I, C.Name.c_str()) == 0;
49  });
50  if (Comp == Compressors.end())
51  _error->Warning(_("Unknown compression algorithm '%s'"),string(Start,I).c_str());
52  return Comp;
53 }
54 
55 // MultiCompress::MultiCompress - Constructor /*{{{*/
56 // ---------------------------------------------------------------------
57 /* Setup the file outputs, compression modes and fork the writer child */
58 MultiCompress::MultiCompress(string const &Output, string const &Compress,
59  mode_t const &Permissions, bool const &Write) : Outputter{-1}, Permissions(Permissions)
60 {
61  Outputs = 0;
62  UpdateMTime = 0;
63 
64  auto const Compressors = APT::Configuration::getCompressors();
65  // Parse the compression string, a space separated lists of compression types
66  for (auto I = Compress.cbegin(); I != Compress.cend();)
67  {
68  auto const Comp = findMatchingCompressor(I, Compress.cend(), Compressors);
69  if (Comp == Compressors.end())
70  continue;
71 
72  // Create and link in a new output
73  Files *NewOut = new Files;
74  NewOut->Next = Outputs;
75  Outputs = NewOut;
76  NewOut->CompressProg = *Comp;
77  NewOut->Output = Output + Comp->Extension;
78 
79  struct stat St;
80  if (stat(NewOut->Output.c_str(),&St) == 0)
81  NewOut->OldMTime = St.st_mtime;
82  else
83  NewOut->OldMTime = 0;
84  }
85 
86  if (Write == false)
87  return;
88 
89  /* Open all the temp files now so we can report any errors. File is
90  made unreable to prevent people from touching it during creating. */
91  for (Files *I = Outputs; I != 0; I = I->Next)
92  I->TmpFile.Open(I->Output + ".new", FileFd::WriteOnly | FileFd::Create | FileFd::Empty, FileFd::Extension, 0600);
93  if (_error->PendingError() == true)
94  return;
95 
96  if (Outputs == 0)
97  {
98  _error->Error(_("Compressed output %s needs a compression set"),Output.c_str());
99  return;
100  }
101 
102  Start();
103 }
104  /*}}}*/
105 // MultiCompress::~MultiCompress - Destructor /*{{{*/
106 // ---------------------------------------------------------------------
107 /* Just erase the file linked list. */
109 {
110  Die();
111 
112  for (; Outputs != 0;)
113  {
114  Files *Tmp = Outputs->Next;
115  delete Outputs;
116  Outputs = Tmp;
117  }
118 }
119  /*}}}*/
120 // MultiCompress::GetStat - Get stat information for compressed files /*{{{*/
121 // ---------------------------------------------------------------------
122 /* This checks each compressed file to make sure it exists and returns
123  stat information for a random file from the collection. False means
124  one or more of the files is missing. */
125 bool MultiCompress::GetStat(string const &Output,string const &Compress,struct stat &St)
126 {
127  auto const Compressors = APT::Configuration::getCompressors();
128 
129  // Parse the compression string, a space separated lists of compression types
130  bool DidStat = false;
131  for (auto I = Compress.cbegin(); I != Compress.cend();)
132  {
133  auto const Comp = findMatchingCompressor(I, Compress.cend(), Compressors);
134  if (Comp == Compressors.end())
135  continue;
136 
137  string Name = Output + Comp->Extension;
138  if (stat(Name.c_str(),&St) != 0)
139  return false;
140  DidStat = true;
141  }
142  return DidStat;
143 }
144  /*}}}*/
145 // MultiCompress::Start - Start up the writer child /*{{{*/
146 // ---------------------------------------------------------------------
147 /* Fork a child and setup the communication pipe. */
149 {
150  // Create a data pipe
151  int Pipe[2] = {-1,-1};
152  if (pipe(Pipe) != 0)
153  return _error->Errno("pipe",_("Failed to create IPC pipe to subprocess"));
154  for (int I = 0; I != 2; I++)
155  SetCloseExec(Pipe[I],true);
156 
157  // The child..
158  Outputter = fork();
159  if (Outputter == 0)
160  {
161  close(Pipe[1]);
162  Child(Pipe[0]);
163  if (_error->PendingError() == true)
164  {
165  _error->DumpErrors();
166  _exit(100);
167  }
168  _exit(0);
169  };
170 
171  close(Pipe[0]);
172  if (Input.OpenDescriptor(Pipe[1], FileFd::WriteOnly, true) == false)
173  return false;
174 
175  if (Outputter == -1)
176  return _error->Errno("fork",_("Failed to fork"));
177  return true;
178 }
179  /*}}}*/
180 // MultiCompress::Die - Clean up the writer /*{{{*/
181 // ---------------------------------------------------------------------
182 /* */
184 {
185  if (Input.IsOpen() == false)
186  return true;
187 
188  Input.Close();
189  bool Res = ExecWait(Outputter,_("Compress child"),false);
190  Outputter = -1;
191  return Res;
192 }
193  /*}}}*/
194 // MultiCompress::Finalize - Finish up writing /*{{{*/
195 // ---------------------------------------------------------------------
196 /* This is only necessary for statistics reporting. */
197 bool MultiCompress::Finalize(unsigned long long &OutSize)
198 {
199  OutSize = 0;
200  if (Input.IsOpen() == false || Die() == false)
201  return false;
202 
203  time_t Now;
204  time(&Now);
205 
206  // Check the mtimes to see if the files were replaced.
207  bool Changed = false;
208  for (Files *I = Outputs; I != 0; I = I->Next)
209  {
210  struct stat St;
211  if (stat(I->Output.c_str(),&St) != 0)
212  return _error->Error(_("Internal error, failed to create %s"),
213  I->Output.c_str());
214 
215  if (I->OldMTime != St.st_mtime)
216  Changed = true;
217  else
218  {
219  // Update the mtime if necessary
220  if (UpdateMTime > 0 &&
221  (Now - St.st_mtime > (signed)UpdateMTime || St.st_mtime > Now))
222  {
223  utimes(I->Output.c_str(), NULL);
224  Changed = true;
225  }
226  }
227 
228  // Force the file permissions
229  if (St.st_mode != Permissions)
230  chmod(I->Output.c_str(),Permissions);
231 
232  OutSize += St.st_size;
233  }
234 
235  if (Changed == false)
236  OutSize = 0;
237 
238  return true;
239 }
240  /*}}}*/
241 // MultiCompress::OpenOld - Open an old file /*{{{*/
242 // ---------------------------------------------------------------------
243 /* This opens one of the original output files, possibly decompressing it. */
245 {
246  Files *Best = Outputs;
247  for (Files *I = Outputs; I != 0; I = I->Next)
248  if (Best->CompressProg.Cost > I->CompressProg.Cost)
249  Best = I;
250 
251  // Open the file
252  return Fd.Open(Best->Output, FileFd::ReadOnly, FileFd::Extension);
253 }
254  /*}}}*/
255 // MultiCompress::Child - The writer child /*{{{*/
256 // ---------------------------------------------------------------------
257 /* The child process forks a bunch of compression children and takes
258  input on FD and passes it to all the compressor child. On the way it
259  computes the MD5 of the raw data. After this the raw data in the
260  original files is compared to see if this data is new. If the data
261  is new then the temp files are renamed, otherwise they are erased. */
262 bool MultiCompress::Child(int const &FD)
263 {
264  /* Okay, now we just feed data from FD to all the other FDs. Also
265  stash a hash of the data to use later. */
266  SetNonBlock(FD,false);
267  unsigned char Buffer[32*1024];
268  unsigned long long FileSize = 0;
269  Hashes MD5(Hashes::MD5SUM);
270  while (1)
271  {
272  WaitFd(FD,false);
273  int Res = read(FD,Buffer,sizeof(Buffer));
274  if (Res == 0)
275  break;
276  if (Res < 0)
277  continue;
278 
279  MD5.Add(Buffer,Res);
280  FileSize += Res;
281  for (Files *I = Outputs; I != 0; I = I->Next)
282  {
283  if (I->TmpFile.Write(Buffer, Res) == false)
284  {
285  _error->Errno("write",_("IO to subprocess/file failed"));
286  break;
287  }
288  }
289  }
290 
291  if (_error->PendingError() == true)
292  return false;
293 
294  /* Now we have to copy the files over, or erase them if they
295  have not changed. First find the cheapest decompressor */
296  bool Missing = false;
297  for (Files *I = Outputs; I != 0; I = I->Next)
298  {
299  if (I->OldMTime == 0)
300  {
301  Missing = true;
302  break;
303  }
304  }
305 
306  // Check the MD5 of the lowest cost entity.
307  while (Missing == false)
308  {
309  FileFd CompFd;
310  if (OpenOld(CompFd) == false)
311  {
312  _error->Discard();
313  break;
314  }
315 
316  // Compute the hash
317  Hashes OldMD5(Hashes::MD5SUM);
318  unsigned long long NewFileSize = 0;
319  while (1)
320  {
321  unsigned long long Res = 0;
322  if (CompFd.Read(Buffer,sizeof(Buffer), &Res) == false)
323  return _error->Errno("read",_("Failed to read while computing MD5"));
324  if (Res == 0)
325  break;
326  NewFileSize += Res;
327  OldMD5.Add(Buffer,Res);
328  }
329  CompFd.Close();
330 
331  // Check the hash
333  FileSize == NewFileSize)
334  {
335  for (Files *I = Outputs; I != 0; I = I->Next)
336  {
337  I->TmpFile.Close();
338  RemoveFile("MultiCompress::Child", I->TmpFile.Name());
339  }
340  return !_error->PendingError();
341  }
342  break;
343  }
344 
345  // Finalize
346  for (Files *I = Outputs; I != 0; I = I->Next)
347  {
348  // Set the correct file modes
349  chmod(I->TmpFile.Name().c_str(),Permissions);
350 
351  if (rename(I->TmpFile.Name().c_str(),I->Output.c_str()) != 0)
352  _error->Errno("rename",_("Failed to rename %s to %s"),
353  I->TmpFile.Name().c_str(),I->Output.c_str());
354  I->TmpFile.Close();
355  }
356 
357  return !_error->PendingError();
358 }
359  /*}}}*/
360 
static bool std::string const metaIndex const *const pkgAcqMetaClearSig *const pkgAcquire::Item *const I
Definition: fileutl.h:39
bool OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose=false)
Definition: fileutl.cc:2572
bool IsOpen()
Definition: fileutl.h:150
@ Extension
Definition: fileutl.h:80
@ WriteOnly
Definition: fileutl.h:60
@ Empty
Definition: fileutl.h:66
@ Create
Definition: fileutl.h:63
@ ReadOnly
Definition: fileutl.h:59
bool Open(std::string FileName, unsigned int const Mode, CompressMode Compress, unsigned long const AccessMode=0666)
Definition: fileutl.cc:2415
bool Read(void *To, unsigned long long Size, bool AllowEof)
Definition: fileutl.h:89
bool Close()
Definition: fileutl.cc:2977
Definition: hashes.h:170
@ MD5SUM
Definition: hashes.h:185
HashString GetHashString(SupportedHashes hash)
Definition: hashes.cc:442
bool Add(const unsigned char *const Data, unsigned long long const Size) APT_NONNULL(2)
Definition: hashes.cc:353
Files * Outputs
Definition: multicompress.h:39
unsigned long UpdateMTime
Definition: multicompress.h:51
bool OpenOld(FileFd &Fd)
bool Finalize(unsigned long long &OutSize)
bool Child(int const &Fd)
mode_t Permissions
Definition: multicompress.h:41
MultiCompress(std::string const &Output, std::string const &Compress, mode_t const &Permissions, bool const &Write=true)
static bool GetStat(std::string const &Output, std::string const &Compress, struct stat &St)
void SetCloseExec(int Fd, bool Close)
Definition: fileutl.cc:792
void SetNonBlock(int Fd, bool Block)
Definition: fileutl.cc:804
bool ExecWait(pid_t Pid, const char *Name, bool Reap)
Definition: fileutl.cc:942
bool WaitFd(int Fd, bool write, unsigned long timeout)
Definition: fileutl.cc:819
bool RemoveFile(char const *const Function, std::string const &FileName)
Definition: fileutl.cc:198
static std::vector< APT::Configuration::Compressor >::const_iterator findMatchingCompressor(std::string::const_iterator &I, std::string::const_iterator const &End, std::vector< APT::Configuration::Compressor > const &Compressors)
APT_PUBLIC std::vector< Compressor > const getCompressors(bool const Cached=true)
Return a vector of Compressors supported for data.tar's.
Representation of supported compressors.
APT::Configuration::Compressor CompressProg
Definition: multicompress.h:32