tcpflow  1.6.1
About: tcpflow is a TCP/IP packet demultiplexer that captures data transmitted as part of TCP connections (flows), and stores the data in a way that is convenient for protocol analysis and debugging.
  Fossies Dox: tcpflow-1.6.1.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

tcpflow.cpp
Go to the documentation of this file.
1 /*
2  * This file is part of tcpflow by Simson Garfinkel <simsong@acm.org>.
3  * Originally by Jeremy Elson <jelson@circlemud.org>.
4  *
5  * This source code is under the GNU Public License (GPL) version 3.
6  * See COPYING for details.
7  *
8  */
9 
10 #define __MAIN_C__
11 
12 #include "config.h"
13 
14 #include "tcpflow.h"
15 
16 #include "tcpip.h"
17 #include "tcpdemux.h"
18 #include "bulk_extractor_i.h"
19 #include "iptree.h"
20 
21 #include "be13_api/utils.h"
22 
23 #include <string>
24 #include <vector>
25 #include <sys/types.h>
26 #include <dirent.h>
27 #include <getopt.h> // getopt_long()
28 
29 #ifdef HAVE_GRP_H
30 # include <grp.h> // initgroups()
31 #endif
32 
33 /* bring in inet_ntop if it is not present */
34 #define ETH_ALEN 6
35 #ifndef HAVE_INET_NTOP
36 #include "inet_ntop.c"
37 #endif
38 
39 #ifdef HAVE_CAP_NG_H
40 #include <cap-ng.h>
41 #endif
42 
43 /* droproot is from tcpdump.
44  * See https://github.com/the-tcpdump-group/tcpdump/blob/master/tcpdump.c#L611
45  */
46 const char *program_name = 0;
47 const char *tcpflow_droproot_username = 0;
48 const char *tcpflow_chroot_dir = 0;
49 
51 
52 scanner_info::scanner_config be_config; // system configuration
53 
54 typedef struct {
55  const char *name;
56  const char *dvalue;
57  const char *help;
58 } default_t;
59 
61  {"tdelta","0","Time delta in seconds"},
62  {"packet-buffer-timeout", "10", "Time in milliseconds between each callback from libpcap"},
63  {0,0,0}
64 };
65 
66 #ifdef HAVE_NETINET_IP_H
67 #include <netinet/ip.h>
68 #endif
69 
70 const char *progname = 0; // name of the program
71 int debug = DEFAULT_DEBUG_LEVEL; // global variable, not clear why
72 
73 /* semaphore prevents multiple copies from outputing on top of each other */
74 #ifdef HAVE_PTHREAD_H
75 #include <semaphore.h>
76 sem_t *semlock = 0;
77 #endif
78 
79 #define DEFAULT_REPORT_FILENAME "report.xml"
80 
81 /****************************************************************
82  *** SCANNER PLUG-IN SYSTEM
83  ****************************************************************/
84 
86  scan_md5,
87  scan_http,
89 // removed scan_python becasue it does not support Python 3
90 // scan_python,
92 #ifdef USE_WIFI
94 #endif
95  0};
96 
97 bool opt_no_promisc = false; // true if we should not use promiscious mode
98 
99 /* Long options!
100  *
101  * We need more long options; developers looking at this file should
102  * feel free to submit more!
103  */
104 
105 static const struct option longopts[] = {
106  { "chroot", required_argument, NULL, 'z' },
107  { "help", no_argument, NULL, 'h' },
108  { "relinquish-privileges", required_argument, NULL, 'U' },
109  { "verbose", no_argument, NULL, 'v' },
110  { "version", no_argument, NULL, 'V' },
111  { NULL, 0, NULL, 0 }
112 };
113 
114 
115 /****************************************************************
116  *** USAGE
117  ****************************************************************/
118 
119 static void usage(int level)
120 {
121  std::cout << PACKAGE_NAME << " version " << PACKAGE_VERSION << "\n\n";
122  std::cout << "usage: " << progname << " [-aBcCDhIpsvVZ] [-b max_bytes] [-d debug_level] \n";
123  std::cout << " [-[eE] scanner] [-f max_fds] [-F[ctTXMkmg]] [-h|--help] [-i iface]\n";
124  std::cout << " [-l files...] [-L semlock] [-m min_bytes] [-o outdir] [-r file] [-R file]\n";
125  std::cout << " [-S name=value] [-T template] [-U|--relinquish-privileges user] [-v|--verbose]\n";
126  std::cout << " [-w file] [-x scanner] [-X xmlfile] [-z|--chroot dir] [expression]\n\n";
127  std::cout << " -a: do ALL post-processing.\n";
128  std::cout << " -b max_bytes: max number of bytes per flow to save\n";
129  std::cout << " -d debug_level: debug level; default is " << DEFAULT_DEBUG_LEVEL << "\n";
130  std::cout << " -f: maximum number of file descriptors to use\n";
131  std::cout << " -h: print this help message (-hh for more help)\n";
132  std::cout << " -H: print detailed information about each scanner\n";
133  std::cout << " -i: network interface on which to listen\n";
134  std::cout << " -I: write for each flow another file *.findx to provide byte-indexed timestamps\n";
135  std::cout << " -g: output each flow in alternating colors (note change!)\n";
136  std::cout << " -l: treat non-flag arguments as input files rather than a pcap expression\n";
137  std::cout << " -L semlock - specifies that writes are locked using a named semaphore\n";
138  std::cout << " -p: don't use promiscuous mode\n";
139  std::cout << " -q: quiet mode - do not print warnings\n";
140 
141  std::cout << " -r file : read packets from tcpdump pcap file (may be repeated)\n";
142  std::cout << " -R file : read packets from tcpdump pcap file TO FINISH CONNECTIONS\n";
143  std::cout << " -v : verbose operation equivalent to -d 10\n";
144  std::cout << " -V : print version number and exit\n";
145  std::cout << " -w file : write packets not processed to file\n";
146  std::cout << " -o outdir : specify output directory (default '.')\n";
147  std::cout << " -X filename : DFXML output to filename\n";
148  std::cout << " -m bytes : specifies skip that starts a new stream (default "
149  << (unsigned)tcpdemux::options::MAX_SEEK << ").\n";
150  std::cout << " -F{p} : filename prefix/suffix (-hh for options)\n";
151  std::cout << " -T{t} : filename template (-hh for options; default "
152  << flow::filename_template << ")\n";
153  std::cout << " -Z do not decompress gzip-compressed HTTP transactions\n";
154  std::cout << " -K: output|keep pcap flow structure.\n";
155 
156  std::cout << "\nSecurity:\n";
157  std::cout << " -U user relinquish privleges and become user (if running as root)\n";
158  std::cout << " -z dir chroot to dir (requires that -U be used).\n";
159 
160  std::cout << "\nControl of Scanners:\n";
161  std::cout << " -E scanner - turn off all scanners except scanner\n";
162  std::cout << " -S name=value Set a configuration parameter (-hh for info)\n";
163  if(level > 1) {
164  std::cout << "\n" "Activated options -S name=value:";
165  for(int i=0;defaults[i].name;i++){
166  std::cout <<"\n -S "<< defaults[i].name << "=" << defaults[i].dvalue <<'\t'<< defaults[i].help;
167  }
168  std::cout << '\n';
169  be13::plugin::info_scanners(false,true,scanners_builtin,'e','x');
170  }
171  std::cout << "\n"
172  "Console output options:\n";
173  std::cout << " -B: binary output, even with -c or -C (normally -c or -C turn it off)\n";
174  std::cout << " -c: console print only (don't create files)\n";
175  std::cout << " -C: console print only, but without the display of source/dest header\n";
176  std::cout << " -0: don't print newlines after packets when printing to console\n";
177  std::cout << " -s: strip non-printable characters (change to '.')\n";
178  std::cout << " -J: output json format.\n";
179  std::cout << " -D: output in hex (useful to combine with -c or -C)\n";
180  std::cout << "\n";
181 #ifndef HAVE_LIBCAIRO
182  std::cout << "Rendering not available because Cairo was not installed.\n\n";
183 #endif
184  std::cout << "expression: tcpdump-like filtering expression\n";
185  std::cout << "\nSee the man page for additional information.\n\n";
186  if(level<2) return;
187  std::cout << "Filename Prefixes:\n";
188  std::cout << " -Fc : append the connection counter to ALL filenames\n";
189  std::cout << " -Ft : prepend the time_t UTC timestamp to ALL filenames\n";
190  std::cout << " -FT : prepend the ISO8601 UTC timestamp to ALL filenames\n";
191  std::cout << " -FX : Do not output any files (other than report files)\n";
192  std::cout << " -FM : Calculate the MD5 for every flow (stores in DFXML)\n";
193  std::cout << " -Fk : Bin output in 1K directories\n";
194  std::cout << " -Fm : Bin output in 1M directories (2 levels)\n";
195  std::cout << " -Fg : Bin output in 1G directories (3 levels)\n";
196  flow::usage();
197  std::cout << "\n" "Current limitations:"
198  "\n" " get_max_fds() = " << tcpdemux::getInstance()->get_max_fds();
199  std::cout << "\n" " NUM_RESERVED_FDS = " << NUM_RESERVED_FDS;
200  std::cout << '\n';
201 }
202 
203 /**
204  * Create the dfxml output
205  */
206 
207 static void dfxml_create(class dfxml_writer &xreport,int argc,char * const *argv)
208 {
209  xreport.push("dfxml","xmloutputversion='1.0'");
210  xreport.push("metadata",
211  "\n xmlns='http://afflib.org/tcpflow/' "
212  "\n xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
213  "\n xmlns:dc='http://purl.org/dc/elements/1.1/'" );
214  xreport.xmlout("dc:type","Feature Extraction","",false);
215  xreport.pop();
216  xreport.add_DFXML_creator(PACKAGE_NAME,PACKAGE_VERSION,"","command line to be provided");
217 }
218 
219 
220 /* String replace. Perhaps not the most efficient, but it works */
221 void replace(std::string &str,const std::string &from,const std::string &to)
222 {
223  if(from.size()==0) return;
224  bool changed = false;
225 
226  std::stringstream ss;
227  for(unsigned int i=0;i<str.size();){
228  if(str.substr(i,from.size())==from){
229  ss << to;
230  i+=from.size();
231  changed = true;
232  } else {
233  ss << str.at(i);
234  i++;
235  }
236  }
237  if(changed) str = ss.str(); // copy over original
238 }
239 
240 /* These must be global variables so they are available in the signal handler */
242 dfxml_writer *xreport = 0;
243 pcap_t *pd = 0;
244 void terminate(int sig)
245 {
246  if (sig == SIGHUP || sig == SIGINT || sig == SIGTERM) {
247  DEBUG(1) ("terminating orderly");
248  pcap_breakloop(pd);
249  return;
250  } else {
251  DEBUG(1) ("terminating");
252  be13::plugin::phase_shutdown(*the_fs); // give plugins a chance to do a clean shutdown
253  exit(0); /* libpcap uses onexit to clean up */
254  }
255 }
256 
257 #ifdef HAVE_FORK
258 #include <sys/wait.h>
259 // transparent decompression for process_infile
260 class inflater {
261  const std::string suffix;
262  const std::string invoc_format;
263 public:
264  inflater(const std::string &suffix_, const std::string &invoc_format_) :
265  suffix(suffix_), invoc_format(invoc_format_) {}
266  // is this inflater appropriate for a given file?
267  bool appropriate(const std::string &file_path) const
268  {
269  return ends_with(file_path,suffix);
270  }
271  // invoke the inflater in a shell, and return the file descriptor to read the inflated file from
272  int invoke(const std::string &file_path, int* ppid) const
273  {
274  std::string invocation = ssprintf(invoc_format.c_str(), file_path.c_str());
275  int pipe_fds[2];
276  if(!system(NULL)) {
277  std::cerr << "no shell available to decompress '" << file_path << "'" << std::endl;
278  return -1;
279  }
280  if(pipe(pipe_fds)) {
281  std::cerr << "failed to create pipe to decompress '" << file_path << "'" << std::endl;
282  return -1;
283  }
284 
285  pid_t child_pid;
286  child_pid = fork();
287  if(child_pid == -1) {
288  std::cerr << "failed to fork child to decompress '" << file_path << "'" << std::endl;
289  return -1;
290  }
291  if(child_pid == 0) {
292  // decompressor
293  close(pipe_fds[0]);
294  dup2(pipe_fds[1], 1);
295  if(system(invocation.c_str())) {
296  std::cerr << "decompressor reported error inflating '" << file_path << "'" << std::endl;
297  exit(1);
298  }
299  exit(0);
300  }
301  *ppid = child_pid;
302  close(pipe_fds[1]);
303  return pipe_fds[0];
304  }
305 };
306 
307 typedef std::vector<inflater *> inflaters_t;
308 static inflaters_t *build_inflaters()
309 {
310  inflaters_t *output = new inflaters_t();
311 
312  // gzip
313  output->push_back(new inflater(".gz", "gunzip -c '%s'"));
314  // zip
315  output->push_back(new inflater(".zip", "unzip -p '%s'"));
316  // bz2
317  output->push_back(new inflater(".bz2", "bunzip2 -c '%s'"));
318  // xz
319  output->push_back(new inflater(".xz", "unxz -c '%s'"));
320  // lzma
321  output->push_back(new inflater(".lzma", "unlzma -c '%s'"));
322 
323  return output;
324 }
325 
326 #define HAVE_INFLATER
327 #endif
328 
329 // https://github.com/the-tcpdump-group/tcpdump/blob/master/tcpdump.c#L611
330 #ifndef _WIN32
331 /* Drop root privileges and chroot if necessary */
332 static void
333 droproot(tcpdemux &demux,const char *username, const char *chroot_dir)
334 {
335  struct passwd *pw = NULL;
336 
337  if (chroot_dir && !username) {
338  fprintf(stderr, "%s: Chroot without dropping root is insecure\n",
339  program_name);
340  exit(1);
341  }
342 
343  pw = getpwnam(username);
344  if (pw) {
345  /* Begin tcpflow add */
346  if(demux.xreport){
347  const char *outfilename = demux.xreport->get_outfilename().c_str();
348  if(chown(outfilename,pw->pw_uid,pw->pw_gid)){
349  fprintf(stderr, "%s: Coudln't change owner of '%.64s' to %s (uid %d): %s\n",
350  program_name, outfilename, username, pw->pw_uid, strerror(errno));
351  exit(1);
352  }
353  }
354  /* end tcpflow add */
355  if (chroot_dir) {
356  if (chroot(chroot_dir) != 0 || chdir ("/") != 0) {
357  fprintf(stderr, "%s: Couldn't chroot/chdir to '%.64s': %s\n",
358  program_name, chroot_dir, pcap_strerror(errno));
359  exit(1);
360  }
361  }
362 #ifdef HAVE_LIBCAP_NG
363  {
364  int ret = capng_change_id(pw->pw_uid, pw->pw_gid, CAPNG_NO_FLAG);
365  if (ret < 0) {
366  fprintf(stderr, "error : ret %d\n", ret);
367  } else {
368  fprintf(stderr, "dropped privs to %s\n", username);
369  }
370  }
371 #else
372  if (initgroups(pw->pw_name, pw->pw_gid) != 0 ||
373  setgid(pw->pw_gid) != 0 || setuid(pw->pw_uid) != 0) {
374  fprintf(stderr, "%s: Couldn't change to '%.32s' uid=%lu gid=%lu: %s\n",
375  program_name, username,
376  (unsigned long)pw->pw_uid,
377  (unsigned long)pw->pw_gid,
378  pcap_strerror(errno));
379  exit(1);
380  }
381  else {
382  fprintf(stderr, "dropped privs to %s\n", username);
383  }
384 #endif /* HAVE_LIBCAP_NG */
385  }
386  else {
387  fprintf(stderr, "%s: Couldn't find user '%.32s'\n",
388  program_name, username);
389  exit(1);
390  }
391 #ifdef HAVE_LIBCAP_NG
392  /* We don't need CAP_SETUID, CAP_SETGID and CAP_SYS_CHROOT any more. */
393  capng_updatev(
394  CAPNG_DROP,
395  (capng_type_t)(CAPNG_EFFECTIVE | CAPNG_PERMITTED),
396  CAP_SETUID,
397  CAP_SETGID,
398  CAP_SYS_CHROOT,
399  -1);
400  capng_apply(CAPNG_SELECT_BOTH);
401 #endif /* HAVE_LIBCAP_NG */
402 
403 }
404 #endif /* _WIN32 */
405 
406 /**
407  * Perform the droproot operation for tcpflow. This needs to be called immediately after pcap_open()
408  */
410 {
413  }
414 }
415 
416 /*
417  * process an input file or device
418  * May be repeated.
419  * If start is false, do not initiate new connections
420  * Return 0 on success or -1 on error
421  */
422 #ifdef HAVE_INFLATER
423 static inflaters_t *inflaters = 0;
424 #endif
425 static int process_infile(tcpdemux &demux,const std::string &expression,std::string &device,const std::string &infile)
426 {
427  char error[PCAP_ERRBUF_SIZE];
428  int dlt=0;
429  pcap_handler handler;
430  int waitfor = -1;
431  int pipefd = -1;
432 
433 #ifdef HAVE_INFLATER
434  if(inflaters==0) inflaters = build_inflaters();
435 #endif
436 
437  if (infile!=""){
438  std::string file_path = infile;
439  // decompress input if necessary
440 #ifdef HAVE_INFLATER
441  for(inflaters_t::const_iterator it = inflaters->begin(); it != inflaters->end(); it++) {
442  if((*it)->appropriate(infile)) {
443  pipefd = (*it)->invoke(infile, &waitfor);
444  if(pipefd < 0) {
445  std::cerr << "decompression of '" << infile << "' failed: " << strerror (errno) << std::endl;
446  exit(1);
447  }
448  file_path = ssprintf("/dev/fd/%d", pipefd);
449  if(access(file_path.c_str(), R_OK)) {
450  std::cerr << "decompression of '" << infile << "' is not available on this system" << std::endl;
451  exit(1);
452  }
453  break;
454  }
455  }
456 #endif
457  if ((pd = pcap_open_offline(file_path.c_str(), error)) == NULL){ /* open the capture file */
458  die("%s", error);
459  }
460  tcpflow_droproot(demux); // drop root if requested
461  dlt = pcap_datalink(pd); /* get the handler for this kind of packets */
462  handler = find_handler(dlt, infile.c_str());
463  } else {
464  /* if the user didn't specify a device, try to find a reasonable one */
465  if (device.empty()){
466 #ifdef HAVE_PCAP_FINDALLDEVS
467  char errbuf[PCAP_ERRBUF_SIZE];
468  pcap_if_t *alldevs = 0;
469  if (pcap_findalldevs(&alldevs,errbuf)){
470  die("%s", errbuf);
471  }
472 
473  if (alldevs == 0) {
474  die("found 0 devices, maybe you don't have permissions, switch to root or equivalent user instead.");
475  }
476 
477  device.assign(alldevs[0].name);
478  pcap_freealldevs(alldevs);
479 #else
480  const char* dev = pcap_lookupdev(error);
481  if (dev == NULL)
482  die("%s", error);
483 
484  device.assign(dev);
485 #endif
486  }
487 
488  /* make sure we can open the device */
489  if ((pd = pcap_open_live(device.c_str(), SNAPLEN, !opt_no_promisc, packet_buffer_timeout, error)) == NULL){
490  die("%s", error);
491  }
492  tcpflow_droproot(demux); // drop root if requested
493  /* get the handler for this kind of packets */
494  dlt = pcap_datalink(pd);
495  handler = find_handler(dlt, device.c_str());
496  }
497 
498  DEBUG(20) ("filter expression: '%s'",expression.c_str());
499 
500  /* install the filter expression in libpcap */
501  struct bpf_program fcode;
502  if (pcap_compile(pd, &fcode, expression.c_str(), 1, 0) < 0){
503  die("%s", pcap_geterr(pd));
504  }
505 
506  if (pcap_setfilter(pd, &fcode) < 0){
507  die("%s", pcap_geterr(pd));
508  }
509 
510  /* initialize our flow state structures */
511 
512  /* set up signal handlers for graceful exit (pcap uses onexit to put
513  * interface back into non-promiscuous mode
514  */
515  portable_signal(SIGTERM, terminate);
516  portable_signal(SIGINT, terminate);
517 #ifdef SIGHUP
518  portable_signal(SIGHUP, terminate);
519 #endif
520 
521  /* start listening or reading from the input file */
522  if (infile == "") DEBUG(1) ("listening on %s", device.c_str());
523  int pcap_retval = pcap_loop(pd, -1, handler, (u_char *)tcpdemux::getInstance());
524 
525  if (pcap_retval < 0 && pcap_retval != -2){
526  DEBUG(1) ("%s: %s", infile.c_str(),pcap_geterr(pd));
527  return -1;
528  }
529  pcap_close (pd);
530 #ifdef HAVE_FORK
531  if (waitfor != -1) {
532  wait (0);
533  }
534  if (pipefd != -1) {
535  close (pipefd);
536  }
537 #endif
538 
539  return 0;
540 }
541 
542 
543 /* be_hash. Currently this just returns the MD5 of the sbuf,
544  * but eventually it will allow the use of different hashes.
545  */
546 static std::string be_hash_name("md5");
547 static std::string be_hash_func(const uint8_t *buf,size_t bufsize)
548 {
549  if(be_hash_name=="md5" || be_hash_name=="MD5"){
550  return md5_generator::hash_buf(buf,bufsize).hexdigest();
551  }
552  if(be_hash_name=="sha1" || be_hash_name=="SHA1" || be_hash_name=="sha-1" || be_hash_name=="SHA-1"){
553  return sha1_generator::hash_buf(buf,bufsize).hexdigest();
554  }
555  if(be_hash_name=="sha256" || be_hash_name=="SHA256" || be_hash_name=="sha-256" || be_hash_name=="SHA-256"){
556  return sha256_generator::hash_buf(buf,bufsize).hexdigest();
557  }
558  std::cerr << "Invalid hash name: " << be_hash_name << "\n";
559  std::cerr << "This version of bulk_extractor only supports MD5, SHA1, and SHA256\n";
560  exit(1);
561 }
563 
564 
565 int main(int argc, char *argv[])
566 {
567  int argc_original = argc;
568  char **argv_original = argv;
569  program_name = argv[0];
570  int opt_help = 0;
571  int opt_Help = 0;
573  sbuf_t::set_map_file_delimiter(""); // no delimiter on carving
574 #ifdef BROKEN
575  std::cerr << "WARNING: YOU ARE USING AN EXPERIMENTAL VERSION OF TCPFLOW \n";
576  std::cerr << "THAT DOES NOT WORK PROPERLY. PLEASE USE A RELEASE DOWNLOADED\n";
577  std::cerr << "FROM http://digitalcorpora.org/downloads/tcpflow\n";
578  std::cerr << "\n";
579 #endif
580 
581  bool opt_enable_report = true;
582  bool force_binary_output = false;
583  std::string device; // default device
584  const char *lockname = 0;
585  std::string reportfilename;
586  std::vector<std::string> Rfiles; // files for finishing
587  std::vector<std::string> rfiles; // files to read
588  tcpdemux &demux = *tcpdemux::getInstance(); // the demux object we will be using.
589  std::string command_line = dfxml_writer::make_command_line(argc,argv);
590  std::string opt_unk_packets;
591  bool opt_quiet = false;
592 
593  /* Set up debug system */
594  progname = argv[0];
595  init_debug(progname,1);
596 
597  /* Make sure that the system was compiled properly */
598  if(sizeof(struct be13::ip4)!=20 || sizeof(struct be13::tcphdr)!=20){
599  fprintf(stderr,"COMPILE ERROR.\n");
600  fprintf(stderr," sizeof(struct ip)=%d; should be 20.\n", (int)sizeof(struct be13::ip4));
601  fprintf(stderr," sizeof(struct tcphdr)=%d; should be 20.\n", (int)sizeof(struct be13::tcphdr));
602  fprintf(stderr,"CANNOT CONTINUE\n");
603  exit(1);
604  }
605 
606  bool trailing_input_list = false;
607  int arg;
608  while ((arg = getopt_long(argc, argv, "aA:Bb:cCd:DE:e:E:F:f:gHhIi:lL:m:o:pqR:r:S:sT:U:Vvw:x:X:z:ZK0J", longopts, NULL)) != EOF) {
609  switch (arg) {
610  case 'a':
611  demux.opt.post_processing = true;
612  demux.opt.opt_md5 = true;
614  break;
615 
616  case 'A':
617  fprintf(stderr,"-AH has been deprecated. Just use -a\n");
618  break;
619 
620  case 'b':
621  demux.opt.max_bytes_per_flow = atoi(optarg);
622  if(debug > 1) {
623  std::cout << "capturing max of " << demux.opt.max_bytes_per_flow << " bytes per flow." << std::endl;
624  }
625  break;
626  case 'B':
627  force_binary_output = true;
628  demux.opt.output_strip_nonprint = false; DEBUG(10) ("converting non-printable characters to '.'");
629  break;
630  case 'C':
631  demux.opt.console_output = true; DEBUG(10) ("printing packets to console only");
632  demux.opt.suppress_header = 1; DEBUG(10) ("packet header dump suppressed");
633  break;
634  case 'c':
635  demux.opt.console_output = true; DEBUG(10) ("printing packets to console only");
636  break;
637  case '0':
638  demux.opt.console_output_nonewline = true;
639  break;
640  case 'd':
641  if ((debug = atoi(optarg)) < 0) {
643  DEBUG(1) ("warning: -d flag with 0 debug level '%s'", optarg);
644  }
645  break;
646  case 'D':
647  demux.opt.output_hex = true;DEBUG(10) ("Console output in hex");
648  demux.opt.output_strip_nonprint = false; DEBUG(10) ("Will not convert non-printablesto '.'");
649  break;
650  case 'E':
653  break;
654  case 'e':
656  demux.opt.post_processing = true; // enable post processing if anything is turned on
657  break;
658  case 'F':
659  for(const char *cc=optarg;*cc;cc++){
660  switch(*cc){
661  case 'c': replace(flow::filename_template,"%c","%C"); break;
662  case 'k': flow::filename_template = "%K/" + flow::filename_template; break;
663  case 'm': flow::filename_template = "%M000-%M999/%M%K/" + flow::filename_template; break;
664  case 'g': flow::filename_template = "%G000000-%G999999/%G%M000-%G%M999/%G%M%K/" + flow::filename_template; break;
665  case 't': flow::filename_template = "%tT" + flow::filename_template; break;
666  case 'T': flow::filename_template = "%T" + flow::filename_template; break;
667  case 'X': demux.opt.store_output = false;break;
668  case 'M': demux.opt.opt_md5 = true;break;
669  default:
670  fprintf(stderr,"-F invalid format specification '%c'\n",*cc);
671  }
672  }
673  break;
674  case 'f':
675  {
676  int mnew = atoi(optarg);
677  DEBUG(1)("changing max_fds from %d to %d",demux.max_fds,mnew);
678  demux.max_fds = mnew;
679  break;
680  }
681  case 'i': device = std::string(optarg); break;
682  case 'I':
683  DEBUG(10) ("creating packet index files");
684  demux.opt.output_packet_index = true;
685  break;
686  case 'g':
687  demux.opt.use_color = 1;
688  DEBUG(10) ("using colors");
689  break;
690  case 'l': trailing_input_list = true; break;
691  case 'J':
692  demux.opt.output_json = true;
693  break;
694  case 'K':;
695  demux.opt.output_pcap = true;
696  demux.alter_processing_core();
697  break;
698  case 'L': lockname = optarg; break;
699  case 'm':
700  demux.opt.max_seek = atoi(optarg);
701  DEBUG(10) ("max_seek set to %d",demux.opt.max_seek); break;
702  case 'o':
703  demux.outdir = optarg;
704  flow::outdir = optarg;
705  break;
706  case 'p': opt_no_promisc = true; DEBUG(10) ("NOT turning on promiscuous mode"); break;
707  case 'q': opt_quiet = true; break;
708  case 'R': Rfiles.push_back(optarg); break;
709  case 'r': rfiles.push_back(optarg); break;
710  case 'S':
711  {
712  std::vector<std::string> params = split(optarg,'=');
713  if(params.size()!=2){
714  std::cerr << "Invalid paramter: " << optarg << "\n";
715  exit(1);
716  }
717  be_config.namevals[params[0]] = params[1];
718  continue;
719  }
720 
721  case 's':
722  demux.opt.output_strip_nonprint = 1; DEBUG(10) ("converting non-printable characters to '.'");
723  break;
724  case 'T':
725  flow::filename_template = optarg;
726  if(flow::filename_template.find("%c")==std::string::npos){
727  flow::filename_template += std::string("%C%c"); // append %C%c if not present
728  }
729  break;
730  case 'U': tcpflow_droproot_username = optarg; break;
731  case 'V': std::cout << PACKAGE_NAME << " " << PACKAGE_VERSION << "\n"; exit (1);
732  case 'v': debug = 10; break;
733  case 'w': opt_unk_packets = optarg;break;
734  case 'x': be13::plugin::scanners_disable(optarg);break;
735  case 'X': reportfilename = optarg;break;
736  case 'z': tcpflow_chroot_dir = optarg; break;
737  case 'Z': demux.opt.gzip_decompress = 0; break;
738  case 'H': opt_Help += 1; break;
739  case 'h': opt_help += 1; break;
740  default:
741  DEBUG(1) ("error: unrecognized switch '%c'", arg);
742  opt_help += 1;
743  break;
744  }
745  }
746 
748  err(1,"-z option requires -U option");
749  }
750 
751  argc -= optind;
752  argv += optind;
753 
754 
755  /* Load all the scanners and enable the ones we care about */
756  scanner_info si;
757  si.config = &be_config;
758 
759  si.get_config("enable_report",&opt_enable_report,"Enable report.xml");
761 
762  if(opt_Help){
764  exit(0);
765  }
766 
767  if(opt_help) {
768  usage(opt_help);
769  exit(0);
770  }
771 
772 
773  if(demux.opt.post_processing && !demux.opt.store_output){
774  std::cerr << "ERROR: post_processing currently requires storing output.\n";
775  exit(1);
776  }
777 
778  if(demux.opt.opt_md5) be13::plugin::scanners_enable("md5");
780 
781  /* If there is no report filename, call it report.xml in the output directory */
782  if( reportfilename.size()==0 ){
783  reportfilename = demux.outdir + "/" + DEFAULT_REPORT_FILENAME;
784  }
785 
786  /* remaining arguments are either an input list (-l flag) or a pcap expression (default) */
787  std::string expression = "";
788  if(trailing_input_list) {
789  for(int ii = 0; ii < argc; ii++) {
790  rfiles.push_back(argv[ii]);
791  }
792  }
793  else {
794  /* get the user's expression out of remainder of the arg... */
795  for(int i=0;i<argc;i++){
796  if(expression.size()>0) expression+=" ";
797  expression += argv[i];
798  }
799  }
800 
801  /* More option processing */
802 
803  /* was a semaphore provided for the lock? */
804  if(lockname){
805 #if defined(HAVE_SEMAPHORE_H) && defined(HAVE_PTHREAD_H)
806  semlock = sem_open(lockname,O_CREAT,0777,1); // get the semaphore
807 #else
808  fprintf(stderr,"%s: attempt to create lock pthreads not present\n",argv[0]);
809  exit(1);
810 #endif
811  }
812 
813  if(force_binary_output) demux.opt.output_strip_nonprint = false;
814  /* make sure outdir is a directory. If it isn't, try to make it.*/
815  struct stat stbuf;
816  if(stat(demux.outdir.c_str(),&stbuf)==0){
817  if(!S_ISDIR(stbuf.st_mode)){
818  std::cerr << "outdir is not a directory: " << demux.outdir << "\n";
819  exit(1);
820  }
821  } else {
822  if(MKDIR(demux.outdir.c_str(),0777)){
823  std::cerr << "cannot create " << demux.outdir << ": " << strerror(errno) << "\n";
824  exit(1);
825  }
826  }
827 
828  std::string input_fname;
829  if(rfiles.size() > 0) {
830  input_fname = rfiles.at(0);
831  if(rfiles.size() > 1) {
832  input_fname += ssprintf(" + %d more", rfiles.size() - 1);
833  }
834  }
835 
836  /* report file specified? If so, open it.
837  * Note: If we are going to chroot, we need apply the chroot prefix also,
838  * but we need to open the file *now*.
839  */
840  if(reportfilename.size()>0 && opt_enable_report){
841  if (tcpflow_chroot_dir){
842  reportfilename = std::string(tcpflow_chroot_dir) + std::string("/") + reportfilename;
843  }
844  std::cerr << "reportfilename: " << reportfilename << "\n";
845  xreport = new dfxml_writer(reportfilename,false);
846  dfxml_create(*xreport,argc_original,argv_original);
847  demux.xreport = xreport;
848  }
849  if(opt_unk_packets.size()>0){
850  if(input_fname.size()==0){
851  std::cerr << "currently the -w option requires the -r option\n";
852  exit(1);
853  }
854  if(access(input_fname.c_str(),R_OK)) die("cannot read: %s: %s",input_fname.c_str(),strerror(errno));
855  demux.save_unk_packets(opt_unk_packets,input_fname);
856  }
857 
858 
859  /* Debug prefix set? */
860  std::string debug_prefix=progname;
861  si.get_config("debug-prefix",&debug_prefix,"Prefix for debug output");
862  init_debug(debug_prefix.c_str(),0);
863 
864  DEBUG(10) ("%s version %s ", PACKAGE_NAME, PACKAGE_VERSION);
865 
866  const char *name = device.c_str();
867  if(input_fname.size()>0) name=input_fname.c_str();
868  if(name==0) name="<default>";
869 
870  feature_file_names_t feature_file_names;
873  fs.init(feature_file_names);
874  the_fs = &fs;
875  demux.fs = &fs;
876 
877  si.get_config("tdelta",&datalink_tdelta,"Time offset for packets");
878  si.get_config("packet-buffer-timeout", &packet_buffer_timeout, "Time in milliseconds between each callback from libpcap");
879 
880  /* Record the configuration */
881  if(xreport){
882  xreport->push("configuration");
883  xreport->pop(); // configuration
884  xreport->xmlout("tdelta",datalink_tdelta);
885  }
886 
887 
888 
889  /* Process r files and R files */
890  int exit_val = 0;
891  if(xreport){
892  xreport->push("configuration");
893  }
894  if(rfiles.size()==0 && Rfiles.size()==0){
895  /* live capture */
896  demux.start_new_connections = true;
897  int err = process_infile(demux,expression,device,"");
898  if (err < 0) {
899  exit_val = 1;
900  }
901  input_fname = device;
902  }
903  else {
904  /* first pick up the new connections with -r */
905  demux.start_new_connections = true;
906  for(std::vector<std::string>::const_iterator it=rfiles.begin();it!=rfiles.end();it++){
907  int err = process_infile(demux,expression,device,*it);
908  if (err < 0) {
909  exit_val = 1;
910  }
911  }
912  /* now pick up the outstanding connection with -R, but don't start new connections */
913  demux.start_new_connections = false;
914  for(std::vector<std::string>::const_iterator it=Rfiles.begin();it!=Rfiles.end();it++){
915  int err = process_infile(demux,expression,device,*it);
916  if (err < 0) {
917  exit_val = 1;
918  }
919  }
920  }
921 
922  /* -1 causes pcap_loop to loop forever, but it finished when the input file is exhausted. */
923 
924  DEBUG(2)("Open FDs at end of processing: %d",(int)demux.open_flows.size());
925  DEBUG(2)("demux.max_open_flows: %d",(int)demux.max_open_flows);
926  DEBUG(2)("Flow map size at end of processing: %d",(int)demux.flow_map.size());
927  DEBUG(2)("Flows seen: %d",(int)demux.flow_counter);
928 
929  int open_fds = (int)demux.open_flows.size();
930  int flow_map_size = (int)demux.flow_map.size();
931 
932  demux.remove_all_flows(); // empty the map to capture the state
933  std::stringstream ss;
935 
936  /*
937  * Note: funny formats below are a result of mingw problems with PRId64.
938  */
939  const std::string total_flow_processed("Total flows processed: %" PRId64);
940  const std::string total_packets_processed("Total packets processed: %" PRId64);
941 
942  DEBUG(2)(total_flow_processed.c_str(),demux.flow_counter);
943  DEBUG(2)(total_packets_processed.c_str(),demux.packet_counter);
944 
945  if(xreport){
946  xreport->pop(); // fileobjects
947  xreport->xmlout("summary",ss.str(),"",false);
948  xreport->xmlout("open_fds_at_end",open_fds);
949  xreport->xmlout("max_open_flows",demux.max_open_flows);
950  xreport->xmlout("total_flows",demux.flow_counter);
951  xreport->xmlout("flow_map_size",flow_map_size);
952  xreport->xmlout("total_packets",demux.packet_counter);
953  xreport->add_rusage();
954  xreport->pop(); // bulk_extractor
955  xreport->close();
956  delete xreport;
957  }
958 
960  if(!opt_quiet){
961  /* Start counting how many files we have in the output directory.
962  * If we find more than 10,000, print the warning, and keep counting...
963  */
964  uint64_t filecount=0;
965  DIR *dirp = opendir(demux.outdir.c_str());
966  if(dirp){
967  struct dirent *dp=0;
968  while((dp=readdir(dirp))!=NULL){
969  filecount++;
970  if(filecount==10000){
971  std::cerr << "*** tcpflow WARNING:\n";
972  std::cerr << "*** Modern operating systems do not perform well \n";
973  std::cerr << "*** with more than 10,000 entries in a directory.\n";
974  std::cerr << "***\n";
975  }
976  }
977  closedir(dirp);
978  }
979  if(filecount>=10000){
980  std::cerr << "*** tcpflow created " << filecount
981  << " files in output directory " << demux.outdir << "\n";
982  std::cerr << "***\n";
983  std::cerr << "*** Next time, specify command-line options: -Fk , -Fm , or -Fg \n";
984  std::cerr << "*** This will automatically bin output into subdirectories.\n";
985  std::cerr << "*** type 'tcpflow -hhh' for more information.\n";
986  }
987  }
988  }
989 
990  exit(exit_val); // return(0) causes crash on Windows
991 }
static void set_main_threadid()
static std::string outdir
Definition: tcpip.h:160
static void usage()
Definition: flow.cpp:31
static std::string filename_template
Definition: tcpip.h:159
static hash__< md, SIZE > hash_buf(const uint8_t *buf, size_t bufsize)
Definition: hash_t.h:264
bool console_output_nonewline
Definition: tcpdemux.h:109
int64_t max_bytes_per_flow
Definition: tcpdemux.h:114
bool suppress_header
Definition: tcpdemux.h:116
bool output_strip_nonprint
Definition: tcpdemux.h:117
bool post_processing
Definition: tcpdemux.h:112
bool gzip_decompress
Definition: tcpdemux.h:113
bool output_packet_index
Definition: tcpdemux.h:122
int32_t max_seek
Definition: tcpdemux.h:124
intrusive_list< tcpip > open_flows
Definition: tcpdemux.h:139
@ WARN_TOO_MANY_FILES
Definition: tcpdemux.h:127
uint64_t packet_counter
Definition: tcpdemux.h:131
options opt
Definition: tcpdemux.h:146
bool start_new_connections
Definition: tcpdemux.h:144
dfxml_writer * xreport
Definition: tcpdemux.h:132
void remove_all_flows()
Definition: tcpdemux.cpp:273
void alter_processing_core()
Definition: tcpdemux.cpp:47
std::string outdir
Definition: tcpdemux.h:129
class feature_recorder_set * fs
Definition: tcpdemux.h:147
void save_unk_packets(const std::string &wfname, const std::string &ifname)
Definition: tcpdemux.cpp:358
unsigned int max_fds
Definition: tcpdemux.h:135
static unsigned int get_max_fds(void)
Definition: tcpdemux.cpp:293
uint64_t flow_counter
Definition: tcpdemux.h:130
flow_map_t flow_map
Definition: tcpdemux.h:138
static tcpdemux * getInstance()
Definition: tcpdemux.cpp:103
unsigned int max_open_flows
Definition: tcpdemux.h:134
static void set_map_file_delimiter(const std::string &new_delim)
Definition: sbuf.h:373
static const uint32_t NO_ALERT
void init(const feature_file_names_t &feature_files)
std::set< std::string > feature_file_names_t
static void phase_shutdown(feature_recorder_set &fs, std::stringstream *sxml=0)
Definition: plugin.cpp:395
void scanner_t(const class scanner_params &sp, const class recursion_control_block &rcb)
virtual void get_config(const scanner_info::config_t &c, const std::string &name, std::string *val, const std::string &help)
Definition: plugin.cpp:415
static void scanners_enable(const std::string &name)
Definition: plugin.cpp:363
static void info_scanners(bool detailed_info, bool detailed_settings, scanner_t *const *scanners_builtin, const char enable_opt, const char disable_opt)
Definition: plugin.cpp:486
const scanner_config * config
static void scanners_process_enable_disable_commands()
Definition: plugin.cpp:375
static void scanners_enable_all()
Definition: plugin.cpp:357
static void load_scanners(scanner_t *const *scanners_builtin, const scanner_info::scanner_config &sc)
Definition: plugin.cpp:229
static void get_scanner_feature_file_names(feature_file_names_t &feature_file_names)
Definition: plugin.cpp:715
static void scanners_disable(const std::string &name)
Definition: plugin.cpp:369
static void scanners_disable_all()
Definition: plugin.cpp:351
const char * name
Definition: http_parser.c:465
char * pcap_lookupdev(char *)
Definition: pcap_fake.cpp:59
pcap_t * pcap_open_live(const char *, int, int, int, char *)
Definition: pcap_fake.cpp:65
int pcap_datalink(pcap_t *p)
Definition: pcap_fake.cpp:159
int pcap_compile(pcap_t *p, struct bpf_program *program, const char *expression, int optimize, uint32_t mask)
Definition: pcap_fake.cpp:150
int pcap_loop(pcap_t *p, int cnt, pcap_handler callback, uint8_t *user)
Definition: pcap_fake.cpp:169
char * pcap_geterr(pcap_t *p)
Definition: pcap_fake.cpp:35
pcap_t * pcap_open_offline(const char *fname, char *errbuf)
Definition: pcap_fake.cpp:47
void pcap_close(pcap_t *p)
Definition: pcap_fake.cpp:218
int pcap_setfilter(pcap_t *p, struct bpf_program *prog)
Definition: pcap_fake.cpp:163
void(* pcap_handler)(uint8_t *, const struct pcap_pkthdr *, const uint8_t *)
Definition: pcap_fake.h:48
#define PCAP_ERRBUF_SIZE
Definition: pcap_fake.h:23
const char * dvalue
Definition: tcpflow.cpp:56
const char * help
Definition: tcpflow.cpp:57
const char * name
Definition: tcpflow.cpp:55
int main(int argc, char *argv[])
Definition: tcpflow.cpp:565
static std::string be_hash_name("md5")
int packet_buffer_timeout
Definition: tcpflow.cpp:50
static feature_recorder_set::hash_def be_hash(be_hash_name, be_hash_func)
void tcpflow_droproot(tcpdemux &demux)
Definition: tcpflow.cpp:409
pcap_t * pd
Definition: tcpflow.cpp:243
#define DEFAULT_REPORT_FILENAME
Definition: tcpflow.cpp:79
bool opt_no_promisc
Definition: tcpflow.cpp:97
const char * tcpflow_chroot_dir
Definition: tcpflow.cpp:48
dfxml_writer * xreport
Definition: tcpflow.cpp:242
scanner_info::scanner_config be_config
Definition: tcpflow.cpp:52
static std::string be_hash_func(const uint8_t *buf, size_t bufsize)
Definition: tcpflow.cpp:547
static void usage(int level)
Definition: tcpflow.cpp:119
static const struct option longopts[]
Definition: tcpflow.cpp:105
void terminate(int sig)
Definition: tcpflow.cpp:244
void replace(std::string &str, const std::string &from, const std::string &to)
Definition: tcpflow.cpp:221
feature_recorder_set * the_fs
Definition: tcpflow.cpp:241
scanner_t * scanners_builtin[]
Definition: tcpflow.cpp:85
int debug
Definition: tcpflow.cpp:71
static void dfxml_create(class dfxml_writer &xreport, int argc, char *const *argv)
Definition: tcpflow.cpp:207
static int process_infile(tcpdemux &demux, const std::string &expression, std::string &device, const std::string &infile)
Definition: tcpflow.cpp:425
static void droproot(tcpdemux &demux, const char *username, const char *chroot_dir)
Definition: tcpflow.cpp:333
const char * tcpflow_droproot_username
Definition: tcpflow.cpp:47
default_t defaults[]
Definition: tcpflow.cpp:60
const char * progname
Definition: tcpflow.cpp:70
const char * program_name
Definition: tcpflow.cpp:46
#define DEFAULT_DEBUG_LEVEL
Definition: tcpflow.h:243
#define PACKAGE_NAME
Definition: tcpflow.h:28
void(*)(int) portable_signal(int signo, void(*func)(int))
Definition: tcpflow.h:311
#define MKDIR(a, b)
Definition: tcpflow.h:49
scanner_t scan_wifiviz
Definition: tcpflow.h:322
void die(const char *fmt,...)
Definition: util.cpp:175
scanner_t scan_md5
Definition: tcpflow.h:317
scanner_t scan_http
Definition: tcpflow.h:318
void init_debug(const char *progname, int include_pid)
Definition: util.cpp:74
#define DEBUG(message_level)
Definition: tcpflow.h:273
scanner_t scan_netviz
Definition: tcpflow.h:321
#define PACKAGE_VERSION
Definition: tcpflow.h:24
std::string ssprintf(const char *fmt,...)
Definition: util.cpp:20
#define NUM_RESERVED_FDS
Definition: tcpflow.h:251
#define SNAPLEN
Definition: tcpflow.h:245
scanner_t scan_tcpdemux
Definition: tcpflow.h:320
static char * debug_prefix
Definition: util.cpp:15
unsigned char uint8_t
Definition: util.h:6
bool ends_with(const std::string &buf, const std::string &with)
Definition: utils.cpp:114
std::vector< std::string > & split(const std::string &s, char delim, std::vector< std::string > &elems)
Definition: utils.cpp:133
void err(int eval, const char *fmt,...)
Definition: utils.cpp:33