"Fossies" - the Fresh Open Source Software Archive

Member "tcpflow-1.6.1/src/tcpflow.cpp" (19 Feb 2021, 34930 Bytes) of package /linux/misc/tcpflow-1.6.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "tcpflow.cpp" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 1.5.0_vs_1.6.1.

    1 /*
    2  * This file is part of tcpflow by Simson Garfinkel <simsong@acm.org>.
    3  * Originally by Jeremy Elson <jelson@circlemud.org>.
    4  *
    5  * This source code is under the GNU Public License (GPL) version 3.
    6  * See COPYING for details.
    7  *
    8  */
    9 
   10 #define __MAIN_C__
   11 
   12 #include "config.h"
   13 
   14 #include "tcpflow.h"
   15 
   16 #include "tcpip.h"
   17 #include "tcpdemux.h"
   18 #include "bulk_extractor_i.h"
   19 #include "iptree.h"
   20 
   21 #include "be13_api/utils.h"
   22 
   23 #include <string>
   24 #include <vector>
   25 #include <sys/types.h>
   26 #include <dirent.h>
   27 #include <getopt.h>      // getopt_long()
   28 
   29 #ifdef HAVE_GRP_H
   30 #  include <grp.h>       // initgroups()
   31 #endif
   32 
   33 /* bring in inet_ntop if it is not present */
   34 #define ETH_ALEN 6
   35 #ifndef HAVE_INET_NTOP
   36 #include "inet_ntop.c"
   37 #endif
   38 
   39 #ifdef HAVE_CAP_NG_H
   40 #include <cap-ng.h>
   41 #endif
   42 
   43 /* droproot is from tcpdump.
   44  * See https://github.com/the-tcpdump-group/tcpdump/blob/master/tcpdump.c#L611
   45  */
   46 const char *program_name = 0;
   47 const char *tcpflow_droproot_username = 0;
   48 const char *tcpflow_chroot_dir = 0;
   49 
   50 int packet_buffer_timeout = 10;
   51 
   52 scanner_info::scanner_config be_config; // system configuration
   53 
   54 typedef struct {
   55     const char *name;
   56     const char *dvalue;
   57     const char *help;
   58 } default_t;
   59 
   60 default_t defaults[] = {
   61     {"tdelta","0","Time delta in seconds"},
   62     {"packet-buffer-timeout", "10", "Time in milliseconds between each callback from libpcap"},
   63     {0,0,0}
   64 };
   65 
   66 #ifdef HAVE_NETINET_IP_H
   67 #include <netinet/ip.h>
   68 #endif
   69 
   70 const char *progname = 0;       // name of the program
   71 int debug = DEFAULT_DEBUG_LEVEL;    // global variable, not clear why
   72 
   73 /* semaphore prevents multiple copies from outputing on top of each other */
   74 #ifdef HAVE_PTHREAD_H
   75 #include <semaphore.h>
   76 sem_t *semlock = 0;
   77 #endif
   78 
   79 #define DEFAULT_REPORT_FILENAME "report.xml"
   80 
   81 /****************************************************************
   82  *** SCANNER PLUG-IN SYSTEM
   83  ****************************************************************/
   84 
   85 scanner_t *scanners_builtin[] = {
   86     scan_md5,
   87     scan_http,
   88     scan_netviz,
   89 // removed scan_python becasue it does not support Python 3
   90 //    scan_python,
   91     scan_tcpdemux,
   92 #ifdef USE_WIFI
   93     scan_wifiviz,
   94 #endif
   95     0};
   96 
   97 bool opt_no_promisc = false;        // true if we should not use promiscious mode
   98 
   99 /* Long options!
  100  *
  101  * We need more long options; developers looking at this file should
  102  * feel free to submit more!
  103  */
  104 
  105 static const struct option longopts[] = {
  106     { "chroot", required_argument, NULL, 'z' },
  107     { "help", no_argument, NULL, 'h' },
  108     { "relinquish-privileges", required_argument, NULL, 'U' },
  109     { "verbose", no_argument, NULL, 'v' },
  110     { "version", no_argument, NULL, 'V' },
  111     { NULL, 0, NULL, 0 }
  112 };
  113 
  114 
  115 /****************************************************************
  116  *** USAGE
  117  ****************************************************************/
  118 
  119 static void usage(int level)
  120 {
  121     std::cout << PACKAGE_NAME << " version " << PACKAGE_VERSION << "\n\n";
  122     std::cout << "usage: " << progname << " [-aBcCDhIpsvVZ] [-b max_bytes] [-d debug_level] \n";
  123     std::cout << "     [-[eE] scanner] [-f max_fds] [-F[ctTXMkmg]] [-h|--help] [-i iface]\n";
  124     std::cout << "     [-l files...] [-L semlock] [-m min_bytes] [-o outdir] [-r file] [-R file]\n";
  125     std::cout << "     [-S name=value] [-T template] [-U|--relinquish-privileges user] [-v|--verbose]\n";
  126     std::cout << "     [-w file] [-x scanner] [-X xmlfile] [-z|--chroot dir] [expression]\n\n";
  127     std::cout << "   -a: do ALL post-processing.\n";
  128     std::cout << "   -b max_bytes: max number of bytes per flow to save\n";
  129     std::cout << "   -d debug_level: debug level; default is " << DEFAULT_DEBUG_LEVEL << "\n";
  130     std::cout << "   -f: maximum number of file descriptors to use\n";
  131     std::cout << "   -h: print this help message (-hh for more help)\n";
  132     std::cout << "   -H: print detailed information about each scanner\n";
  133     std::cout << "   -i: network interface on which to listen\n";
  134     std::cout << "   -I: write for each flow another file *.findx to provide byte-indexed timestamps\n";
  135     std::cout << "   -g: output each flow in alternating colors (note change!)\n";
  136     std::cout << "   -l: treat non-flag arguments as input files rather than a pcap expression\n";
  137     std::cout << "   -L  semlock - specifies that writes are locked using a named semaphore\n";
  138     std::cout << "   -p: don't use promiscuous mode\n";
  139     std::cout << "   -q: quiet mode - do not print warnings\n";
  140 
  141     std::cout << "   -r file      : read packets from tcpdump pcap file (may be repeated)\n";
  142     std::cout << "   -R file      : read packets from tcpdump pcap file TO FINISH CONNECTIONS\n";
  143     std::cout << "   -v           : verbose operation equivalent to -d 10\n";
  144     std::cout << "   -V           : print version number and exit\n";
  145     std::cout << "   -w  file     : write packets not processed to file\n";
  146     std::cout << "   -o  outdir   : specify output directory (default '.')\n";
  147     std::cout << "   -X  filename : DFXML output to filename\n";
  148     std::cout << "   -m  bytes    : specifies skip that starts a new stream (default "
  149               << (unsigned)tcpdemux::options::MAX_SEEK << ").\n";
  150     std::cout << "   -F{p} : filename prefix/suffix (-hh for options)\n";
  151     std::cout << "   -T{t} : filename template (-hh for options; default "
  152               << flow::filename_template << ")\n";
  153     std::cout << "   -Z       do not decompress gzip-compressed HTTP transactions\n";
  154     std::cout << "   -K: output|keep pcap flow structure.\n";
  155 
  156     std::cout << "\nSecurity:\n";
  157     std::cout << "   -U user  relinquish privleges and become user (if running as root)\n";
  158     std::cout << "   -z dir   chroot to dir (requires that -U be used).\n";
  159 
  160     std::cout << "\nControl of Scanners:\n";
  161     std::cout << "   -E scanner   - turn off all scanners except scanner\n";
  162     std::cout << "   -S name=value  Set a configuration parameter (-hh for info)\n";
  163     if(level > 1) {
  164         std::cout << "\n" "Activated options -S name=value:";
  165         for(int i=0;defaults[i].name;i++){
  166             std::cout <<"\n   -S "<< defaults[i].name << "=" << defaults[i].dvalue <<'\t'<< defaults[i].help;
  167         }
  168         std::cout << '\n';
  169         be13::plugin::info_scanners(false,true,scanners_builtin,'e','x');
  170     }
  171     std::cout << "\n"
  172                  "Console output options:\n";
  173     std::cout << "   -B: binary output, even with -c or -C (normally -c or -C turn it off)\n";
  174     std::cout << "   -c: console print only (don't create files)\n";
  175     std::cout << "   -C: console print only, but without the display of source/dest header\n";
  176     std::cout << "   -0: don't print newlines after packets when printing to console\n";
  177     std::cout << "   -s: strip non-printable characters (change to '.')\n";
  178     std::cout << "   -J: output json format.\n";
  179     std::cout << "   -D: output in hex (useful to combine with -c or -C)\n";
  180     std::cout << "\n";
  181 #ifndef HAVE_LIBCAIRO
  182     std::cout << "Rendering not available because Cairo was not installed.\n\n";
  183 #endif
  184     std::cout << "expression: tcpdump-like filtering expression\n";
  185     std::cout << "\nSee the man page for additional information.\n\n";
  186     if(level<2) return;
  187     std::cout << "Filename Prefixes:\n";
  188     std::cout << "   -Fc : append the connection counter to ALL filenames\n";
  189     std::cout << "   -Ft : prepend the time_t UTC timestamp to ALL filenames\n";
  190     std::cout << "   -FT : prepend the ISO8601 UTC timestamp to ALL filenames\n";
  191     std::cout << "   -FX : Do not output any files (other than report files)\n";
  192     std::cout << "   -FM : Calculate the MD5 for every flow (stores in DFXML)\n";
  193     std::cout << "   -Fk : Bin output in 1K directories\n";
  194     std::cout << "   -Fm : Bin output in 1M directories (2 levels)\n";
  195     std::cout << "   -Fg : Bin output in 1G directories (3 levels)\n";
  196     flow::usage();
  197     std::cout << "\n" "Current limitations:"
  198                  "\n" "  get_max_fds() = " << tcpdemux::getInstance()->get_max_fds();
  199     std::cout << "\n" "  NUM_RESERVED_FDS = " << NUM_RESERVED_FDS;
  200     std::cout << '\n';
  201 }
  202 
  203 /**
  204  * Create the dfxml output
  205  */
  206 
  207 static void dfxml_create(class dfxml_writer &xreport,int argc,char * const *argv)
  208 {
  209     xreport.push("dfxml","xmloutputversion='1.0'");
  210     xreport.push("metadata",
  211          "\n  xmlns='http://afflib.org/tcpflow/' "
  212          "\n  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
  213          "\n  xmlns:dc='http://purl.org/dc/elements/1.1/'" );
  214     xreport.xmlout("dc:type","Feature Extraction","",false);
  215     xreport.pop();
  216     xreport.add_DFXML_creator(PACKAGE_NAME,PACKAGE_VERSION,"","command line to be provided");
  217 }
  218 
  219 
  220 /* String replace. Perhaps not the most efficient, but it works */
  221 void replace(std::string &str,const std::string &from,const std::string &to)
  222 {
  223     if(from.size()==0) return;
  224     bool changed = false;
  225 
  226     std::stringstream ss;
  227     for(unsigned int i=0;i<str.size();){
  228     if(str.substr(i,from.size())==from){
  229         ss << to;
  230         i+=from.size();
  231         changed = true;
  232     } else {
  233         ss << str.at(i);
  234         i++;
  235     }
  236     }
  237     if(changed) str = ss.str();         // copy over original
  238 }
  239 
  240 /* These must be global variables so they are available in the signal handler */
  241 feature_recorder_set *the_fs = 0;
  242 dfxml_writer *xreport = 0;
  243 pcap_t *pd = 0;
  244 void terminate(int sig)
  245 {
  246     if (sig == SIGHUP || sig == SIGINT || sig == SIGTERM) {
  247         DEBUG(1) ("terminating orderly");
  248         pcap_breakloop(pd);
  249         return;
  250     } else {
  251         DEBUG(1) ("terminating");
  252         be13::plugin::phase_shutdown(*the_fs);  // give plugins a chance to do a clean shutdown
  253         exit(0); /* libpcap uses onexit to clean up */
  254     }
  255 }
  256 
  257 #ifdef HAVE_FORK
  258 #include <sys/wait.h>
  259 // transparent decompression for process_infile
  260 class inflater {
  261     const std::string suffix;
  262     const std::string invoc_format;
  263 public:
  264     inflater(const std::string &suffix_, const std::string &invoc_format_) :
  265         suffix(suffix_), invoc_format(invoc_format_) {}
  266     // is this inflater appropriate for a given file?
  267     bool appropriate(const std::string &file_path) const
  268     {
  269         return ends_with(file_path,suffix);
  270     }
  271     // invoke the inflater in a shell, and return the file descriptor to read the inflated file from
  272     int invoke(const std::string &file_path, int* ppid) const
  273     {
  274         std::string invocation = ssprintf(invoc_format.c_str(), file_path.c_str());
  275         int pipe_fds[2];
  276         if(!system(NULL)) {
  277             std::cerr << "no shell available to decompress '" << file_path << "'" << std::endl;
  278             return -1;
  279         }
  280         if(pipe(pipe_fds)) {
  281             std::cerr << "failed to create pipe to decompress '" << file_path << "'" << std::endl;
  282             return -1;
  283         }
  284 
  285         pid_t child_pid;
  286         child_pid = fork();
  287         if(child_pid == -1) {
  288             std::cerr << "failed to fork child to decompress '" << file_path << "'" << std::endl;
  289             return -1;
  290         }
  291         if(child_pid == 0) {
  292             // decompressor
  293             close(pipe_fds[0]);
  294             dup2(pipe_fds[1], 1);
  295             if(system(invocation.c_str())) {
  296                 std::cerr << "decompressor reported error inflating '" << file_path << "'" << std::endl;
  297                 exit(1);
  298             }
  299             exit(0);
  300         }
  301         *ppid = child_pid;
  302         close(pipe_fds[1]);
  303         return pipe_fds[0];
  304     }
  305 };
  306 
  307 typedef std::vector<inflater *> inflaters_t;
  308 static inflaters_t *build_inflaters()
  309 {
  310     inflaters_t *output = new inflaters_t();
  311 
  312     // gzip
  313     output->push_back(new inflater(".gz", "gunzip -c '%s'"));
  314     // zip
  315     output->push_back(new inflater(".zip", "unzip -p '%s'"));
  316     // bz2
  317     output->push_back(new inflater(".bz2", "bunzip2 -c '%s'"));
  318     // xz
  319     output->push_back(new inflater(".xz", "unxz -c '%s'"));
  320     // lzma
  321     output->push_back(new inflater(".lzma", "unlzma -c '%s'"));
  322 
  323     return output;
  324 }
  325 
  326 #define HAVE_INFLATER
  327 #endif
  328 
  329 // https://github.com/the-tcpdump-group/tcpdump/blob/master/tcpdump.c#L611
  330 #ifndef _WIN32
  331 /* Drop root privileges and chroot if necessary */
  332 static void
  333 droproot(tcpdemux &demux,const char *username, const char *chroot_dir)
  334 {
  335     struct passwd *pw = NULL;
  336 
  337     if (chroot_dir && !username) {
  338         fprintf(stderr, "%s: Chroot without dropping root is insecure\n",
  339                 program_name);
  340         exit(1);
  341     }
  342 
  343     pw = getpwnam(username);
  344     if (pw) {
  345         /* Begin tcpflow add */
  346         if(demux.xreport){
  347             const char *outfilename = demux.xreport->get_outfilename().c_str();
  348             if(chown(outfilename,pw->pw_uid,pw->pw_gid)){
  349                 fprintf(stderr, "%s: Coudln't change owner of '%.64s' to %s (uid %d): %s\n",
  350                         program_name, outfilename, username, pw->pw_uid, strerror(errno));
  351                 exit(1);
  352             }
  353         }
  354         /* end tcpflow add */
  355         if (chroot_dir) {
  356             if (chroot(chroot_dir) != 0 || chdir ("/") != 0) {
  357                 fprintf(stderr, "%s: Couldn't chroot/chdir to '%.64s': %s\n",
  358                         program_name, chroot_dir, pcap_strerror(errno));
  359                 exit(1);
  360             }
  361         }
  362 #ifdef HAVE_LIBCAP_NG
  363         {
  364             int ret = capng_change_id(pw->pw_uid, pw->pw_gid, CAPNG_NO_FLAG);
  365             if (ret < 0) {
  366                 fprintf(stderr, "error : ret %d\n", ret);
  367             } else {
  368                 fprintf(stderr, "dropped privs to %s\n", username);
  369             }
  370         }
  371 #else
  372         if (initgroups(pw->pw_name, pw->pw_gid) != 0 ||
  373             setgid(pw->pw_gid) != 0 || setuid(pw->pw_uid) != 0) {
  374             fprintf(stderr, "%s: Couldn't change to '%.32s' uid=%lu gid=%lu: %s\n",
  375                     program_name, username,
  376                     (unsigned long)pw->pw_uid,
  377                     (unsigned long)pw->pw_gid,
  378                     pcap_strerror(errno));
  379             exit(1);
  380         }
  381         else {
  382             fprintf(stderr, "dropped privs to %s\n", username);
  383         }
  384 #endif /* HAVE_LIBCAP_NG */
  385     }
  386     else {
  387         fprintf(stderr, "%s: Couldn't find user '%.32s'\n",
  388                 program_name, username);
  389         exit(1);
  390     }
  391 #ifdef HAVE_LIBCAP_NG
  392     /* We don't need CAP_SETUID, CAP_SETGID and CAP_SYS_CHROOT any more. */
  393     capng_updatev(
  394                   CAPNG_DROP,
  395                   (capng_type_t)(CAPNG_EFFECTIVE | CAPNG_PERMITTED),
  396                   CAP_SETUID,
  397                   CAP_SETGID,
  398                   CAP_SYS_CHROOT,
  399                   -1);
  400     capng_apply(CAPNG_SELECT_BOTH);
  401 #endif /* HAVE_LIBCAP_NG */
  402 
  403 }
  404 #endif /* _WIN32 */
  405 
  406 /**
  407  * Perform the droproot operation for tcpflow. This needs to be called immediately after pcap_open()
  408  */
  409 void tcpflow_droproot(tcpdemux &demux)
  410 {
  411     if (tcpflow_droproot_username){
  412         droproot(demux,tcpflow_droproot_username,tcpflow_chroot_dir);
  413     }
  414 }
  415 
  416 /*
  417  * process an input file or device
  418  * May be repeated.
  419  * If start is false, do not initiate new connections
  420  * Return 0 on success or -1 on error
  421  */
  422 #ifdef HAVE_INFLATER
  423 static inflaters_t *inflaters = 0;
  424 #endif
  425 static int process_infile(tcpdemux &demux,const std::string &expression,std::string &device,const std::string &infile)
  426 {
  427     char error[PCAP_ERRBUF_SIZE];
  428     int dlt=0;
  429     pcap_handler handler;
  430     int waitfor = -1;
  431     int pipefd = -1;
  432 
  433 #ifdef HAVE_INFLATER
  434     if(inflaters==0) inflaters = build_inflaters();
  435 #endif
  436 
  437     if (infile!=""){
  438         std::string file_path = infile;
  439         // decompress input if necessary
  440 #ifdef HAVE_INFLATER
  441         for(inflaters_t::const_iterator it = inflaters->begin(); it != inflaters->end(); it++) {
  442             if((*it)->appropriate(infile)) {
  443                 pipefd = (*it)->invoke(infile, &waitfor);
  444                 if(pipefd < 0) {
  445                     std::cerr << "decompression of '" << infile << "' failed: " << strerror (errno) << std::endl;
  446                     exit(1);
  447                 }
  448                 file_path = ssprintf("/dev/fd/%d", pipefd);
  449                 if(access(file_path.c_str(), R_OK)) {
  450                     std::cerr << "decompression of '" << infile << "' is not available on this system" << std::endl;
  451                     exit(1);
  452                 }
  453                 break;
  454             }
  455         }
  456 #endif
  457     if ((pd = pcap_open_offline(file_path.c_str(), error)) == NULL){    /* open the capture file */
  458         die("%s", error);
  459     }
  460         tcpflow_droproot(demux);        // drop root if requested
  461     dlt = pcap_datalink(pd);    /* get the handler for this kind of packets */
  462     handler = find_handler(dlt, infile.c_str());
  463     } else {
  464     /* if the user didn't specify a device, try to find a reasonable one */
  465     if (device.empty()){
  466 #ifdef HAVE_PCAP_FINDALLDEVS
  467         char errbuf[PCAP_ERRBUF_SIZE];
  468         pcap_if_t *alldevs = 0;
  469         if (pcap_findalldevs(&alldevs,errbuf)){
  470             die("%s", errbuf);
  471         }
  472 
  473         if (alldevs == 0) {
  474             die("found 0 devices, maybe you don't have permissions, switch to root or equivalent user instead.");
  475         }
  476 
  477         device.assign(alldevs[0].name);
  478         pcap_freealldevs(alldevs);
  479 #else
  480         const char* dev = pcap_lookupdev(error);
  481         if (dev == NULL)
  482             die("%s", error);
  483 
  484         device.assign(dev);
  485 #endif
  486     }
  487 
  488     /* make sure we can open the device */
  489     if ((pd = pcap_open_live(device.c_str(), SNAPLEN, !opt_no_promisc, packet_buffer_timeout, error)) == NULL){
  490         die("%s", error);
  491     }
  492         tcpflow_droproot(demux);                     // drop root if requested
  493     /* get the handler for this kind of packets */
  494     dlt = pcap_datalink(pd);
  495     handler = find_handler(dlt, device.c_str());
  496     }
  497 
  498     DEBUG(20) ("filter expression: '%s'",expression.c_str());
  499 
  500     /* install the filter expression in libpcap */
  501     struct bpf_program  fcode;
  502     if (pcap_compile(pd, &fcode, expression.c_str(), 1, 0) < 0){
  503     die("%s", pcap_geterr(pd));
  504     }
  505 
  506     if (pcap_setfilter(pd, &fcode) < 0){
  507     die("%s", pcap_geterr(pd));
  508     }
  509 
  510     /* initialize our flow state structures */
  511 
  512     /* set up signal handlers for graceful exit (pcap uses onexit to put
  513      * interface back into non-promiscuous mode
  514      */
  515     portable_signal(SIGTERM, terminate);
  516     portable_signal(SIGINT, terminate);
  517 #ifdef SIGHUP
  518     portable_signal(SIGHUP, terminate);
  519 #endif
  520 
  521     /* start listening or reading from the input file */
  522     if (infile == "") DEBUG(1) ("listening on %s", device.c_str());
  523     int pcap_retval = pcap_loop(pd, -1, handler, (u_char *)tcpdemux::getInstance());
  524 
  525     if (pcap_retval < 0 && pcap_retval != -2){
  526     DEBUG(1) ("%s: %s", infile.c_str(),pcap_geterr(pd));
  527     return -1;
  528     }
  529     pcap_close (pd);
  530 #ifdef HAVE_FORK
  531     if (waitfor != -1) {
  532         wait (0);
  533     }
  534     if (pipefd != -1) {
  535         close (pipefd);
  536     }
  537 #endif
  538 
  539     return 0;
  540 }
  541 
  542 
  543 /* be_hash. Currently this just returns the MD5 of the sbuf,
  544  * but eventually it will allow the use of different hashes.
  545  */
  546 static std::string be_hash_name("md5");
  547 static std::string be_hash_func(const uint8_t *buf,size_t bufsize)
  548 {
  549     if(be_hash_name=="md5" || be_hash_name=="MD5"){
  550         return md5_generator::hash_buf(buf,bufsize).hexdigest();
  551     }
  552     if(be_hash_name=="sha1" || be_hash_name=="SHA1" || be_hash_name=="sha-1" || be_hash_name=="SHA-1"){
  553         return sha1_generator::hash_buf(buf,bufsize).hexdigest();
  554     }
  555     if(be_hash_name=="sha256" || be_hash_name=="SHA256" || be_hash_name=="sha-256" || be_hash_name=="SHA-256"){
  556         return sha256_generator::hash_buf(buf,bufsize).hexdigest();
  557     }
  558     std::cerr << "Invalid hash name: " << be_hash_name << "\n";
  559     std::cerr << "This version of bulk_extractor only supports MD5, SHA1, and SHA256\n";
  560     exit(1);
  561 }
  562 static feature_recorder_set::hash_def be_hash(be_hash_name,be_hash_func);
  563 
  564 
  565 int main(int argc, char *argv[])
  566 {
  567     int argc_original = argc;
  568     char **argv_original = argv;
  569     program_name = argv[0];
  570     int opt_help = 0;
  571     int opt_Help = 0;
  572     feature_recorder::set_main_threadid();
  573     sbuf_t::set_map_file_delimiter(""); // no delimiter on carving
  574 #ifdef BROKEN
  575     std::cerr << "WARNING: YOU ARE USING AN EXPERIMENTAL VERSION OF TCPFLOW \n";
  576     std::cerr << "THAT DOES NOT WORK PROPERLY. PLEASE USE A RELEASE DOWNLOADED\n";
  577     std::cerr << "FROM http://digitalcorpora.org/downloads/tcpflow\n";
  578     std::cerr << "\n";
  579 #endif
  580 
  581     bool opt_enable_report = true;
  582     bool force_binary_output = false;
  583     std::string device;             // default device
  584     const char *lockname = 0;
  585     std::string reportfilename;
  586     std::vector<std::string> Rfiles;    // files for finishing
  587     std::vector<std::string> rfiles;    // files to read
  588     tcpdemux &demux = *tcpdemux::getInstance();         // the demux object we will be using.
  589     std::string command_line = dfxml_writer::make_command_line(argc,argv);
  590     std::string opt_unk_packets;
  591     bool opt_quiet = false;
  592 
  593     /* Set up debug system */
  594     progname = argv[0];
  595     init_debug(progname,1);
  596 
  597     /* Make sure that the system was compiled properly */
  598     if(sizeof(struct be13::ip4)!=20 || sizeof(struct be13::tcphdr)!=20){
  599     fprintf(stderr,"COMPILE ERROR.\n");
  600     fprintf(stderr,"  sizeof(struct ip)=%d; should be 20.\n", (int)sizeof(struct be13::ip4));
  601     fprintf(stderr,"  sizeof(struct tcphdr)=%d; should be 20.\n", (int)sizeof(struct be13::tcphdr));
  602     fprintf(stderr,"CANNOT CONTINUE\n");
  603     exit(1);
  604     }
  605 
  606     bool trailing_input_list = false;
  607     int arg;
  608     while ((arg = getopt_long(argc, argv, "aA:Bb:cCd:DE:e:E:F:f:gHhIi:lL:m:o:pqR:r:S:sT:U:Vvw:x:X:z:ZK0J", longopts, NULL)) != EOF) {
  609     switch (arg) {
  610     case 'a':
  611         demux.opt.post_processing = true;
  612         demux.opt.opt_md5 = true;
  613             be13::plugin::scanners_enable_all();
  614         break;
  615 
  616     case 'A':
  617         fprintf(stderr,"-AH has been deprecated. Just use -a\n");
  618         break;
  619 
  620     case 'b':
  621         demux.opt.max_bytes_per_flow = atoi(optarg);
  622         if(debug > 1) {
  623         std::cout << "capturing max of " << demux.opt.max_bytes_per_flow << " bytes per flow." << std::endl;
  624         }
  625         break;
  626     case 'B':
  627             force_binary_output = true;
  628         demux.opt.output_strip_nonprint  = false;   DEBUG(10) ("converting non-printable characters to '.'");
  629         break;
  630     case 'C':
  631         demux.opt.console_output  = true;   DEBUG(10) ("printing packets to console only");
  632         demux.opt.suppress_header = 1;  DEBUG(10) ("packet header dump suppressed");
  633         break;
  634     case 'c':
  635         demux.opt.console_output = true;    DEBUG(10) ("printing packets to console only");
  636         break;
  637     case '0':
  638         demux.opt.console_output_nonewline = true;
  639         break;
  640     case 'd':
  641         if ((debug = atoi(optarg)) < 0) {
  642         debug = DEFAULT_DEBUG_LEVEL;
  643         DEBUG(1) ("warning: -d flag with 0 debug level '%s'", optarg);
  644         }
  645         break;
  646     case 'D':
  647         demux.opt.output_hex = true;DEBUG(10) ("Console output in hex");
  648         demux.opt.output_strip_nonprint = false;    DEBUG(10) ("Will not convert non-printablesto '.'");
  649         break;
  650     case 'E':
  651         be13::plugin::scanners_disable_all();
  652         be13::plugin::scanners_enable(optarg);
  653         break;
  654         case 'e':
  655             be13::plugin::scanners_enable(optarg);
  656             demux.opt.post_processing = true; // enable post processing if anything is turned on
  657             break;
  658     case 'F':
  659         for(const char *cc=optarg;*cc;cc++){
  660         switch(*cc){
  661         case 'c': replace(flow::filename_template,"%c","%C"); break;
  662                 case 'k': flow::filename_template = "%K/" + flow::filename_template; break;
  663                 case 'm': flow::filename_template = "%M000-%M999/%M%K/" + flow::filename_template; break;
  664                 case 'g': flow::filename_template = "%G000000-%G999999/%G%M000-%G%M999/%G%M%K/" + flow::filename_template; break;
  665         case 't': flow::filename_template = "%tT" + flow::filename_template; break;
  666         case 'T': flow::filename_template = "%T"  + flow::filename_template; break;
  667         case 'X': demux.opt.store_output = false;break;
  668         case 'M': demux.opt.opt_md5 = true;break;
  669         default:
  670             fprintf(stderr,"-F invalid format specification '%c'\n",*cc);
  671         }
  672         }
  673         break;
  674     case 'f':
  675         {
  676             int mnew = atoi(optarg);
  677             DEBUG(1)("changing max_fds from %d to %d",demux.max_fds,mnew);
  678             demux.max_fds = mnew;
  679         break;
  680         }
  681     case 'i': device = std::string(optarg); break;
  682     case 'I':
  683         DEBUG(10) ("creating packet index files");
  684         demux.opt.output_packet_index = true;
  685         break;
  686     case 'g':
  687         demux.opt.use_color  = 1;
  688         DEBUG(10) ("using colors");
  689         break;
  690         case 'l': trailing_input_list = true; break;
  691     case 'J':
  692         demux.opt.output_json = true;
  693         break;
  694     case 'K':;
  695         demux.opt.output_pcap = true;
  696         demux.alter_processing_core();
  697         break;
  698     case 'L': lockname = optarg; break;
  699     case 'm':
  700         demux.opt.max_seek = atoi(optarg);
  701         DEBUG(10) ("max_seek set to %d",demux.opt.max_seek); break;
  702     case 'o':
  703             demux.outdir = optarg;
  704             flow::outdir = optarg;
  705             break;
  706     case 'p': opt_no_promisc = true; DEBUG(10) ("NOT turning on promiscuous mode"); break;
  707         case 'q': opt_quiet = true; break;
  708     case 'R': Rfiles.push_back(optarg); break;
  709     case 'r': rfiles.push_back(optarg); break;
  710         case 'S':
  711         {
  712         std::vector<std::string> params = split(optarg,'=');
  713         if(params.size()!=2){
  714             std::cerr << "Invalid paramter: " << optarg << "\n";
  715             exit(1);
  716         }
  717         be_config.namevals[params[0]] = params[1];
  718         continue;
  719         }
  720 
  721     case 's':
  722             demux.opt.output_strip_nonprint = 1; DEBUG(10) ("converting non-printable characters to '.'");
  723             break;
  724     case 'T':
  725             flow::filename_template = optarg;
  726             if(flow::filename_template.find("%c")==std::string::npos){
  727                 flow::filename_template += std::string("%C%c"); // append %C%c if not present
  728             }
  729             break;
  730         case 'U': tcpflow_droproot_username = optarg; break;
  731     case 'V': std::cout << PACKAGE_NAME << " " << PACKAGE_VERSION << "\n"; exit (1);
  732     case 'v': debug = 10; break;
  733         case 'w': opt_unk_packets = optarg;break;
  734     case 'x': be13::plugin::scanners_disable(optarg);break;
  735     case 'X': reportfilename = optarg;break;
  736         case 'z': tcpflow_chroot_dir = optarg; break;
  737     case 'Z': demux.opt.gzip_decompress = 0; break;
  738     case 'H': opt_Help += 1; break;
  739     case 'h': opt_help += 1; break;
  740     default:
  741         DEBUG(1) ("error: unrecognized switch '%c'", arg);
  742         opt_help += 1;
  743         break;
  744     }
  745     }
  746 
  747     if(tcpflow_chroot_dir && !tcpflow_droproot_username){
  748         err(1,"-z option requires -U option");
  749     }
  750 
  751     argc -= optind;
  752     argv += optind;
  753 
  754 
  755     /* Load all the scanners and enable the ones we care about */
  756     scanner_info si;
  757     si.config = &be_config;
  758 
  759     si.get_config("enable_report",&opt_enable_report,"Enable report.xml");
  760     be13::plugin::load_scanners(scanners_builtin,be_config);
  761 
  762     if(opt_Help){
  763         be13::plugin::info_scanners(true,true,scanners_builtin,'e','x');
  764         exit(0);
  765     }
  766 
  767     if(opt_help) {
  768         usage(opt_help);
  769         exit(0);
  770     }
  771 
  772 
  773     if(demux.opt.post_processing && !demux.opt.store_output){
  774         std::cerr << "ERROR: post_processing currently requires storing output.\n";
  775         exit(1);
  776     }
  777 
  778     if(demux.opt.opt_md5) be13::plugin::scanners_enable("md5");
  779     be13::plugin::scanners_process_enable_disable_commands();
  780 
  781     /* If there is no report filename, call it report.xml in the output directory */
  782     if( reportfilename.size()==0 ){
  783     reportfilename = demux.outdir + "/" + DEFAULT_REPORT_FILENAME;
  784     }
  785 
  786     /* remaining arguments are either an input list (-l flag) or a pcap expression (default) */
  787     std::string expression = "";
  788     if(trailing_input_list) {
  789         for(int ii = 0; ii < argc; ii++) {
  790             rfiles.push_back(argv[ii]);
  791         }
  792     }
  793     else {
  794         /* get the user's expression out of remainder of the arg... */
  795         for(int i=0;i<argc;i++){
  796             if(expression.size()>0) expression+=" ";
  797             expression += argv[i];
  798         }
  799     }
  800 
  801     /* More option processing */
  802 
  803     /* was a semaphore provided for the lock? */
  804     if(lockname){
  805 #if defined(HAVE_SEMAPHORE_H) && defined(HAVE_PTHREAD_H)
  806     semlock = sem_open(lockname,O_CREAT,0777,1); // get the semaphore
  807 #else
  808     fprintf(stderr,"%s: attempt to create lock pthreads not present\n",argv[0]);
  809     exit(1);
  810 #endif
  811     }
  812 
  813     if(force_binary_output) demux.opt.output_strip_nonprint = false;
  814     /* make sure outdir is a directory. If it isn't, try to make it.*/
  815     struct stat stbuf;
  816     if(stat(demux.outdir.c_str(),&stbuf)==0){
  817     if(!S_ISDIR(stbuf.st_mode)){
  818         std::cerr << "outdir is not a directory: " << demux.outdir << "\n";
  819         exit(1);
  820     }
  821     } else {
  822     if(MKDIR(demux.outdir.c_str(),0777)){
  823         std::cerr << "cannot create " << demux.outdir << ": " << strerror(errno) << "\n";
  824         exit(1);
  825     }
  826     }
  827 
  828     std::string input_fname;
  829     if(rfiles.size() > 0) {
  830         input_fname = rfiles.at(0);
  831         if(rfiles.size() > 1) {
  832             input_fname += ssprintf(" + %d more", rfiles.size() - 1);
  833         }
  834     }
  835 
  836     /* report file specified? If so, open it.
  837      * Note: If we are going to chroot, we need apply the chroot prefix also,
  838      * but we need to open the file *now*.
  839      */
  840     if(reportfilename.size()>0 && opt_enable_report){
  841         if (tcpflow_chroot_dir){
  842             reportfilename = std::string(tcpflow_chroot_dir) + std::string("/") + reportfilename;
  843         }
  844         std::cerr << "reportfilename: " << reportfilename << "\n";
  845     xreport = new dfxml_writer(reportfilename,false);
  846     dfxml_create(*xreport,argc_original,argv_original);
  847     demux.xreport = xreport;
  848     }
  849     if(opt_unk_packets.size()>0){
  850         if(input_fname.size()==0){
  851             std::cerr << "currently the -w option requires the -r option\n";
  852             exit(1);
  853         }
  854         if(access(input_fname.c_str(),R_OK)) die("cannot read: %s: %s",input_fname.c_str(),strerror(errno));
  855         demux.save_unk_packets(opt_unk_packets,input_fname);
  856     }
  857 
  858 
  859     /* Debug prefix set? */
  860     std::string debug_prefix=progname;
  861     si.get_config("debug-prefix",&debug_prefix,"Prefix for debug output");
  862     init_debug(debug_prefix.c_str(),0);
  863 
  864     DEBUG(10) ("%s version %s ", PACKAGE_NAME, PACKAGE_VERSION);
  865 
  866     const char *name = device.c_str();
  867     if(input_fname.size()>0) name=input_fname.c_str();
  868     if(name==0) name="<default>";
  869 
  870     feature_file_names_t feature_file_names;
  871     be13::plugin::get_scanner_feature_file_names(feature_file_names);
  872     feature_recorder_set fs(feature_recorder_set::NO_ALERT,be_hash,name,demux.outdir);
  873     fs.init(feature_file_names);
  874     the_fs   = &fs;
  875     demux.fs = &fs;
  876 
  877     si.get_config("tdelta",&datalink_tdelta,"Time offset for packets");
  878     si.get_config("packet-buffer-timeout", &packet_buffer_timeout, "Time in milliseconds between each callback from libpcap");
  879 
  880     /* Record the configuration */
  881     if(xreport){
  882         xreport->push("configuration");
  883         xreport->pop();         // configuration
  884         xreport->xmlout("tdelta",datalink_tdelta);
  885     }
  886 
  887 
  888 
  889     /* Process r files and R files */
  890     int exit_val = 0;
  891     if(xreport){
  892         xreport->push("configuration");
  893     }
  894     if(rfiles.size()==0 && Rfiles.size()==0){
  895     /* live capture */
  896     demux.start_new_connections = true;
  897         int err = process_infile(demux,expression,device,"");
  898         if (err < 0) {
  899             exit_val = 1;
  900         }
  901         input_fname = device;
  902     }
  903     else {
  904     /* first pick up the new connections with -r */
  905     demux.start_new_connections = true;
  906     for(std::vector<std::string>::const_iterator it=rfiles.begin();it!=rfiles.end();it++){
  907         int err = process_infile(demux,expression,device,*it);
  908         if (err < 0) {
  909             exit_val = 1;
  910         }
  911     }
  912     /* now pick up the outstanding connection with -R, but don't start new connections */
  913     demux.start_new_connections = false;
  914     for(std::vector<std::string>::const_iterator it=Rfiles.begin();it!=Rfiles.end();it++){
  915         int err = process_infile(demux,expression,device,*it);
  916         if (err < 0) {
  917             exit_val = 1;
  918         }
  919     }
  920     }
  921 
  922     /* -1 causes pcap_loop to loop forever, but it finished when the input file is exhausted. */
  923 
  924     DEBUG(2)("Open FDs at end of processing:      %d",(int)demux.open_flows.size());
  925     DEBUG(2)("demux.max_open_flows:               %d",(int)demux.max_open_flows);
  926     DEBUG(2)("Flow map size at end of processing: %d",(int)demux.flow_map.size());
  927     DEBUG(2)("Flows seen:                         %d",(int)demux.flow_counter);
  928 
  929     int open_fds = (int)demux.open_flows.size();
  930     int flow_map_size = (int)demux.flow_map.size();
  931 
  932     demux.remove_all_flows();   // empty the map to capture the state
  933     std::stringstream ss;
  934     be13::plugin::phase_shutdown(fs,xreport ? &ss : 0);
  935 
  936     /*
  937      * Note: funny formats below are a result of mingw problems with PRId64.
  938      */
  939     const std::string total_flow_processed("Total flows processed: %" PRId64);
  940     const std::string total_packets_processed("Total packets processed: %" PRId64);
  941 
  942     DEBUG(2)(total_flow_processed.c_str(),demux.flow_counter);
  943     DEBUG(2)(total_packets_processed.c_str(),demux.packet_counter);
  944 
  945     if(xreport){
  946         xreport->pop();                 // fileobjects
  947         xreport->xmlout("summary",ss.str(),"",false);
  948         xreport->xmlout("open_fds_at_end",open_fds);
  949         xreport->xmlout("max_open_flows",demux.max_open_flows);
  950         xreport->xmlout("total_flows",demux.flow_counter);
  951         xreport->xmlout("flow_map_size",flow_map_size);
  952         xreport->xmlout("total_packets",demux.packet_counter);
  953     xreport->add_rusage();
  954     xreport->pop();                 // bulk_extractor
  955     xreport->close();
  956     delete xreport;
  957     }
  958 
  959     if(demux.flow_counter > tcpdemux::WARN_TOO_MANY_FILES){
  960         if(!opt_quiet){
  961             /* Start counting how many files we have in the output directory.
  962              * If we find more than 10,000, print the warning, and keep counting...
  963              */
  964             uint64_t filecount=0;
  965             DIR *dirp = opendir(demux.outdir.c_str());
  966             if(dirp){
  967                 struct dirent *dp=0;
  968                 while((dp=readdir(dirp))!=NULL){
  969                     filecount++;
  970                     if(filecount==10000){
  971                         std::cerr << "*** tcpflow WARNING:\n";
  972                         std::cerr << "*** Modern operating systems do not perform well \n";
  973                         std::cerr << "*** with more than 10,000 entries in a directory.\n";
  974                         std::cerr << "***\n";
  975                     }
  976                 }
  977                 closedir(dirp);
  978             }
  979             if(filecount>=10000){
  980                 std::cerr << "*** tcpflow created " << filecount
  981                           << " files in output directory " << demux.outdir << "\n";
  982                 std::cerr << "***\n";
  983                 std::cerr << "*** Next time, specify command-line options: -Fk , -Fm , or -Fg \n";
  984                 std::cerr << "*** This will automatically bin output into subdirectories.\n";
  985                 std::cerr << "*** type 'tcpflow -hhh' for more information.\n";
  986             }
  987         }
  988     }
  989 
  990     exit(exit_val);                     // return(0) causes crash on Windows
  991 }