tcpflow  1.6.1
About: tcpflow is a TCP/IP packet demultiplexer that captures data transmitted as part of TCP connections (flows), and stores the data in a way that is convenient for protocol analysis and debugging.
  Fossies Dox: tcpflow-1.6.1.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

tcpdemux.h
Go to the documentation of this file.
1 #ifndef TCPDEMUX_H
2 #define TCPDEMUX_H
3 
4 /**
5  * tcpdemux.h
6  *
7  * a tcpip demultiplier.
8  *
9  * Defines the basic classes used by the tcpflow program. This includes:
10  * - IP, TCP and UDP structures
11  * - class ipaddr - IP address (IPv4 and IPv6)
12  * - class flow_addr - The flow address (source addr & port; dest addr & port; family)
13  * - class flow - All of the information for a flow that's being tracked
14  * - class tcp_header_t - convenience class for working with TCP headers
15  * - class tcpip - A one-sided TCP implementation
16  * - class tcpdemux - Processes individual packets, identifies flows,
17  * and creates tcpip objects as required
18  */
19 
20 #include "pcap_writer.h"
21 #include "dfxml/src/dfxml_writer.h"
22 #include "dfxml/src/hash_t.h"
23 
24 #if defined(HAVE_SQLITE3_H)
25 #include <sqlite3.h>
26 #endif
27 
28 #if defined(HAVE_UNORDERED_MAP)
29 # include <unordered_map>
30 # include <unordered_set>
31 # undef HAVE_TR1_UNORDERED_MAP // be sure we don't use it
32 #else
33 # if defined(HAVE_TR1_UNORDERED_MAP)
34 # include <tr1/unordered_map>
35 # include <tr1/unordered_set>
36 # else
37 # error Requires <unordered_map> or <tr1/unordered_map>
38 # endif
39 #endif
40 
41 #include <queue>
42 #include "intrusive_list.h"
43 
44 /**
45  * the tcp demultiplixer
46  * This is a singleton class; we only need a single demultiplexer.
47  */
48 class tcpdemux {
49  /* see http://mikecvet.wordpress.com/tag/hashing/ */
50  typedef struct {
51  long operator() (const flow_addr &k) const {return k.hash(); }
52  } flow_addr_hash;
53 
54  typedef struct {
55  bool operator() (const flow_addr &x, const flow_addr &y) const { return x==y;}
56  } flow_addr_key_eq;
57 
58 #ifdef HAVE_TR1_UNORDERED_MAP
59  typedef std::tr1::unordered_map<flow_addr,tcpip *,flow_addr_hash,flow_addr_key_eq> flow_map_t; // active flows
60  typedef std::tr1::unordered_map<flow_addr,saved_flow *,flow_addr_hash,flow_addr_key_eq> saved_flow_map_t; // flows that have been saved
61  typedef std::tr1::unordered_map<flow_addr,sparse_saved_flow *,flow_addr_hash,flow_addr_key_eq> sparse_saved_flow_map_t; // flows ctxt caching for pcap dissection
62 #else
63  typedef std::unordered_map<flow_addr,tcpip *,flow_addr_hash,flow_addr_key_eq> flow_map_t; // active flows
64  typedef std::unordered_map<flow_addr,saved_flow *,flow_addr_hash,flow_addr_key_eq> saved_flow_map_t; // flows that have been saved
65  typedef std::unordered_map<flow_addr,sparse_saved_flow *,flow_addr_hash,flow_addr_key_eq> sparse_saved_flow_map_t; // flows ctxt caching for pcap dissection
66 #endif
67  typedef std::vector<class saved_flow *> saved_flows_t; // needs to be ordered
68 
69 
70  tcpdemux();
71 #ifdef HAVE_SQLITE3
72  sqlite3 *db;
73  sqlite3_stmt *insert_flow;
74 #endif
76 
77  /* facility logic hinge */
78  int (tcpdemux::*tcp_processor)(const ipaddr &src, const ipaddr &dst,sa_family_t family,
79  const u_char *tcp_data, uint32_t tcp_length,
80  const be13::packet_info &pi);
81 
82 public:
84  static std::string tcp_cmd; // command to run on each tcp flow
85  static int tcp_subproc_max; // how many subprocesses are we allowed?
86  static int tcp_subproc; // how many do we currently have?
87  static int tcp_alert_fd;
88 
89  static unsigned int get_max_fds(void); // returns the max
90  virtual ~tcpdemux(){
91  delete xreport;
92  delete pwriter;
93  }
94 
95  /* The pure options class means we can add new options without having to modify the tcpdemux constructor. */
96  class options {
97  public:;
98  enum { MAX_SEEK=1024*1024*16 };
100  store_output(true),opt_md5(false),
101  post_processing(false),gzip_decompress(true),
102  max_bytes_per_flow(-1),
104  output_strip_nonprint(true),output_json(false),
105  output_pcap(false),output_hex(false),use_color(0),
107  }
110  bool store_output; // do we output?
111  bool opt_md5; // do we calculate MD5 on DFXML output?
112  bool post_processing; // decode headers after tcp connection closes
121  bool use_color;
122  bool output_packet_index; // Generate a packet index file giving the timestamp and location
123  // bytes written to the flow file.
124  int32_t max_seek; // signed becuase we compare with abs()
125  };
126 
127  enum { WARN_TOO_MANY_FILES=10000}; // warn if more than this number of files in a directory
128 
129  std::string outdir; /* output directory */
130  uint64_t flow_counter; // how many flows have we seen?
131  uint64_t packet_counter; // monotomically increasing
132  dfxml_writer *xreport; // DFXML output file
133  pcap_writer *pwriter; // where we should write packets
134  unsigned int max_open_flows; // how large did it ever get?
135  unsigned int max_fds; // maximum number of file descriptors for this tcpdemux
136  uint64_t unique_id; // next unique id to assign
137 
138  flow_map_t flow_map; // db of open tcpip objects, indexed by flow
139  intrusive_list<tcpip> open_flows; // the tcpip flows with open files in access order
140 
141  saved_flow_map_t saved_flow_map; // db of saved flows, indexed by flow
142  sparse_saved_flow_map_t flow_fd_cache_map; // db caching saved flows descriptors, indexed by flow
143  saved_flows_t saved_flows; // the flows that were saved
144  bool start_new_connections; // true if we should start new connections
145 
147  class feature_recorder_set *fs; // where features extracted from each flow should be stored
148 
149  static uint32_t max_saved_flows; // how many saved flows are kept in the saved_flow_map
150 
151  void alter_processing_core();
152  static tcpdemux *getInstance();
153 
154  /* Databse */
155 
156  void openDB(); // open the database file if we are using it in outdir directory.
157  void write_flow_record(const std::string &starttime,const std::string &endtime,
158  const std::string &src_ipn,const std::string &dst_ipn,
159  const std::string &mac_daddr,const std::string &mac_saddr,
160  uint64_t packets,uint16_t srcport,uint16_t dstport,
161  const std::string &hashdigest_md5);
162 
163 
164  void save_unk_packets(const std::string &wfname,const std::string &ifname);
165  // save unknown packets at this location
166  void post_process(tcpip *tcp); // just before closing; writes XML and closes fd
167 
168  /* management of open fds and in-process tcpip flows*/
170  void close_oldest_fd();
171  void remove_flow(const flow_addr &flow); // remove a flow from the database, closing open files if necessary
172  void remove_all_flows(); // stop processing all tcpip connections
173 
174  /* open a new file, closing an fd in the openflow database if necessary */
175  int retrying_open(const std::string &filename,int oflag,int mask);
176 
177  /* the flow database holds in-process tcpip connections */
179  tcpip *find_tcpip(const flow_addr &flow);
180 
181  /* saved flows are completed flows that we remember in case straggling packets
182  * show up. Remembering the flows lets us resolve the packets rather than creating
183  * new flows.
184  */
185  void save_flow(tcpip *);
186 
187  /** packet processing.
188  * Each returns 0 if processed, 1 if not processed, -1 if error.
189  */
190  int process_tcp(const ipaddr &src, const ipaddr &dst,sa_family_t family,
191  const u_char *tcp_data, uint32_t tcp_length,
192  const be13::packet_info &pi);
193  int dissect_tcp(const ipaddr &src, const ipaddr &dst,sa_family_t family,
194  const u_char *tcp_data, uint32_t tcp_length,
195  const be13::packet_info &pi);
196  int process_ip4(const be13::packet_info &pi);
197  int process_ip6(const be13::packet_info &pi);
198  int process_pkt(const be13::packet_info &pi);
199 private:;
200  /* These are not implemented */
201  tcpdemux(const tcpdemux &t);
202  tcpdemux &operator=(const tcpdemux &that);
203 
204 };
205 
206 
207 #endif
uint64_t hash() const
Definition: tcpip.h:103
Definition: tcpip.h:156
Definition: tcpip.h:25
uint32_t max_flows
Definition: tcpdemux.h:115
bool console_output_nonewline
Definition: tcpdemux.h:109
int64_t max_bytes_per_flow
Definition: tcpdemux.h:114
bool suppress_header
Definition: tcpdemux.h:116
bool output_strip_nonprint
Definition: tcpdemux.h:117
bool post_processing
Definition: tcpdemux.h:112
bool gzip_decompress
Definition: tcpdemux.h:113
bool output_packet_index
Definition: tcpdemux.h:122
int32_t max_seek
Definition: tcpdemux.h:124
void remove_flow(const flow_addr &flow)
Definition: tcpdemux.cpp:264
int retrying_open(const std::string &filename, int oflag, int mask)
Definition: tcpdemux.cpp:121
int dissect_tcp(const ipaddr &src, const ipaddr &dst, sa_family_t family, const u_char *tcp_data, uint32_t tcp_length, const be13::packet_info &pi)
Definition: tcpdemux.cpp:393
static int tcp_alert_fd
Definition: tcpdemux.h:87
tcpdemux(const tcpdemux &t)
void save_flow(tcpip *)
Definition: tcpdemux.cpp:367
intrusive_list< tcpip > open_flows
Definition: tcpdemux.h:139
int(tcpdemux::* tcp_processor)(const ipaddr &src, const ipaddr &dst, sa_family_t family, const u_char *tcp_data, uint32_t tcp_length, const be13::packet_info &pi)
Definition: tcpdemux.h:78
int process_ip4(const be13::packet_info &pi)
Definition: tcpdemux.cpp:697
@ WARN_TOO_MANY_FILES
Definition: tcpdemux.h:127
uint64_t packet_counter
Definition: tcpdemux.h:131
std::unordered_map< flow_addr, tcpip *, flow_addr_hash, flow_addr_key_eq > flow_map_t
Definition: tcpdemux.h:63
options opt
Definition: tcpdemux.h:146
std::vector< class saved_flow * > saved_flows_t
Definition: tcpdemux.h:67
static uint32_t tcp_timeout
Definition: tcpdemux.h:83
saved_flow_map_t saved_flow_map
Definition: tcpdemux.h:141
bool start_new_connections
Definition: tcpdemux.h:144
uint64_t unique_id
Definition: tcpdemux.h:136
static int tcp_subproc_max
Definition: tcpdemux.h:85
int process_tcp(const ipaddr &src, const ipaddr &dst, sa_family_t family, const u_char *tcp_data, uint32_t tcp_length, const be13::packet_info &pi)
Definition: tcpdemux.cpp:441
void close_tcpip_fd(tcpip *)
dfxml_writer * xreport
Definition: tcpdemux.h:132
int process_pkt(const be13::packet_info &pi)
Definition: tcpdemux.cpp:796
pcap_writer * flow_sorter
Definition: tcpdemux.h:75
tcpip * find_tcpip(const flow_addr &flow)
Definition: tcpdemux.cpp:144
void remove_all_flows()
Definition: tcpdemux.cpp:273
void alter_processing_core()
Definition: tcpdemux.cpp:47
std::string outdir
Definition: tcpdemux.h:129
tcpdemux()
Definition: tcpdemux.cpp:33
void post_process(tcpip *tcp)
Definition: tcpdemux.cpp:200
class feature_recorder_set * fs
Definition: tcpdemux.h:147
static int tcp_subproc
Definition: tcpdemux.h:86
int process_ip6(const be13::packet_info &pi)
Definition: tcpdemux.cpp:764
void save_unk_packets(const std::string &wfname, const std::string &ifname)
Definition: tcpdemux.cpp:358
tcpip * create_tcpip(const flow_addr &flow, be13::tcp_seq isn, const be13::packet_info &pi)
Definition: tcpdemux.cpp:170
unsigned int max_fds
Definition: tcpdemux.h:135
void close_oldest_fd()
Definition: tcpdemux.cpp:113
static unsigned int get_max_fds(void)
Definition: tcpdemux.cpp:293
std::unordered_map< flow_addr, sparse_saved_flow *, flow_addr_hash, flow_addr_key_eq > sparse_saved_flow_map_t
Definition: tcpdemux.h:65
tcpdemux & operator=(const tcpdemux &that)
virtual ~tcpdemux()
Definition: tcpdemux.h:90
static std::string tcp_cmd
Definition: tcpdemux.h:84
void write_flow_record(const std::string &starttime, const std::string &endtime, const std::string &src_ipn, const std::string &dst_ipn, const std::string &mac_daddr, const std::string &mac_saddr, uint64_t packets, uint16_t srcport, uint16_t dstport, const std::string &hashdigest_md5)
Definition: tcpdemux.cpp:93
pcap_writer * pwriter
Definition: tcpdemux.h:133
sparse_saved_flow_map_t flow_fd_cache_map
Definition: tcpdemux.h:142
uint64_t flow_counter
Definition: tcpdemux.h:130
flow_map_t flow_map
Definition: tcpdemux.h:138
static uint32_t max_saved_flows
Definition: tcpdemux.h:149
static tcpdemux * getInstance()
Definition: tcpdemux.cpp:103
saved_flows_t saved_flows
Definition: tcpdemux.h:143
void openDB()
Definition: tcpdemux.cpp:54
unsigned int max_open_flows
Definition: tcpdemux.h:134
std::unordered_map< flow_addr, saved_flow *, flow_addr_hash, flow_addr_key_eq > saved_flow_map_t
Definition: tcpdemux.h:64
Definition: tcpip.h:274
uint32_t tcp_seq
unsigned int uint32_t
Definition: core.h:40
unsigned short int sa_family_t
Definition: tcpip.h:16
unsigned short uint16_t
Definition: util.h:7