tcpflow  1.6.1
About: tcpflow is a TCP/IP packet demultiplexer that captures data transmitted as part of TCP connections (flows), and stores the data in a way that is convenient for protocol analysis and debugging.
  Fossies Dox: tcpflow-1.6.1.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

feature_recorder_set.h
Go to the documentation of this file.
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 #ifndef FEATURE_RECORDER_SET_H
3 #define FEATURE_RECORDER_SET_H
4 
5 #include "feature_recorder.h"
6 #include "cppmutex.h"
8 #include "dfxml/src/hash_t.h"
10 #include <map>
11 #include <set>
12 
13 /** \addtogroup internal_interfaces
14  * @{
15  */
16 /** \file */
17 
18 /**
19  * \class feature_recorder_set
20  * The feature_recorder_set is an object that controls output. It knows where the output goes (outdir),
21  * the various feature recorders that write to that output, and provides for synchronization.
22  * It also has the factory method for new feature_recorders. Therefore if you want a different feature_recorder,
23  * this set should be subclassed as well.
24  */
25 
26 typedef std::map<std::string,class feature_recorder *> feature_recorder_map;
27 typedef std::set<std::string>feature_file_names_t;
29  // neither copying nor assignment is implemented
33  atomic_set<std::string> seen_set; // hex hash values of pages that have been seen
34  const std::string input_fname; // input file
35  const std::string outdir; // where output goes
36  feature_recorder_map frm; // map of feature recorders, by name; TK-replace with an atomic_set
37  mutable cppmutex Mscanner_stats; // locks frm and scanner_stats_map
38  histogram_defs_t histogram_defs; // histograms that are to be created.
41 public:
42  BEAPI_SQLITE3 *db3; // opened in SQLITE_OPEN_FULLMUTEX mode
43  virtual void heartbeat(){}; // called at a regular basis
44  struct hash_def {
45  hash_def(std::string name_,std::string (*func_)(const uint8_t *buf,const size_t bufsize)):name(name_),func(func_){};
46  std::string name; // name of hash
47  std::string (*func)(const uint8_t *buf,const size_t bufsize); // hash function
48  };
49  struct pstats {
50  double seconds;
51  uint64_t calls;
52  };
53  /** create an emptry feature recorder set. If disabled, create a disabled recorder. */
54  feature_recorder_set(uint32_t flags_,const hash_def &hasher_,
55  const std::string &input_fname_,const std::string &outdir_);
56 
57  typedef std::map<std::string,struct pstats> scanner_stats_map;
58 
59  const word_and_context_list *alert_list; /* shold be flagged */
60  const word_and_context_list *stop_list; /* should be ignored */
62 
63  const hash_def &hasher; // function for hashing; specified at creation
64  static hash_def null_hasher; // a default hasher available for all to use (it doesn't hash)
65 
66 
67  static const std::string ALERT_RECORDER_NAME; // the name of the alert recorder
68  static const std::string DISABLED_RECORDER_NAME; // the fake disabled feature recorder
69  static const std::string NO_INPUT; // 'filename' indicator that the FRS has no input file
70  static const std::string NO_OUTDIR; // 'dirname' indicator that the FRS produces no file output
71 
72  /* flags */
73  static const uint32_t ONLY_ALERT = 0x01; // always return the alert recorder
74  static const uint32_t SET_DISABLED = 0x02; // the set is effectively disabled; for path-printer
75  static const uint32_t CREATE_STOP_LIST_RECORDERS= 0x04; //
76  static const uint32_t MEM_HISTOGRAM = 0x20; // enable the in-memory histogram
77  static const uint32_t ENABLE_SQLITE3_RECORDERS = 0x40; // save features to an SQLITE3 databse
78  static const uint32_t DISABLE_FILE_RECORDERS = 0x80; // do not save features to file-based recorders
79  static const uint32_t NO_ALERT = 0x100; // no alert recorder
80 
82  for(feature_recorder_map::iterator i = frm.begin();i!=frm.end();i++){
83  delete i->second;
84  }
85  db_close();
86  }
87 
88  std::string get_input_fname() const {return input_fname;}
89  virtual const std::string &get_outdir() const { return outdir;}
90  void set_stop_list(const word_and_context_list *alist){stop_list=alist;}
91  void set_alert_list(const word_and_context_list *alist){alert_list=alist;}
92 
93 
94  /** Initialize a feature_recorder_set. Previously this was a constructor, but it turns out that
95  * virtual functions for the create_name_factory aren't honored in constructors.
96  *
97  * init() is called after all of the scanners have been loaded. It
98  * tells each feature file about its histograms (among other
99  * things)
100  */
101  void init(const feature_file_names_t &feature_files);
102 
103  void flush_all();
104  void close_all();
105  bool has_name(std::string name) const; /* does the named feature exist? */
106 
107  /* flags */
108  void set_flag(uint32_t f);
109  void unset_flag(uint32_t f);
110  bool flag_set(uint32_t f) const {return flags & f;}
111  bool flag_notset(uint32_t f) const {return !(flags & f);}
112  uint32_t get_flags() const {return flags;}
113 
114  typedef void (*xml_notifier_t)(const std::string &xmlstring);
115  void add_histogram(const histogram_def &def); // adds it to a local set or to the specific feature recorder
116  void dump_histograms(void *user,feature_recorder::dump_callback_t cb, xml_notifier_t xml_error_notifier) const;
117  virtual feature_recorder *create_name_factory(const std::string &name_);
118  virtual void create_name(const std::string &name,bool create_stop_also);
119 
120  void add_stats(const std::string &bucket,double seconds);
121  typedef int (*stat_callback_t)(void *user,const std::string &name,uint64_t calls,double seconds);
122  void get_stats(void *user,stat_callback_t stat_callback) const;
123  void dump_name_count_stats(dfxml_writer &writer) const;
124 
125  /****************************************************************
126  *** SQLite3 interface
127  ****************************************************************/
128 
129 
130  virtual void db_send_sql(BEAPI_SQLITE3 *db3,const char **stmts, ...) ;
131  virtual BEAPI_SQLITE3 *db_create_empty(const std::string &name) ;
132  void db_create_table(const std::string &name) ;
133  void db_create() ;
134  void db_transaction_begin() ;
135  void db_transaction_commit() ; // commit current transaction
136  void db_close() ; //
137 
138  /****************************************************************
139  *** External Functions
140  ****************************************************************/
141 
142 
143  // Management of previously seen data
144  virtual bool check_previously_processed(const uint8_t *buf,size_t bufsize);
145 
146  // NOTE:
147  // only virtual functions may be called by plugins!
148  virtual feature_recorder *get_name(const std::string &name) const;
149  virtual feature_recorder *get_alert_recorder() const;
150  virtual void get_feature_file_list(std::vector<std::string> &ret); // clears ret and fills with a list of feature file names
151 };
152 
153 
154 #endif
int() dump_callback_t(void *user, const feature_recorder &fr, const histogram_def &def, const std::string &feature, const uint64_t &count)
std::set< histogram_def > histogram_defs_t
#define BEAPI_SQLITE3
std::string(* func)(const uint8_t *buf, const size_t bufsize)
void add_stats(const std::string &bucket, double seconds)
bool flag_set(uint32_t f) const
static const uint32_t DISABLE_FILE_RECORDERS
void dump_name_count_stats(dfxml_writer &writer) const
const std::string outdir
void add_histogram(const histogram_def &def)
std::map< std::string, class feature_recorder * > feature_recorder_map
feature_recorder_set & operator=(const feature_recorder_set &fs)
uint32_t get_flags() const
static const uint32_t SET_DISABLED
static const uint32_t NO_ALERT
const word_and_context_list * alert_list
std::map< std::string, struct pstats > scanner_stats_map
virtual feature_recorder * get_alert_recorder() const
void init(const feature_file_names_t &feature_files)
static const uint32_t ENABLE_SQLITE3_RECORDERS
static const std::string DISABLED_RECORDER_NAME
static const std::string NO_INPUT
void db_create_table(const std::string &name)
virtual bool check_previously_processed(const uint8_t *buf, size_t bufsize)
virtual feature_recorder * create_name_factory(const std::string &name_)
const std::string input_fname
void set_stop_list(const word_and_context_list *alist)
atomic_set< std::string > seen_set
void get_stats(void *user, stat_callback_t stat_callback) const
void set_alert_list(const word_and_context_list *alist)
static const uint32_t ONLY_ALERT
scanner_stats_map scanner_stats
void(* xml_notifier_t)(const std::string &xmlstring)
virtual void db_send_sql(void *db3, const char **stmts,...)
virtual void create_name(const std::string &name, bool create_stop_also)
static const uint32_t CREATE_STOP_LIST_RECORDERS
hash_def(std::string name_, std::string(*func_)(const uint8_t *buf, const size_t bufsize))
static const uint32_t MEM_HISTOGRAM
bool flag_notset(uint32_t f) const
feature_recorder_set(const feature_recorder_set &fs)
std::set< std::string > feature_file_names_t
static hash_def null_hasher
virtual feature_recorder * get_name(const std::string &name) const
static const std::string ALERT_RECORDER_NAME
virtual void get_feature_file_list(std::vector< std::string > &ret)
virtual const std::string & get_outdir() const
std::string get_input_fname() const
void dump_histograms(void *user, feature_recorder::dump_callback_t cb, xml_notifier_t xml_error_notifier) const
histogram_defs_t histogram_defs
feature_recorder_map frm
static const std::string NO_OUTDIR
int(* stat_callback_t)(void *user, const std::string &name, uint64_t calls, double seconds)
const word_and_context_list * stop_list
bool has_name(std::string name) const
virtual void * db_create_empty(const std::string &name)
const char * name
Definition: http_parser.c:465
flags
Definition: http_parser.h:216
unsigned int uint32_t
Definition: core.h:40
unsigned char uint8_t
Definition: util.h:6