tcpflow  1.6.1
About: tcpflow is a TCP/IP packet demultiplexer that captures data transmitted as part of TCP connections (flows), and stores the data in a way that is convenient for protocol analysis and debugging.
  Fossies Dox: tcpflow-1.6.1.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

feature_recorder_set.cpp
Go to the documentation of this file.
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 
3 #include "config.h"
4 #include "bulk_extractor_i.h"
5 #include "histogram.h"
6 
7 /****************************************************************
8  *** feature_recorder_set:
9  *** Manage the set of feature recorders.
10  *** Handles both file-based feature recorders and the SQLite3 feature recorder.
11  ****************************************************************/
12 
13 const std::string feature_recorder_set::ALERT_RECORDER_NAME = "alerts";
14 const std::string feature_recorder_set::DISABLED_RECORDER_NAME = "disabled";
15 const std::string feature_recorder_set::NO_INPUT = "<NO-INPUT>";
16 const std::string feature_recorder_set::NO_OUTDIR = "<NO-OUTDIR>";
17 
18 static std::string null_hasher_name("null");
19 static std::string null_hasher_func(const uint8_t *buf,size_t bufsize)
20 {
21  return std::string("0000000000000000");
22 }
23 
25 
26 /* Create an empty recorder with no outdir. */
28  const std::string &input_fname_,const std::string &outdir_):
29  flags(flags_),seen_set(),input_fname(input_fname_),
30  outdir(outdir_),
31  frm(),Mscanner_stats(),
32  histogram_defs(),
33  Min_transaction(),in_transaction(),db3(),
34  alert_list(),stop_list(),
35  scanner_stats(),hasher(hasher_)
36 {
37  if(flags & SET_DISABLED){
40  }
41 }
42 
43 /**
44  * Initialize a properly functioning feature recorder set.
45  * If disabled, create a disabled feature_recorder that can respond to functions as requested.
46  */
48 {
49  /* Make sure we can write to the outdir if one is provided */
50  if ((outdir != NO_OUTDIR) && (access(outdir.c_str(),W_OK)!=0)) {
51  throw new std::invalid_argument("output directory not writable");
52  }
53 
55  db_create();
56  }
57 
58  if (flag_notset(NO_ALERT)) {
59  create_name(feature_recorder_set::ALERT_RECORDER_NAME,false); // make the alert recorder
60  }
61 
62  /* Create the requested feature files */
63  for(std::set<std::string>::const_iterator it=feature_files.begin();it!=feature_files.end();it++){
65  }
66 }
67 
68 /** Flush all of the feature recorder files.
69  * Typically done at the end of an sbuf.
70  */
72 {
73  for(feature_recorder_map::iterator i = frm.begin();i!=frm.end();i++){
74  i->second->flush();
75  }
76 }
77 
79 {
80  for(feature_recorder_map::iterator i = frm.begin();i!=frm.end();i++){
81  i->second->close();
82  }
85  }
86 }
87 
88 
89 bool feature_recorder_set::has_name(std::string name) const
90 {
91  return frm.find(name) != frm.end();
92 }
93 
94 /*
95  * Gets a feature_recorder_set.
96  */
98 {
99  const std::string *thename = &name;
100  if(flags & SET_DISABLED){ // if feature recorder set is disabled, return the disabled recorder.
102  }
103 
104  if(flags & ONLY_ALERT){
106  }
107 
109  feature_recorder_map::const_iterator it = frm.find(*thename);
110  if(it!=frm.end()) return it->second;
111  return(0); // feature recorder does not exist
112 }
113 
114 
116 {
117  return new feature_recorder(*this,name_);
118 }
119 
120 
121 /*
122  * Create a named feature recorder, any associated stoplist recorders, and open the files
123  */
124 void feature_recorder_set::create_name(const std::string &name,bool create_stop_recorder)
125 {
126  if(frm.find(name)!=frm.end()){
127  std::cerr << "create_name: feature recorder '" << name << "' already exists\n";
128  return;
129  }
130 
132 
133  frm[name] = fr;
134  if (create_stop_recorder){
135  std::string name_stopped = name+"_stopped";
136 
137  feature_recorder *fr_stopped = create_name_factory(name_stopped);
138  fr->set_stop_list_recorder(fr_stopped);
139  frm[name_stopped] = fr_stopped;
140  }
141 }
142 
144 {
145  if (flag_set(NO_ALERT)) return 0;
146 
148 }
149 
150 
151 /*
152  * uses md5 to determine if a block was prevously seen.
153  */
155 {
156  std::string md5 = md5_generator::hash_buf(buf,bufsize).hexdigest();
158 }
159 
160 void feature_recorder_set::add_stats(const std::string &bucket,double seconds)
161 {
163  struct pstats &p = scanner_stats[bucket]; // get the location of the stats
164  p.seconds += seconds;
165  p.calls ++;
166 }
167 
168 /*
169  * Send the stats to a callback; if the callback returns less than 0, abort.
170  */
171 void feature_recorder_set::get_stats(void *user,stat_callback_t stat_callback) const
172 {
173  for(scanner_stats_map::const_iterator it = scanner_stats.begin();it!=scanner_stats.end();it++){
174  if((*stat_callback)(user,(*it).first,(*it).second.calls,(*it).second.seconds)<0){
175  break;
176  }
177  }
178 }
179 
180 void feature_recorder_set::dump_name_count_stats(dfxml_writer &writer) const
181 {
183  writer.push("feature_files");
184  for(feature_recorder_map::const_iterator ij = frm.begin(); ij != frm.end(); ij++){
185  writer.set_oneline(true);
186  writer.push("feature_file");
187  writer.xmlout("name",ij->second->name);
188  writer.xmlout("count",ij->second->count());
189  writer.pop();
190  writer.set_oneline(false);
191  }
192 }
193 
194 
196 {
197  if(f & MEM_HISTOGRAM){
198  if(flags & MEM_HISTOGRAM){
199  std::cerr << "MEM_HISTOGRAM flag cannot be set twice\n";
200  assert(0);
201  }
202  /* Create the in-memory histograms for all of the feature recorders */
203  for(feature_recorder_map::const_iterator it = frm.begin(); it!=frm.end(); it++){
204  feature_recorder *fr = it->second;
206  }
207  }
208  flags |= f;
209 }
210 
212 {
213  if(f & MEM_HISTOGRAM){
214  std::cerr << "MEM_HISTOGRAM flag cannot be cleared\n";
215  assert(0);
216  }
217  flags &= ~f;
218 }
219 
220 /****************************************************************
221  *** PHASE HISTOGRAM (formerly phase 3): Create the histograms
222  ****************************************************************/
223 
224 /**
225  * We now have three kinds of histograms:
226  * 1 - Traditional post-processing histograms specified by the histogram library
227  1a - feature-file based traditional ones
228  1b - SQL-based traditional ones.
229  * 2 - In-memory histograms (used primarily by beapi)
230  */
231 
232 
234 {
235  feature_recorder *fr = get_name(def.feature);
236  if(fr) fr->add_histogram(def);
237 }
238 
240  feature_recorder_set::xml_notifier_t xml_error_notifier) const
241 {
242  /* Ask each feature recorder to dump its histograms */
243  for(feature_recorder_map::const_iterator it = frm.begin(); it!=frm.end(); it++){
244  feature_recorder *fr = it->second;
245  fr->dump_histograms(user,cb,xml_error_notifier);
246  }
247 }
248 
249 void feature_recorder_set::get_feature_file_list(std::vector<std::string> &ret)
250 {
251  for(feature_recorder_map::const_iterator it = frm.begin(); it!=frm.end(); it++){
252  ret.push_back(it->first);
253  }
254 }
bool check_for_presence_and_insert(const TYPE &s)
virtual void add_histogram(const histogram_def &def)
int() dump_callback_t(void *user, const feature_recorder &fr, const histogram_def &def, const std::string &feature, const uint64_t &count)
virtual void dump_histograms(void *user, feature_recorder::dump_callback_t cb, xml_notifier_t xml_error_notifier) const
static const int FLAG_DISABLED
void set_stop_list_recorder(class feature_recorder *fr)
static hash__< md, SIZE > hash_buf(const uint8_t *buf, size_t bufsize)
Definition: hash_t.h:264
static std::string null_hasher_name("null")
static std::string null_hasher_func(const uint8_t *buf, size_t bufsize)
void add_stats(const std::string &bucket, double seconds)
bool flag_set(uint32_t f) const
void dump_name_count_stats(dfxml_writer &writer) const
const std::string outdir
void add_histogram(const histogram_def &def)
static const uint32_t SET_DISABLED
static const uint32_t NO_ALERT
virtual feature_recorder * get_alert_recorder() const
void init(const feature_file_names_t &feature_files)
static const uint32_t ENABLE_SQLITE3_RECORDERS
static const std::string DISABLED_RECORDER_NAME
static const std::string NO_INPUT
virtual bool check_previously_processed(const uint8_t *buf, size_t bufsize)
virtual feature_recorder * create_name_factory(const std::string &name_)
atomic_set< std::string > seen_set
void get_stats(void *user, stat_callback_t stat_callback) const
static const uint32_t ONLY_ALERT
scanner_stats_map scanner_stats
void(* xml_notifier_t)(const std::string &xmlstring)
virtual void create_name(const std::string &name, bool create_stop_also)
static const uint32_t CREATE_STOP_LIST_RECORDERS
static const uint32_t MEM_HISTOGRAM
bool flag_notset(uint32_t f) const
feature_recorder_set(const feature_recorder_set &fs)
std::set< std::string > feature_file_names_t
static hash_def null_hasher
virtual feature_recorder * get_name(const std::string &name) const
static const std::string ALERT_RECORDER_NAME
virtual void get_feature_file_list(std::vector< std::string > &ret)
void dump_histograms(void *user, feature_recorder::dump_callback_t cb, xml_notifier_t xml_error_notifier) const
feature_recorder_map frm
static const std::string NO_OUTDIR
bool has_name(std::string name) const
const char * name
Definition: http_parser.c:465
flags
Definition: http_parser.h:216
unsigned int uint32_t
Definition: core.h:40
const std::string feature
unsigned char uint8_t
Definition: util.h:6