"Fossies" - the Fresh Open Source Software Archive

Member "tcpflow-1.6.1/src/be13_api/feature_recorder_set.cpp" (19 Feb 2021, 8205 Bytes) of package /linux/misc/tcpflow-1.6.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "feature_recorder_set.cpp" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 1.4.4_vs_1.4.5.

    1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
    2 
    3 #include "config.h"
    4 #include "bulk_extractor_i.h"
    5 #include "histogram.h"
    6 
    7 /****************************************************************
    8  *** feature_recorder_set:
    9  *** Manage the set of feature recorders.
   10  *** Handles both file-based feature recorders and the SQLite3 feature recorder.
   11  ****************************************************************/
   12 
   13 const std::string feature_recorder_set::ALERT_RECORDER_NAME = "alerts";
   14 const std::string feature_recorder_set::DISABLED_RECORDER_NAME = "disabled";
   15 const std::string feature_recorder_set::NO_INPUT = "<NO-INPUT>";
   16 const std::string feature_recorder_set::NO_OUTDIR = "<NO-OUTDIR>";
   17 
   18 static std::string null_hasher_name("null");
   19 static std::string null_hasher_func(const uint8_t *buf,size_t bufsize)
   20 {
   21     return std::string("0000000000000000");
   22 }
   23 
   24 feature_recorder_set::hash_def feature_recorder_set::null_hasher(null_hasher_name,null_hasher_func);
   25 
   26 /* Create an empty recorder with no outdir. */
   27 feature_recorder_set::feature_recorder_set(uint32_t flags_,const feature_recorder_set::hash_def &hasher_,
   28                                            const std::string &input_fname_,const std::string &outdir_):
   29     flags(flags_),seen_set(),input_fname(input_fname_),
   30     outdir(outdir_),
   31     frm(),Mscanner_stats(),
   32     histogram_defs(),
   33     Min_transaction(),in_transaction(),db3(),
   34     alert_list(),stop_list(),
   35     scanner_stats(),hasher(hasher_)
   36 {
   37     if(flags & SET_DISABLED){
   38         create_name(DISABLED_RECORDER_NAME,false);
   39         frm[DISABLED_RECORDER_NAME]->set_flag(feature_recorder::FLAG_DISABLED);
   40     }
   41 }
   42 
   43 /**
   44  * Initialize a properly functioning feature recorder set.
   45  * If disabled, create a disabled feature_recorder that can respond to functions as requested.
   46  */
   47 void feature_recorder_set::init(const feature_file_names_t &feature_files)
   48 {
   49     /* Make sure we can write to the outdir if one is provided */
   50     if ((outdir != NO_OUTDIR) && (access(outdir.c_str(),W_OK)!=0)) {
   51         throw new std::invalid_argument("output directory not writable");
   52     }
   53         
   54     if (flag_set(ENABLE_SQLITE3_RECORDERS)) {
   55         db_create();
   56     }
   57 
   58     if (flag_notset(NO_ALERT)) {
   59         create_name(feature_recorder_set::ALERT_RECORDER_NAME,false); // make the alert recorder
   60     }
   61 
   62     /* Create the requested feature files */
   63     for(std::set<std::string>::const_iterator it=feature_files.begin();it!=feature_files.end();it++){
   64         create_name(*it,flags & CREATE_STOP_LIST_RECORDERS);
   65     }
   66 }
   67 
   68 /** Flush all of the feature recorder files.
   69  * Typically done at the end of an sbuf.
   70  */
   71 void feature_recorder_set::flush_all()
   72 {
   73     for(feature_recorder_map::iterator i = frm.begin();i!=frm.end();i++){
   74         i->second->flush();
   75     } 
   76 }
   77 
   78 void feature_recorder_set::close_all()
   79 {
   80     for(feature_recorder_map::iterator i = frm.begin();i!=frm.end();i++){
   81         i->second->close();
   82     } 
   83     if ( flag_set(feature_recorder_set::ENABLE_SQLITE3_RECORDERS )) {
   84         db_transaction_commit();
   85     }
   86 }
   87 
   88 
   89 bool feature_recorder_set::has_name(std::string name) const
   90 {
   91     return frm.find(name) != frm.end();
   92 }
   93 
   94 /*
   95  * Gets a feature_recorder_set.
   96  */
   97 feature_recorder *feature_recorder_set::get_name(const std::string &name) const
   98 {
   99     const std::string *thename = &name;
  100     if(flags & SET_DISABLED){           // if feature recorder set is disabled, return the disabled recorder.
  101         thename = &feature_recorder_set::DISABLED_RECORDER_NAME;
  102     }
  103 
  104     if(flags & ONLY_ALERT){
  105         thename = &feature_recorder_set::ALERT_RECORDER_NAME;
  106     }
  107 
  108     cppmutex::lock lock(Mscanner_stats);
  109     feature_recorder_map::const_iterator it = frm.find(*thename);
  110     if(it!=frm.end()) return it->second;
  111     return(0);                          // feature recorder does not exist
  112 }
  113 
  114 
  115 feature_recorder *feature_recorder_set::create_name_factory(const std::string &name_)
  116 {
  117     return new feature_recorder(*this,name_);
  118 }
  119 
  120 
  121 /*
  122  * Create a named feature recorder, any associated stoplist recorders, and open the files
  123  */
  124 void feature_recorder_set::create_name(const std::string &name,bool create_stop_recorder) 
  125 {
  126     if(frm.find(name)!=frm.end()){
  127         std::cerr << "create_name: feature recorder '" << name << "' already exists\n";
  128         return;
  129     }
  130 
  131     feature_recorder *fr = create_name_factory(name);
  132 
  133     frm[name] = fr;
  134     if (create_stop_recorder){
  135         std::string name_stopped = name+"_stopped";
  136         
  137         feature_recorder *fr_stopped = create_name_factory(name_stopped);
  138         fr->set_stop_list_recorder(fr_stopped);
  139         frm[name_stopped] = fr_stopped;
  140     }
  141 }
  142 
  143 feature_recorder *feature_recorder_set::get_alert_recorder() const
  144 {
  145     if (flag_set(NO_ALERT)) return 0;
  146 
  147     return get_name(feature_recorder_set::ALERT_RECORDER_NAME);
  148 }
  149 
  150 
  151 /*
  152  * uses md5 to determine if a block was prevously seen.
  153  */
  154 bool feature_recorder_set::check_previously_processed(const uint8_t *buf,size_t bufsize)
  155 {
  156     std::string md5 = md5_generator::hash_buf(buf,bufsize).hexdigest();
  157     return seen_set.check_for_presence_and_insert(md5);
  158 }
  159 
  160 void feature_recorder_set::add_stats(const std::string &bucket,double seconds)
  161 {
  162     cppmutex::lock lock(Mscanner_stats);
  163     struct pstats &p = scanner_stats[bucket]; // get the location of the stats
  164     p.seconds += seconds;
  165     p.calls ++;
  166 }
  167 
  168 /*
  169  * Send the stats to a callback; if the callback returns less than 0, abort.
  170  */
  171 void feature_recorder_set::get_stats(void *user,stat_callback_t stat_callback) const
  172 {
  173     for(scanner_stats_map::const_iterator it = scanner_stats.begin();it!=scanner_stats.end();it++){
  174         if((*stat_callback)(user,(*it).first,(*it).second.calls,(*it).second.seconds)<0){
  175             break;
  176         }
  177     }
  178 }
  179 
  180 void feature_recorder_set::dump_name_count_stats(dfxml_writer &writer) const
  181 {
  182     cppmutex::lock lock(Mscanner_stats);
  183     writer.push("feature_files");
  184     for(feature_recorder_map::const_iterator ij = frm.begin(); ij != frm.end(); ij++){
  185         writer.set_oneline(true);
  186         writer.push("feature_file");
  187         writer.xmlout("name",ij->second->name);
  188         writer.xmlout("count",ij->second->count());
  189         writer.pop();
  190         writer.set_oneline(false);
  191     }
  192 }
  193 
  194 
  195 void    feature_recorder_set::set_flag(uint32_t f)
  196 {
  197     if(f & MEM_HISTOGRAM){
  198         if(flags & MEM_HISTOGRAM){
  199             std::cerr << "MEM_HISTOGRAM flag cannot be set twice\n";
  200             assert(0);
  201         }
  202         /* Create the in-memory histograms for all of the feature recorders */
  203         for(feature_recorder_map::const_iterator it = frm.begin(); it!=frm.end(); it++){
  204             feature_recorder *fr = it->second;
  205             fr->enable_memory_histograms();
  206         }
  207     }
  208     flags |= f;
  209 }         
  210 
  211 void    feature_recorder_set::unset_flag(uint32_t f)
  212 {
  213     if(f & MEM_HISTOGRAM){
  214         std::cerr << "MEM_HISTOGRAM flag cannot be cleared\n";
  215         assert(0);
  216     }
  217     flags &= ~f;
  218 }
  219 
  220 /****************************************************************
  221  *** PHASE HISTOGRAM (formerly phase 3): Create the histograms
  222  ****************************************************************/
  223 
  224 /**
  225  * We now have three kinds of histograms:
  226  * 1 - Traditional post-processing histograms specified by the histogram library
  227      1a - feature-file based traditional ones
  228      1b - SQL-based traditional ones.
  229  * 2 - In-memory histograms (used primarily by beapi)
  230  */
  231 
  232 
  233 void feature_recorder_set::add_histogram(const histogram_def &def)
  234 {
  235     feature_recorder *fr = get_name(def.feature);
  236     if(fr) fr->add_histogram(def);
  237 }
  238 
  239 void feature_recorder_set::dump_histograms(void *user,feature_recorder::dump_callback_t cb,
  240                                            feature_recorder_set::xml_notifier_t xml_error_notifier) const
  241 {
  242     /* Ask each feature recorder to dump its histograms */
  243     for(feature_recorder_map::const_iterator it = frm.begin(); it!=frm.end(); it++){
  244         feature_recorder *fr = it->second;
  245         fr->dump_histograms(user,cb,xml_error_notifier);
  246     }
  247 }
  248 
  249 void feature_recorder_set::get_feature_file_list(std::vector<std::string> &ret)
  250 {
  251     for(feature_recorder_map::const_iterator it = frm.begin(); it!=frm.end(); it++){
  252         ret.push_back(it->first);
  253     }
  254 }