"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "src/fuglu/scansession.py" between
fuglu-0.10.8.tar.gz and fuglu-1.0.0.tar.gz

About: FuGlu is a mail scanning daemon for Postfix written in Python. It acts as a glue application between the MTA and spam checkers and antivirus software.

scansession.py  (fuglu-0.10.8):scansession.py  (fuglu-1.0.0)
skipping to change at line 56 skipping to change at line 56
self._enabletimetracker= False self._enabletimetracker= False
self.enabletimetracker = enable self.enabletimetracker = enable
self.timetracker = time.time() self.timetracker = time.time()
# starttime and realtime will be used for async # starttime and realtime will be used for async
self.starttime = self.timetracker self.starttime = self.timetracker
self.realtime = 0 self.realtime = 0
self.asynctime = defaultdict(lambda: 0.0) self.asynctime = defaultdict(lambda: 0.0)
self.timings = []
self.port = port if port is not None else -1 self.port = port if port is not None else -1
# name is used as hash so it's possible to add to an existing timer by s
ame name
self.timingshash = {}
def resettimer(self): def resettimer(self):
"""Reset time tracker""" """Reset time tracker"""
if self.enabletimetracker: if self.enabletimetracker:
self.timetracker = time.time() self.timetracker = time.time()
self.starttime = self.timetracker self.starttime = self.timetracker
self.realtime = 0 self.realtime = 0
self.asynctime = defaultdict(lambda: 0.0) self.asynctime = defaultdict(lambda: 0.0)
self.timings = [] self.timingshash = {}
def sum_asynctime(self, delta: tp.Union[float, int], key: tp.Optional[str] = None, logid: tp.Optional[str] = None): def sum_asynctime(self, delta: tp.Union[float, int], key: tp.Optional[str] = None, logid: tp.Optional[str] = None):
#oldval = self.asynctime[key] #oldval = self.asynctime[key]
self.asynctime[key] += delta self.asynctime[key] += delta
#self.timings_logger.debug(f"{logid+' ' if logid else ''}Adding {delta} to {oldval} -> asyncdict[{key}]={self.asynctime[key]}") #self.timings_logger.debug(f"{logid+' ' if logid else ''}Adding {delta} to {oldval} -> asyncdict[{key}]={self.asynctime[key]}")
@property @property
def enabletimetracker(self): def enabletimetracker(self):
return self._enabletimetracker return self._enabletimetracker
@enabletimetracker.setter @enabletimetracker.setter
def enabletimetracker(self, enabled): def enabletimetracker(self, enabled):
if not enabled: if not enabled:
self._enabletimetracker = False self._enabletimetracker = False
else: else:
self._enabletimetracker = True self._enabletimetracker = True
self.timetracker = time.time() self.timetracker = time.time()
self.timings = [] self.timingshash = {}
def tracktime(self, tagname, plugin=False, prepender=False, appender=False, mplugin=False): def tracktime(self, tagname, plugin=False, prepender=False, appender=False, mplugin=False):
return self._tracktime(tagname, plugin=plugin, prepender=prepender, appe nder=appender, mplugin=mplugin) if self.enabletimetracker else None return self._tracktime(tagname, plugin=plugin, prepender=prepender, appe nder=appender, mplugin=mplugin) if self.enabletimetracker else None
def gettime(self, plugins=None, prependers=None, appenders=None, mplugins=No ne): def gettime(self, plugins=None, prependers=None, appenders=None, mplugins=No ne):
return self._gettime(plugins=plugins, prependers=prependers, appenders=a ppenders, mplugins=mplugins) if self.enabletimetracker else [] return self._gettime(plugins=plugins, prependers=prependers, appenders=a ppenders, mplugins=mplugins) if self.enabletimetracker else []
def sumtime(self, plugins=None, prependers=None, appenders=None, mplugins=No ne): def sumtime(self, plugins=None, prependers=None, appenders=None, mplugins=No ne):
return self._sumtime(plugins=plugins, prependers=prependers, appenders=a ppenders, mplugins=mplugins) if self.enabletimetracker else 0.0 return self._sumtime(plugins=plugins, prependers=prependers, appenders=a ppenders, mplugins=mplugins) if self.enabletimetracker else 0.0
skipping to change at line 118 skipping to change at line 119
Keyword Args: Keyword Args:
plugin (bool): tag belongs to plugin timing (default=False) plugin (bool): tag belongs to plugin timing (default=False)
prepender (bool): tag belongs to prepender timing (default=False) prepender (bool): tag belongs to prepender timing (default=False)
appender (bool): tag belongs to appender timing (default=False) appender (bool): tag belongs to appender timing (default=False)
mplugin (bool): tag belongs to milterplugin timing (default=False) mplugin (bool): tag belongs to milterplugin timing (default=False)
""" """
newtime = time.time() newtime = time.time()
difftime = newtime - self.timetracker difftime = newtime - self.timetracker
self.realtime = newtime - self.starttime self.realtime = newtime - self.starttime
self.timetracker = newtime self.timetracker = newtime
self.timings.append((tagname, difftime, plugin, prepender, appender, mpl hsh = (tagname, plugin, prepender, appender, mplugin)
ugin)) existing = self.timingshash.get(hsh)
if existing:
_, e_difftime, _, _, _, _ = existing
difftime = e_difftime + difftime
self.timings_logger.debug(f"After updating existing entry for {tagna
me}: {e_difftime} -> {difftime}")
self.timingshash[hsh] = (tagname, difftime, plugin, prepender, appender,
mplugin)
def _gettime(self, plugins=None, prependers=None, appenders=None, mplugins=N one): def _gettime(self, plugins=None, prependers=None, appenders=None, mplugins=N one):
""" """
Get a list of timings stored (all/only plugins/exclude plugins) Get a list of timings stored (all/only plugins/exclude plugins)
Keyword Args: Keyword Args:
plugins (bool or None): 'None': ignore this restriction, True: timin g must have 'plugin' tag, False: timing must not have plugin tag plugins (bool or None): 'None': ignore this restriction, True: timin g must have 'plugin' tag, False: timing must not have plugin tag
prependers (bool or None): 'None': ignore this restriction, True: ti ming must have 'prepender' tag, False: timing must not have plugin tag prependers (bool or None): 'None': ignore this restriction, True: ti ming must have 'prepender' tag, False: timing must not have plugin tag
appenders (bool or None): 'None': ignore this restriction, True: tim ing must have 'appender' tag, False: timing must not have plugin tag appenders (bool or None): 'None': ignore this restriction, True: tim ing must have 'appender' tag, False: timing must not have plugin tag
mplugins (bool or None): 'None': ignore this restriction, True: timi ng must have 'mplugin' tag, False: timing must not have plugin tag mplugins (bool or None): 'None': ignore this restriction, True: timi ng must have 'mplugin' tag, False: timing must not have plugin tag
Returns: Returns:
list of tuples (name, time) list of tuples (name, time)
""" """
timing_list = [] timing_list = []
for timeitem in self.timings: for timeitem in self.timingshash.values():
if plugins is not None and timeitem[2] != plugins: if plugins is not None and timeitem[2] != plugins:
continue continue
if prependers is not None and timeitem[3] != prependers: if prependers is not None and timeitem[3] != prependers:
continue continue
if appenders is not None and timeitem[4] != appenders: if appenders is not None and timeitem[4] != appenders:
continue continue
if mplugins is not None and timeitem[5] != mplugins: if mplugins is not None and timeitem[5] != mplugins:
continue continue
timing_list.append(timeitem[:2]) timing_list.append(timeitem[:2])
return timing_list return timing_list
skipping to change at line 175 skipping to change at line 183
return reduce(lambda x, y: (x + y), puretimings) return reduce(lambda x, y: (x + y), puretimings)
def _report_timings(self, suspectid, withrealtime=False, end=True): def _report_timings(self, suspectid, withrealtime=False, end=True):
""" """
Report all the timings collected Report all the timings collected
Args: Args:
suspectid (id): the suspect id suspectid (id): the suspect id
withrealtime(bool): The real time from start to end, which might dif fer from total time in case of async withrealtime(bool): The real time from start to end, which might dif fer from total time in case of async
""" """
if not self.timings: if not self.timingshash:
self.timings_logger.debug('no timings to report') self.timings_logger.debug('no timings to report')
return return
if end: if end:
self.tracktime('end') self.tracktime('end')
total_asynctime = 0.0
if withrealtime: if withrealtime:
self.timings_logger.info('port: %u, id: %s, real: %.6f' % (self.port , suspectid, self.realtime)) self.timings_logger.info('port: %u, id: %s, real: %.6f' % (self.port , suspectid, self.realtime))
for k, val in self.asynctime.items(): for k, val in self.asynctime.items():
self.timings_logger.info(f'port: {self.port}, id: {suspectid}, a sync{"("+k+")" if k else ""}: {val:.6f}') self.timings_logger.info(f'port: {self.port}, id: {suspectid}, a sync{"("+k+")" if k else ""}: {val:.6f}')
self.timings_logger.info('port: %u, id: %s, total: %.6f' % (self.port, s total_asynctime += val
uspectid, self.sumtime())) self.timings_logger.info('port: %u, id: %s, total(async): %.3f' % (s
elf.port, suspectid,total_asynctime))
self.timings_logger.info('port: %u, id: %s, total: %.6f' % (self.port, s
uspectid, self.sumtime() - total_asynctime))
self.timings_logger.info('port: %u, id: %s, overhead: %.3f' % (self.port , suspectid, self.timings_logger.info('port: %u, id: %s, overhead: %.3f' % (self.port , suspectid,
self.sumt ime(plugins=False, self.sumt ime(plugins=False,
prependers=False, prependers=False,
appenders=False, appenders=False,
mplugins=False))) mplugins=False)))
all_milterplugs = self.gettime(mplugins=True) all_milterplugs = self.gettime(mplugins=True)
for mplugintime in all_milterplugs: for mplugintime in all_milterplugs:
self.timings_logger.info('port: %u, id: %s, (MPL) %s: %.3f' % (self. port, suspectid, mplugintime[0], self.timings_logger.info('port: %u, id: %s, (MPL) %s: %.3f' % (self. port, suspectid, mplugintime[0],
mplug intime[1])) mplug intime[1]))
all_prependertimes = self.gettime(prependers=True) all_prependertimes = self.gettime(prependers=True)
skipping to change at line 335 skipping to change at line 347
self.logger.warning(message_prefix+u'Could not get incoming port: %s' % str(e)) self.logger.warning(message_prefix+u'Could not get incoming port: %s' % str(e))
pluglist, applist = self.run_prependers(suspect) pluglist, applist = self.run_prependers(suspect)
starttime = time.time() starttime = time.time()
self.run_plugins(suspect, pluglist) self.run_plugins(suspect, pluglist)
# Set fuglu spam status if wanted # Set fuglu spam status if wanted
if self.config.getboolean('main', 'spamstatusheader'): if self.config.getboolean('main', 'spamstatusheader'):
if suspect.is_spam(): if suspect.is_spam():
suspect.addheader("%sSpamstatus" % prependheader, 'YES') suspect.add_header("%sSpamstatus" % prependheader, 'YES' )
else: else:
suspect.addheader("%sSpamstatus" % prependheader, 'NO') suspect.add_header("%sSpamstatus" % prependheader, 'NO')
# how long did it all take? # how long did it all take?
difftime = time.time() - starttime difftime = time.time() - starttime
suspect.tags['fuglu.scantime'] = "%.4f" % difftime suspect.tags['fuglu.scantime'] = "%.4f" % difftime
# Debug info to mail # Debug info to mail
if self.config.getboolean('main', 'debuginfoheader'): if self.config.getboolean('main', 'debuginfoheader'):
debuginfo = str(suspect) debuginfo = str(suspect)
suspect.addheader("%sDebuginfo" % prependheader, debuginfo) suspect.add_header("%sDebuginfo" % prependheader, debuginfo)
# add suspect id for tracking # add suspect id for tracking
if self.config.getboolean('main', 'suspectidheader'): if self.config.getboolean('main', 'suspectidheader'):
suspect.addheader('%sSuspect' % prependheader, suspect.id) suspect.add_header('%sSuspect' % prependheader, suspect.id)
self.tracktime("Adding-Headers") self.tracktime("Adding-Headers")
# checks done.. print out suspect status # checks done.. print out suspect status
logformat = self.config.get('main', 'logtemplate') logformat = self.config.get('main', 'logtemplate')
if logformat.strip() != '': if logformat.strip() != '':
self.logger.info(suspect.log_format(logformat)) self.logger.info(suspect.log_format(logformat))
suspect.debug(suspect) suspect.debug(suspect)
self.tracktime("Debug-Suspect") self.tracktime("Debug-Suspect")
# check if one of the plugins made a decision # check if one of the plugins made a decision
skipping to change at line 599 skipping to change at line 611
"""Run scannerplugins on suspect""" """Run scannerplugins on suspect"""
suspect.debug('Will run plugins: %s' % pluglist) suspect.debug('Will run plugins: %s' % pluglist)
self.tracktime("Before-Plugins") self.tracktime("Before-Plugins")
for plugin in pluglist: for plugin in pluglist:
try: try:
self.logger.debug('Running plugin %s' % plugin) self.logger.debug('Running plugin %s' % plugin)
self.set_workerstate( self.set_workerstate(
"%s : Running Plugin %s" % (suspect, plugin)) "%s : Running Plugin %s" % (suspect, plugin))
suspect.debug('Running plugin %s' % str(plugin)) suspect.debug('Running plugin %s' % str(plugin))
starttime = time.time() starttime = time.time()
ans = plugin.examine(suspect) ans = plugin.run_examine(suspect)
plugintime = time.time() - starttime plugintime = time.time() - starttime
suspect.tags['scantimes'].append((plugin.section, plugintime)) suspect.tags['scantimes'].append((plugin.section, plugintime))
message = None message = None
if type(ans) is tuple: if type(ans) is tuple:
result, message = ans result, message = ans
else: else:
result = ans result = ans
if result is None: if result is None:
result = DUNNO result = DUNNO
 End of changes. 14 change blocks. 
14 lines changed or deleted 29 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)