"Fossies" - the Fresh Open Source Software Archive

Member "relax-5.0.0/test_suite/test_suite_runner.py" (6 Dec 2019, 17423 Bytes) of package /linux/privat/relax-5.0.0.src.tar.bz2:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "test_suite_runner.py": 4.1.3_vs_5.0.0.

    1 ###############################################################################
    2 #                                                                             #
    3 # Copyright (C) 2006-2008,2010-2015,2019 Edward d'Auvergne                    #
    4 # Copyright (C) 2019 Troels Schwarz-Linnet                                    #
    5 #                                                                             #
    6 # This file is part of the program relax (http://www.nmr-relax.com).          #
    7 #                                                                             #
    8 # This program is free software: you can redistribute it and/or modify        #
    9 # it under the terms of the GNU General Public License as published by        #
   10 # the Free Software Foundation, either version 3 of the License, or           #
   11 # (at your option) any later version.                                         #
   12 #                                                                             #
   13 # This program is distributed in the hope that it will be useful,             #
   14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
   15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
   16 # GNU General Public License for more details.                                #
   17 #                                                                             #
   18 # You should have received a copy of the GNU General Public License           #
   19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
   20 #                                                                             #
   21 ###############################################################################
   22 
   23 # Dependency checks.
   24 import dep_check
   25 
   26 # Python module imports.
   27 import os
   28 import sys
   29 from traceback import print_exc
   30 if dep_check.wx_module:
   31     import wx
   32 import unittest
   33 
   34 # Formatting.
   35 from test_suite.formatting import summary_line
   36 
   37 # Import the test suite categories.
   38 if dep_check.wx_module:
   39     from test_suite.gui_tests import GUI_test_runner
   40 from test_suite.system_tests import System_test_runner
   41 from test_suite.unit_tests.unit_test_runner import Unit_test_runner
   42 from test_suite.verification_tests import Verification_test_runner
   43 
   44 # relax module imports.
   45 if dep_check.wx_module:
   46     from gui import relax_gui
   47     from gui import interpreter
   48 from lib.text.sectioning import section, title
   49 from test_suite.relax_test_runner import GuiTestRunner, RelaxTestRunner
   50 from status import Status; status = Status()
   51 
   52 
   53 class Test_suite_runner:
   54     """Class for running all components of the relax test suite.
   55 
   56     This currently includes the following categories of tests:
   57         - System/functional tests.
   58         - Unit tests.
   59         - GUI tests.
   60         - Verification tests.
   61     """
   62 
   63     def __init__(self, tests=[], from_gui=False, categories=['system', 'unit', 'gui', 'verification'], timing=False, io_capture=True, list_tests=False):
   64         """Store the list of tests to preform.
   65 
   66         The test list should be something like ['N_state_model.test_stereochem_analysis'].  The first part is the imported test case class, the second is the specific test.
   67 
   68 
   69         @keyword tests:         The list of tests to preform.  If left at [], then all tests will be run.
   70         @type tests:            list of str
   71         @keyword from_gui:      A flag which indicates if the tests are being run from the GUI or not.
   72         @type from_gui:         bool
   73         @keyword categories:    The list of test categories to run, for example ['system', 'unit', 'gui', 'verification'] for all tests.
   74         @type categories:       list of str
   75         @keyword timing:        A flag which if True will enable timing of individual tests.
   76         @type timing:           bool
   77         @keyword io_capture:    A flag which if True will cause all IO to be captured and only printed out for failing or error tests.
   78         @type io_capture:       bool
   79         @keyword list_tests:    A flag which if True will cause the tests to be listed rather than executed.
   80         @type list_tests:       bool
   81         """
   82 
   83         # Store the args.
   84         self.tests = tests
   85         self.from_gui = from_gui
   86         self.categories = categories
   87         self.list_tests = list_tests
   88 
   89         # Set up the test runner.
   90         if from_gui:
   91             self.runner = GuiTestRunner(stream=sys.stdout, timing=timing, io_capture=io_capture)
   92         else:
   93             self.runner = RelaxTestRunner(stream=sys.stdout, timing=timing, io_capture=io_capture)
   94 
   95         # Let the tests handle the keyboard interrupt (for Python 2.7 and higher).
   96         if hasattr(unittest, 'installHandler'):
   97             unittest.installHandler()
   98 
   99 
  100     def run_all_tests(self, reset=True):
  101         """Execute all of the test suite test types.
  102 
  103         @keyword reset:     A flag which if True will reset the relax status objects for the tests.
  104         @type reset:        bool
  105         @return:            The combined status of the test categories.
  106         @rtype:             bool
  107         """
  108 
  109         # Reset the list for skipped tests.
  110         if reset:
  111             status.skipped_tests = []
  112 
  113         # Store the status, so that all test categories are run.
  114         test_status = True
  115 
  116         # Execute the system/functional tests.
  117         if 'system' in self.categories:
  118             test_status = self.run_system_tests(summary=False, reset=False) and test_status
  119 
  120         # Execute the unit tests.
  121         if 'unit' in self.categories:
  122             test_status = self.run_unit_tests(summary=False, reset=False) and test_status
  123 
  124         # Execute the GUI tests.
  125         if 'gui' in self.categories:
  126             test_status = self.run_gui_tests(summary=False, reset=False) and test_status
  127 
  128         # Execute the software verification tests.
  129         if 'verification' in self.categories:
  130             test_status = self.run_verification_tests(summary=False, reset=False) and test_status
  131 
  132         # Print out a summary of the test suite and return the combined status.
  133         if not self.list_tests:
  134             self.summary()
  135         return test_status
  136 
  137 
  138     def run_gui_tests(self, summary=True, reset=True):
  139         """Execute the GUI tests.
  140 
  141         @keyword summary:   A flag which if True will cause a summary to be printed.
  142         @type summary:      bool
  143         @keyword reset:     A flag which if True will reset the relax status objects for the tests.
  144         @type reset:        bool
  145         @return:            True if the tests were run, False if a KeyboardInterrupt occurred.
  146         @rtype:             bool
  147         """
  148 
  149         # Reset the list for skipped tests.
  150         if reset:
  151             status.skipped_tests = []
  152 
  153         # Run the tests, catching the keyboard interrupt.
  154         try:
  155             # Print a header.
  156             title(file=sys.stdout, text='GUI tests')
  157 
  158             # Run the tests.
  159             if dep_check.wx_module:
  160                 # Set up the GUI if needed (i.e. not in GUI mode already).
  161                 app = wx.GetApp()
  162                 if app == None:
  163                     # Initialise.
  164                     app = wx.App(redirect=False)
  165 
  166                     # Build the GUI.
  167                     app.gui = relax_gui.Main(parent=None, id=-1, title="")
  168 
  169                 # Execute the GUI tests.
  170                 gui_runner = GUI_test_runner()
  171                 self.runner.category = 'gui'
  172                 self.gui_result = gui_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests)
  173 
  174                 # Clean up for the GUI, if not in GUI mode.
  175                 if status.test_mode:
  176                     # Terminate the interpreter thread to allow the tests to cleanly exit.
  177                     interpreter_thread = interpreter.Interpreter()
  178                     interpreter_thread.exit()
  179 
  180                     # Stop the GUI main loop.
  181                     app.ExitMainLoop()
  182 
  183             # No wx module installed.
  184             else:
  185                 print("All GUI tests skipped due to the missing/broken wx module.\n")
  186                 self.gui_result = 'skip'
  187 
  188             # Print out a summary of the test suite.
  189             if summary and not self.list_tests:
  190                 self.summary()
  191 
  192         # Catch the keyboard interrupt.
  193         except KeyboardInterrupt:
  194             print("\nKeyboardInterrupt:  Terminating all tests.\n")
  195             sys.exit(1)
  196 
  197         # Catch GUI app set up errors.
  198         except:
  199             print("Failure in setting up the GUI tests.\n")
  200             print_exc()
  201             if summary and not self.list_tests:
  202                 self.summary()
  203             self.gui_result = 'skip'
  204             return False
  205 
  206         # All tests were run successfully.
  207         return self.gui_result
  208 
  209 
  210     def run_system_tests(self, summary=True, reset=True):
  211         """Execute the system/functional tests.
  212 
  213         @keyword summary:   A flag which if True will cause a summary to be printed.
  214         @type summary:      bool
  215         @keyword reset:     A flag which if True will reset the relax status objects for the tests.
  216         @type reset:        bool
  217         @return:            True if the tests were run, False if a KeyboardInterrupt occurred.
  218         @rtype:             bool
  219         """
  220 
  221         # Reset the list for skipped tests.
  222         if reset:
  223             status.skipped_tests = []
  224 
  225         # Run the tests, catching the keyboard interrupt.
  226         try:
  227             # Print a header.
  228             title(file=sys.stdout, text='System / functional tests')
  229 
  230             # Run the tests.
  231             system_runner = System_test_runner()
  232             self.runner.category = 'system'
  233             self.system_result = system_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests)
  234 
  235             # Print out a summary of the test suite.
  236             if summary and not self.list_tests:
  237                 self.summary()
  238 
  239         # Catch the keyboard interrupt.
  240         except KeyboardInterrupt:
  241             print("\nKeyboardInterrupt:  Terminating all tests.\n")
  242             sys.exit(1)
  243 
  244         # All tests were run successfully.
  245         return self.system_result
  246 
  247 
  248     def run_unit_tests(self, summary=True, reset=True):
  249         """Execute the unit tests.
  250 
  251         @keyword summary:   A flag which if True will cause a summary to be printed.
  252         @type summary:      bool
  253         @keyword reset:     A flag which if True will reset the relax status objects for the tests.
  254         @type reset:        bool
  255         @return:            True if the tests were run, False if a KeyboardInterrupt occurred.
  256         @rtype:             bool
  257         """
  258 
  259         # Reset the list for skipped tests.
  260         if reset:
  261             status.skipped_tests = []
  262 
  263         # Run the tests, catching the keyboard interrupt.
  264         try:
  265             # Print a header.
  266             title(file=sys.stdout, text='Unit tests')
  267 
  268             # Run the tests.
  269             unit_runner = Unit_test_runner(root_path=status.install_path+os.sep+'test_suite'+os.sep+'unit_tests')
  270             self.runner.category = 'unit'
  271             self.unit_result = unit_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests)
  272 
  273             # Print out a summary of the test suite.
  274             if summary and not self.list_tests:
  275                 self.summary()
  276 
  277         # Catch the keyboard interrupt.
  278         except KeyboardInterrupt:
  279             print("\nKeyboardInterrupt:  Terminating all tests.\n")
  280             sys.exit(1)
  281 
  282         # All tests were run successfully.
  283         return self.unit_result
  284 
  285 
  286     def run_verification_tests(self, summary=True, reset=True):
  287         """Execute the software verification tests.
  288 
  289         @keyword summary:   A flag which if True will cause a summary to be printed.
  290         @type summary:      bool
  291         @keyword reset:     A flag which if True will reset the relax status objects for the tests.
  292         @type reset:        bool
  293         """
  294 
  295         # Reset the list for skipped tests.
  296         if reset:
  297             status.skipped_tests = []
  298 
  299         # Run the tests, catching the keyboard interrupt.
  300         try:
  301             # Print a header.
  302             title(file=sys.stdout, text='Software verification tests')
  303 
  304             # Run the tests.
  305             verification_runner = Verification_test_runner()
  306             self.runner.category = 'verification'
  307             self.verification_result = verification_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests)
  308 
  309             # Print out a summary of the test suite.
  310             if summary and not self.list_tests:
  311                 self.summary()
  312 
  313         # Catch the keyboard interrupt.
  314         except KeyboardInterrupt:
  315             print("\nKeyboardInterrupt:  Terminating all tests.\n")
  316             sys.exit(1)
  317 
  318         # All tests were run successfully.
  319         return self.verification_result
  320 
  321 
  322     def summary(self):
  323         """Print out a summary of the relax test suite."""
  324 
  325         # Title.
  326         title(file=sys.stdout, text="Summary of the relax test suite")
  327 
  328         # The skipped tests.
  329         if status.skip_blacklisted_tests:
  330             self.summary_skipped()
  331 
  332         # Subtitle.
  333         section(file=sys.stdout, text="Synopsis")
  334 
  335         # System/functional test summary.
  336         if hasattr(self, 'system_result'):
  337             summary_line("System/functional tests", self.system_result, width=status.text_width)
  338 
  339         # Unit test summary.
  340         if hasattr(self, 'unit_result'):
  341             summary_line("Unit tests", self.unit_result, width=status.text_width)
  342 
  343         # GUI test summary.
  344         if hasattr(self, 'gui_result'):
  345             summary_line("GUI tests", self.gui_result, width=status.text_width)
  346 
  347         # Verification test summary.
  348         if hasattr(self, 'verification_result'):
  349             summary_line("Software verification tests", self.verification_result, width=status.text_width)
  350 
  351         # Synopsis.
  352         if hasattr(self, 'system_result') and hasattr(self, 'unit_result') and hasattr(self, 'gui_result') and hasattr(self, 'verification_result'):
  353             if self.gui_result == "skip":
  354                 test_status = self.system_result and self.unit_result and self.verification_result
  355             else:
  356                 test_status = self.system_result and self.unit_result and self.gui_result and self.verification_result
  357             summary_line("Synopsis", test_status, width=status.text_width)
  358 
  359         # End.
  360         print('\n\n')
  361 
  362 
  363     def summary_skipped(self):
  364         """Print out information about skipped tests.""" 
  365 
  366         # Counts.
  367         system_count = {}
  368         unit_count = {}
  369         gui_count = {}
  370         verification_count = {}
  371         for i in range(len(status.skipped_tests)):
  372             # Alias.
  373             test = status.skipped_tests[i]
  374 
  375             # Skip all skipped tests whereby the module is set to None to indicate that the test skipping should not be reported.
  376             if test[1] == None:
  377                 continue
  378 
  379             # Initialise in needed.
  380             if not test[1] in system_count:
  381                 system_count[test[1]] = 0
  382                 unit_count[test[1]] = 0
  383                 gui_count[test[1]] = 0
  384                 verification_count[test[1]] = 0
  385 
  386             # A system test.
  387             if test[2] == 'system':
  388                 system_count[test[1]] += 1
  389 
  390             # A unit test.
  391             if test[2] == 'unit':
  392                 unit_count[test[1]] += 1
  393 
  394             # A GUI test.
  395             if test[2] == 'gui':
  396                 gui_count[test[1]] += 1
  397 
  398             # A verification test.
  399             if test[2] == 'verification':
  400                 verification_count[test[1]] += 1
  401 
  402         # The missing modules.
  403         missing_modules = sorted(system_count.keys())
  404         section(file=sys.stdout, text="Optional packages/modules")
  405 
  406         # Nothing missing.
  407         if not missing_modules:
  408             # Except for the wx module!
  409             if not dep_check.wx_module and hasattr(self, 'gui_result'):
  410                 print("All GUI tests skipped due to the missing wxPython module, no other tests skipped due to missing modules.\n")
  411 
  412             # Normal printout.
  413             else:
  414                 print("No tests skipped due to missing modules.\n")
  415 
  416             # The skip the table.
  417             return
  418 
  419         # Header.
  420         print("Tests skipped due to missing optional packages/modules/software:\n")
  421         header = "%-33s" % "Module/package/software" 
  422         if len(system_count):
  423             header = "%s %20s" % (header, "System test count")
  424         if len(unit_count):
  425             header = "%s %20s" % (header, "Unit test count")
  426         if len(gui_count):
  427             header = "%s %20s" % (header, "GUI test count")
  428         if len(verification_count):
  429             header = "%s %20s" % (header, "Verification test count")
  430         print('-'*len(header))
  431         print(header)
  432         print('-'*len(header))
  433 
  434         # The table.
  435         for module in missing_modules:
  436             text = "%-33s" % module
  437             if len(system_count):
  438                 text = "%s %20s" % (text, system_count[module])
  439             if len(unit_count):
  440                 text = "%s %20s" % (text, unit_count[module])
  441             if len(gui_count):
  442                 text = "%s %20s" % (text, gui_count[module])
  443             if len(verification_count):
  444                 text = "%s %20s" % (text, verification_count[module])
  445             print(text)
  446 
  447         # End the table.
  448         print('-'*len(header))
  449         print("\n")