"Fossies" - the Fresh Open Source Software Archive

Member "glusterfs-8.2/run-tests.sh" (16 Sep 2020, 18867 Bytes) of package /linux/misc/glusterfs-8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Bash source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "run-tests.sh": 8.1_vs_8.2.

    1 #!/bin/bash
    2 # Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
    3 #
    4 
    5 # As many tests are designed to take values of variables from 'env.rc',
    6 # it is good to source the file. While it is also required to source the
    7 # file individually in each tests (as it should be possible to run the
    8 # tests separately), exporting variables from env.rc is not harmful if
    9 # done here
   10 
   11 source ./tests/env.rc
   12 
   13 export TZ=UTC
   14 force="no"
   15 head="yes"
   16 retry="yes"
   17 tests=""
   18 exit_on_failure="yes"
   19 skip_bad_tests="yes"
   20 skip_known_bugs="yes"
   21 result_output="/tmp/gluster_regression.txt"
   22 section_separator="========================================"
   23 run_timeout=200
   24 kill_after_time=5
   25 nfs_tests=$RUN_NFS_TESTS
   26 
   27 # Option below preserves log tarballs for each run of a test separately
   28 #       named: <test>-iteration-<n>.tar
   29 # If set to any other value, then log tarball is just named after the test and
   30 # overwritten in each iteration (saves space)
   31 #       named: <test>.tar
   32 # Use option -p to override default behavior
   33 skip_preserve_logs="yes"
   34 
   35 OSTYPE=$(uname -s)
   36 
   37 # Function for use in generating filenames with increasing "-<n>" index
   38 # In:
   39 #       $1 basepath: Directory where file needs to be created
   40 #       $2 filename: Name of the file sans extension
   41 #       $3 extension: Extension string that would be appended to the generated
   42 #               filename
   43 # Out:
   44 #       string of next available filename with appended "-<n>"
   45 # Example:
   46 #       Interested routines that want to create a file name, say foo-<n>.txt at
   47 #       location /var/log/gluster would pass in "/var/log/gluster" "foo" "txt"
   48 #       and be returned next available foo-<n> filename to create.
   49 # Notes:
   50 #       Function will not accept empty extension, and will return the same name
   51 #       over and over (which can be fixed when there is a need for it)
   52 function get_next_filename()
   53 {
   54         local basepath=$1
   55         local filename=$2
   56         local extension=$3
   57         local next=1
   58         local tfilename="${filename}-${next}"
   59         while [ -e "${basepath}/${tfilename}.${extension}" ]; do
   60                 next=$((next+1))
   61                 tfilename="${filename}-${next}"
   62         done
   63 
   64         echo "$tfilename"
   65 }
   66 
   67 # Tar the gluster logs and generate a tarball named after the first parameter
   68 # passed in to the function. Ideally the test name is passed to this function
   69 # to generate the required tarball.
   70 # Tarball name is further controlled by the variable skip_preserve_logs
   71 function tar_logs()
   72 {
   73         t=$1
   74 
   75         logdir=$(gluster --print-logdir)
   76         basetarname=$(basename "$t" .t)
   77 
   78         if [ -n "$logdir" ]
   79         then
   80                 if [[ $skip_preserve_logs == "yes" ]]; then
   81                         savetarname=$(get_next_filename "${logdir}" \
   82                                 "${basetarname}-iteration" "tar" \
   83                                 | tail -1)
   84                 else
   85                         savetarname="$basetarname"
   86                 fi
   87 
   88                 # Can't use --exclude here because NetBSD doesn't have it.
   89                 # However, both it and Linux have -X to take patterns from
   90                 # a file, so use that.
   91                 (echo '*.tar'; echo .notar) > "${logdir}"/.notar \
   92                         && \
   93                 tar -cf "${logdir}"/"${savetarname}".tar -X "${logdir}"/.notar \
   94                         "${logdir}"/* 2> /dev/null \
   95                         && \
   96                 find "$logdir"/* -maxdepth 0 -name '*.tar' -prune \
   97                                         -o -exec rm -rf '{}' ';'
   98 
   99                 echo "Logs preserved in tarball $savetarname.tar"
  100         else
  101                 echo "Logs not preserved, as logdir is not set"
  102         fi
  103 }
  104 
  105 function check_dependencies()
  106 {
  107     ## Check all dependencies are present
  108     MISSING=""
  109 
  110     # Check for dbench
  111     env dbench --usage > /dev/null 2>&1
  112     if [ $? -ne 0 ]; then
  113         MISSING="$MISSING dbench"
  114     fi
  115 
  116     # Check for git
  117     env git --version > /dev/null 2>&1
  118     if [ $? -ne 0 ]; then
  119         MISSING="$MISSING git"
  120     fi
  121 
  122     # Check for nfs-utils (Linux-only: built-in NetBSD with different name)
  123     if [ "x`uname -s`" = "xLinux" ] ; then
  124       env mount.nfs -V > /dev/null 2>&1
  125       if [ $? -ne 0 ]; then
  126           MISSING="$MISSING nfs-utils"
  127       fi
  128     fi
  129 
  130     # Check for netstat
  131     env netstat --version > /dev/null 2>&1
  132     if [ $? -ne 0 ]; then
  133         MISSING="$MISSING netstat"
  134     fi
  135 
  136     # Check for the Perl Test Harness
  137     env prove --version > /dev/null 2>&1
  138     if [ $? -ne 0 ]; then
  139         MISSING="$MISSING perl-Test-Harness"
  140     fi
  141 
  142     which json_verify > /dev/null
  143     if [ $? -ne 0 ]; then
  144         MISSING="$MISSING json_verify"
  145     fi
  146 
  147     # Check for XFS programs (Linux Only: NetBSD does without)
  148     if [ "x`uname -s`" = "xLinux" ] ; then
  149       env mkfs.xfs -V > /dev/null 2>&1
  150       if [ $? -ne 0 ]; then
  151           MISSING="$MISSING xfsprogs"
  152       fi
  153     fi
  154 
  155     # Check for attr
  156     env getfattr --version > /dev/null 2>&1
  157     if [ $? -ne 0 ]; then
  158         MISSING="$MISSING attr"
  159     fi
  160 
  161     # Check for pidof
  162     pidof pidof > /dev/null 2>&1
  163     if [ $? -ne 0 ]; then
  164         MISSING="$MISSING pidof"
  165     fi
  166 
  167     # Check for netstat
  168     env netstat --version > /dev/null 2>&1
  169     if [ $? -ne 0 ]; then
  170         MISSING="$MISSING netstat"
  171     fi
  172 
  173     # check for psutil python package
  174     test `uname -s` == "Darwin" || test `uname -s` == "FreeBSD" && {
  175         pip show psutil | grep -q psutil >/dev/null 2>&1
  176         if [ $? -ne 0 ]; then
  177             MISSING="$MISSING psutil"
  178         fi
  179     }
  180 
  181     ## If dependencies are missing, warn the user and abort
  182     if [ "x$MISSING" != "x" ]; then
  183         test "x${force}" != "xyes" && echo "Aborting."
  184         echo
  185         echo "The following required tools are missing:"
  186         echo
  187         for pkg in $MISSING; do
  188             echo "  * $pkg"
  189         done
  190         echo
  191         test "x${force}" = "xyes" && return
  192         echo "Please install them and try again."
  193         echo
  194         exit 2
  195     fi
  196 }
  197 
  198 function check_location()
  199 {
  200     regression_testsdir=$(dirname $0);
  201 
  202     if [ ! -f ${regression_testsdir}/tests/include.rc ]; then
  203         echo "Aborting."
  204         echo
  205         echo "The tests/ subdirectory seems to be missing."
  206         echo
  207         echo "Please correct the problem and try again."
  208         echo
  209         exit 1
  210     fi
  211 }
  212 
  213 function check_user()
  214 {
  215     # If we're not running as root, warn the user and abort
  216     MYUID=`/usr/bin/id -u`
  217     if [ 0${MYUID} -ne 0 ]; then
  218         echo "Aborting."
  219         echo
  220         echo "The GlusterFS Test Framework must be run as root."
  221         echo
  222         echo "Please change to the root user and try again."
  223         echo
  224         exit 3
  225     fi
  226 }
  227 
  228 function match()
  229 {
  230         # Patterns considered valid:
  231         # 0. Empty means everything
  232         #    ""           matches ** i.e all
  233         # 1. full or partial file/directory names
  234         #   basic         matches tests/basic
  235         #   basic/afr     matches tests/basic/afr
  236         # 2. globs
  237         #   basic/*       matches all files and directories in basic
  238         #   basic/*/      matches subdirectories in basic (afr|ec)
  239         # 3. numbered bug matching
  240         #   884455        matches bugs/bug-884455.t
  241         #   859927        matches bugs/859927, bugs/bug-859927.t
  242         #   1015990       matches /bugs/bug-1015990-rep.t, bug-1015990.t
  243         # ...lots of other cases accepted as well, since globbing is tricky.
  244         local t=$1
  245         shift
  246         local a
  247         local match=1
  248         if [ -z "$@" ]; then
  249             match=0
  250             return $match
  251         fi
  252         for a in $@ ; do
  253             case "$t" in
  254                 *$a*)
  255                     match=0
  256                     ;;
  257             esac
  258         done
  259         return $match
  260 }
  261 
  262 # Tests can have comment lines with some comma separated values within them.
  263 # Key names used to determine test status are
  264 # G_TESTDEF_TEST_STATUS_CENTOS6
  265 # G_TESTDEF_TEST_STATUS_NETBSD7
  266 # Some examples:
  267 # G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456
  268 # G_TESTDEF_TEST_STATUS_CENTOS6=BRICK_MUX_BAD_TEST,BUG=123456
  269 # G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=4444444
  270 # G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456;555555
  271 # G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TESTS,BUG=1385758
  272 # You can change status of test to enabled or delete the line only if all the
  273 # bugs are closed or modified or if the patch fixes it.
  274 function get_test_status ()
  275 {
  276     local test_name=$1
  277     local host_os=""
  278     local result=""
  279 
  280     host_os=$(uname -s)
  281 
  282     case "$host_os" in
  283     # Leaving out the logic to determine the particular distro and version
  284     # for later. Why does the key have the distro and version then?
  285     # Because changing the key in all test files would be very big process
  286     # updating just this function with a better logic much simpler.
  287     Linux)
  288         result=$(grep -e "^#G_TESTDEF_TEST_STATUS_CENTOS6" $test_name | \
  289                  awk -F"," {'print $1'} | awk -F"=" {'print $2'}) ;;
  290     NetBSD)
  291         result=$(grep -e "^#G_TESTDEF_TEST_STATUS_NETBSD7" $test_name | \
  292                  awk -F"," {'print $1'} | awk -F"=" {'print $2'}) ;;
  293     *)
  294         result="ENABLED" ;;
  295     esac
  296 
  297     echo "$result"
  298 
  299 }
  300 
  301 function get_bug_list_for_disabled_test ()
  302 {
  303     local test_name=$1
  304     local host_os=""
  305     local result=""
  306 
  307     host_os=$(uname -s)
  308 
  309     case "$host_os" in
  310     # Leaving out the logic to determine the particular distro and version
  311     # for later. Why does the key have the distro and version then?
  312     # Because changing the key in all test files would be very big process
  313     # updating just this function with a better logic much simpler.
  314     Linux)
  315         result=$(grep -e "^#G_TESTDEF_TEST_STATUS_CENTOS6" $test_name | \
  316                  awk -F"," {'print $2'} | awk -F"=" {'print $2'}) ;;
  317     NetBSD)
  318         result=$(grep -e "^#G_TESTDEF_TEST_STATUS_NETBSD7" $test_name | \
  319                  awk -F"," {'print $2'} | awk -F"=" {'print $2'}) ;;
  320     *)
  321         result="0000000" ;;
  322     esac
  323 
  324     echo "$result"
  325 
  326 }
  327 
  328 function run_tests()
  329 {
  330     RES=0
  331     FLAKY=''
  332     FAILED=''
  333     TESTS_NEEDED_RETRY=''
  334     GENERATED_CORE=''
  335     total_tests=0
  336     selected_tests=0
  337     skipped_bad_tests=0
  338     skipped_known_issue_tests=0
  339     total_run_tests=0
  340 
  341     # key = path of .t file; value = time taken to run the .t file
  342     declare -A ELAPSEDTIMEMAP
  343 
  344     # Test if -k is supported for timeout command
  345     # This is not supported on centos6, but spuported on centos7
  346     # The flags is required for running the command in both flavors
  347     timeout_cmd_exists="yes"
  348     timeout -k 1 10 echo "testing 'timeout' command"
  349     if [ $? -ne 0 ]; then
  350         timeout_cmd_exists="no"
  351     fi
  352 
  353     for t in $(find ${regression_testsdir}/tests -name '*.t' \
  354                | LC_COLLATE=C sort) ; do
  355         old_cores=$(ls /*-*.core 2> /dev/null | wc -l)
  356         total_tests=$((total_tests+1))
  357         if match $t "$@" ; then
  358             selected_tests=$((selected_tests+1))
  359             echo
  360             echo $section_separator$section_separator
  361             if [[ $(get_test_status $t) =~ "BAD_TEST" ]] && \
  362                [[ $skip_bad_tests == "yes" ]]
  363             then
  364                 skipped_bad_tests=$((skipped_bad_tests+1))
  365                 echo "Skipping bad test file $t"
  366                 echo "Reason: bug(s):" $(get_bug_list_for_disabled_test $t)
  367                 echo $section_separator$section_separator
  368                 echo
  369                 continue
  370             fi
  371             if [[ $(get_test_status $t) == "KNOWN_ISSUE" ]] && \
  372                [[ $skip_known_bugs == "yes" ]]
  373             then
  374                 skipped_known_issue_tests=$((skipped_known_issue_tests+1))
  375                 echo "Skipping test file $t due to known issue"
  376                 echo "Reason: bug(s):" $(get_bug_list_for_disabled_test $t)
  377                 echo $section_separator$section_separator
  378                 echo
  379                 continue
  380             fi
  381             if [[ $(get_test_status $t) == "NFS_TEST" ]] && \
  382                [[ $nfs_tests == "no" ]]
  383             then
  384                 echo "Skipping nfs test file $t"
  385                 echo $section_separator$section_separator
  386                 echo
  387                 continue
  388             fi
  389             total_run_tests=$((total_run_tests+1))
  390             echo "[$(date +%H:%M:%S)] Running tests in file $t"
  391             starttime="$(date +%s)"
  392 
  393             local cmd_timeout=$run_timeout;
  394             if [ ${timeout_cmd_exists} == "yes" ]; then
  395                 if [ $(grep -c "SCRIPT_TIMEOUT=" ${t}) == 1 ] ; then
  396                     cmd_timeout=$(grep "SCRIPT_TIMEOUT=" ${t} | cut -f2 -d'=');
  397                     echo "Timeout set is ${cmd_timeout}, default ${run_timeout}"
  398                 fi
  399                 timeout --foreground -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t}
  400             else
  401                 prove -vmfe '/bin/bash' ${t}
  402             fi
  403             TMP_RES=$?
  404             ELAPSEDTIMEMAP[$t]=`expr $(date +%s) - $starttime`
  405             tar_logs "$t"
  406 
  407             # timeout always return 124 if it is actually a timeout.
  408             if ((${TMP_RES} == 124)); then
  409                 echo "${t} timed out after ${cmd_timeout} seconds"
  410             fi
  411 
  412             if [ ${TMP_RES} -ne 0 ]  && [ "x${retry}" = "xyes" ] ; then
  413                 echo "$t: bad status $TMP_RES"
  414                 echo ""
  415                 echo "       *********************************"
  416                 echo "       *       REGRESSION FAILED       *"
  417                 echo "       * Retrying failed tests in case *"
  418                 echo "       * we got some spurious failures *"
  419                 echo "       *********************************"
  420                 echo ""
  421 
  422                 if [ ${timeout_cmd_exists} == "yes" ]; then
  423                     timeout --foreground -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t}
  424                 else
  425                     prove -vmfe '/bin/bash' ${t}
  426                 fi
  427                 TMP_RES=$?
  428                 tar_logs "$t"
  429 
  430                 if ((${TMP_RES} == 124)); then
  431                     echo "${t} timed out after ${cmd_timeout} seconds"
  432                 fi
  433 
  434                 TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} "
  435             fi
  436 
  437 
  438             if [ ${TMP_RES} -ne 0 ] ; then
  439         if [[ "$t" == *"tests/000-flaky/"* ]]; then
  440                     FLAKY="${FLAKY}${t} "
  441             echo "FAILURE -> SUCCESS: Flaky test"
  442             TMP_RES=0
  443         else
  444                     RES=${TMP_RES}
  445                     FAILED="${FAILED}${t} "
  446         fi
  447             fi
  448 
  449             new_cores=$(ls /*-*.core 2> /dev/null | wc -l)
  450             if [ x"$new_cores" != x"$old_cores" ]; then
  451                 core_diff=$((new_cores-old_cores))
  452                 echo "$t: $core_diff new core files"
  453                 RES=1
  454                 GENERATED_CORE="${GENERATED_CORE}${t} "
  455             fi
  456             echo "End of test $t"
  457             echo $section_separator$section_separator
  458             echo
  459             if [ $RES -ne 0 ] && [ x"$exit_on_failure" = "xyes" ] ; then
  460                 break;
  461             fi
  462         fi
  463     done
  464     echo
  465     echo "Run complete"
  466     echo $section_separator$section_separator
  467     echo "Number of tests found:                             $total_tests"
  468     echo "Number of tests selected for run based on pattern: $selected_tests"
  469     echo "Number of tests skipped as they were marked bad:   $skipped_bad_tests"
  470     echo "Number of tests skipped because of known_issues:   $skipped_known_issue_tests"
  471     echo "Number of tests that were run:                     $total_run_tests"
  472 
  473     echo
  474     echo "Tests ordered by time taken, slowest to fastest: "
  475     echo $section_separator$section_separator
  476     for key in "${!ELAPSEDTIMEMAP[@]}"
  477     do
  478         echo "$key  -  ${ELAPSEDTIMEMAP["$key"]} second"
  479     done | sort -rn -k3
  480 
  481     # initialize the output file
  482     echo > "${result_output}"
  483 
  484     # Output the errors into a file
  485     if [ ${RES} -ne 0 ] ; then
  486         FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
  487         FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' )
  488         echo -e "\n$FAILED_COUNT test(s) failed \n${FAILED}" >> "${result_output}"
  489         GENERATED_CORE=$( echo  ${GENERATED_CORE} | tr ' ' '\n' | sort -u )
  490         GENERATED_CORE_COUNT=$( echo -n "${GENERATED_CORE}" | grep -c '^' )
  491         echo -e "\n$GENERATED_CORE_COUNT test(s) generated core \n${GENERATED_CORE}" >> "${result_output}"
  492         cat "${result_output}"
  493     fi
  494     TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u )
  495     RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' )
  496     if [ ${RETRY_COUNT} -ne 0 ] ; then
  497         echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}"
  498     fi
  499 
  500     FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u )
  501     RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' )
  502     if [ ${RETRY_COUNT} -ne 0 ] ; then
  503         echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}"
  504     fi
  505 
  506     echo
  507     echo "Result is $RES"
  508     echo
  509     return ${RES}
  510 }
  511 
  512 function run_head_tests()
  513 {
  514     [ -d ${regression_testsdir}/.git ] || return 0
  515 
  516     # The git command needs $cwd to be within the repository, but run_tests
  517     # needs it to be back where we started.
  518     pushd $regression_testsdir
  519     git_cmd="git diff-tree --no-commit-id --name-only --diff-filter=ACMRTUXB"
  520     htests=$($git_cmd -r HEAD tests | grep '.t$')
  521     popd
  522     [ -n "$htests" ] || return 0
  523 
  524     # Perhaps it's not ideal that we'll end up re-running these tests, but the
  525     # gains from letting them fail fast in the first round should outweigh the
  526     # losses from running them again in the second.  OTOH, with so many of our
  527     # tests being non-deterministic, maybe it doesn't hurt to give the newest
  528     # tests an extra workout.
  529     run_tests "$htests"
  530 }
  531 
  532 function parse_args () {
  533     args=`getopt frcbkphHno:t: "$@"`
  534     set -- $args
  535     while [ $# -gt 0 ]; do
  536         case "$1" in
  537         -f)    force="yes" ;;
  538         -h)    head="no" ;;
  539         -H)    head="only" ;;
  540         -r)    retry="yes" ;;
  541         -c)    exit_on_failure="no" ;;
  542         -b)    skip_bad_tests="no" ;;
  543         -k)    skip_known_bugs="no" ;;
  544         -p)    skip_preserve_logs="no" ;;
  545         -o)    result_output="$2"; shift;;
  546         -t)    run_timeout="$2"; shift;;
  547         -n)    nfs_tests="no";;
  548         --)    shift; break;;
  549         esac
  550         shift
  551     done
  552     tests="$@"
  553 }
  554 
  555 
  556 echo
  557 echo ... GlusterFS Test Framework ...
  558 echo
  559 
  560 # Get user options
  561 parse_args "$@"
  562 
  563 # Make sure we're running as the root user
  564 check_user
  565 
  566 # Make sure the needed programs are available
  567 check_dependencies
  568 
  569 # Check we're running from the right location
  570 check_location
  571 
  572 # Run the tests
  573 if [ x"$head" != x"no" ]; then
  574         run_head_tests || exit 1
  575 fi
  576 if [ x"$head" != x"only" ]; then
  577         run_tests "$tests" || exit 1
  578 fi