"Fossies" - the Fresh Open Source Software Archive

Member "mod_http2-1.15.17/test/e2e/TestEnv.py" (5 Nov 2020, 13883 Bytes) of package /linux/www/apache_httpd_modules/mod_http2-1.15.17.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "TestEnv.py": 1.15.16_vs_1.15.17.

    1 ###################################################################################################
    2 # h2 end-to-end test environment class
    3 #
    4 # (c) 2019 greenbytes GmbH
    5 ###################################################################################################
    6 
    7 import json
    8 import pytest
    9 import re
   10 import os
   11 import shutil
   12 import subprocess
   13 import sys
   14 import string
   15 import time
   16 import requests
   17 
   18 from datetime import datetime
   19 from datetime import tzinfo
   20 from datetime import timedelta
   21 from configparser import ConfigParser
   22 from shutil import copyfile
   23 from urllib.parse import urlparse
   24 from TestNghttp import Nghttp
   25 
   26 class TestEnv:
   27 
   28     initialized = False
   29 
   30     PREFIX = "/usr"
   31     GEN_DIR = "gen"
   32     WEBROOT = "gen/apache"
   33     CURL = "curl"
   34     TEST_DIR = "test"
   35     NGHTTP = "nghttp"
   36     H2LOAD = "h2load"
   37 
   38     HTTP_PORT = 42001
   39     HTTPS_PORT = 42002
   40     HTTP_TLD = "tests.httpd.apache.org"
   41 
   42     APACHECTL = os.path.join(PREFIX, 'bin', 'apachectl')
   43 
   44     HTTPD_ADDR = "127.0.0.1"
   45     HTTP_URL = "http://{0}:{1}".format(HTTPD_ADDR, HTTP_PORT)
   46     HTTPS_URL = "https://{0}:{1}".format(HTTPD_ADDR, HTTPS_PORT)
   47 
   48     HTTPD_CONF_DIR = os.path.join(WEBROOT, "conf")
   49     HTTPD_DOCS_DIR = os.path.join(WEBROOT, "htdocs")
   50     HTTPD_LOGS_DIR = os.path.join(WEBROOT, "logs")
   51     HTTPD_TEST_CONF = os.path.join(HTTPD_CONF_DIR, "test.conf")
   52     E2E_DIR = os.path.join(TEST_DIR, "e2e")
   53 
   54     VERIFY_CERTIFICATES = False
   55 
   56     @classmethod
   57     def init( cls ) :
   58         if TestEnv.initialized:
   59             return
   60         cls.config = ConfigParser()
   61         cls.config.read('config.ini')
   62         
   63         cls.PREFIX      = cls.config.get('global', 'prefix')
   64         cls.GEN_DIR     = cls.config.get('global', 'gen_dir')
   65         cls.WEBROOT     = cls.config.get('global', 'server_dir')
   66         cls.CURL        = cls.config.get('global', 'curl_bin')
   67         cls.TEST_DIR    = cls.config.get('global', 'test_dir')
   68         cls.NGHTTP      = cls.config.get('global', 'nghttp')
   69         cls.H2LOAD      = cls.config.get('global', 'h2load')
   70 
   71         cls.HTTP_PORT   = cls.config.get('httpd', 'http_port')
   72         cls.HTTPS_PORT  = cls.config.get('httpd', 'https_port')
   73         cls.HTTP_TLD    = cls.config.get('httpd', 'http_tld')
   74 
   75         cls.APACHECTL  = os.path.join(cls.PREFIX, 'bin', 'apachectl')
   76 
   77         cls.HTTPD_ADDR = "127.0.0.1"
   78         cls.HTTP_URL   = "http://" + cls.HTTPD_ADDR + ":" + cls.HTTP_PORT
   79         cls.HTTPS_URL  = "https://" + cls.HTTPD_ADDR + ":" + cls.HTTPS_PORT
   80         
   81         cls.HTTPD_CONF_DIR = os.path.join(cls.WEBROOT, "conf")
   82         cls.HTTPD_DOCS_DIR = os.path.join(cls.WEBROOT, "htdocs")
   83         cls.HTTPD_LOGS_DIR = os.path.join(cls.WEBROOT, "logs")
   84         cls.HTTPD_TEST_CONF = os.path.join(cls.HTTPD_CONF_DIR, "test.conf")
   85         cls.E2E_DIR    = os.path.join(cls.TEST_DIR, "e2e")
   86 
   87         cls.VERIFY_CERTIFICATES = False
   88         
   89         if not os.path.exists(cls.GEN_DIR):
   90             os.makedirs(cls.GEN_DIR)
   91         
   92         TestEnv.initialized = True
   93 
   94 ###################################################################################################
   95 # check features
   96     @classmethod
   97     def has_h2load( cls ) :
   98         cls.init()
   99         return cls.H2LOAD != ""
  100 
  101     @classmethod
  102     def has_nghttp( cls ) :
  103         cls.init()
  104         return cls.NGHTTP != ""
  105 
  106     @classmethod
  107     def has_nghttp_get_assets ( cls ) :
  108         cls.init()
  109 
  110         if not TestEnv.has_nghttp():
  111             return False
  112 
  113         args = [cls.NGHTTP, "-a"]
  114         p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
  115         rv = p.returncode
  116 
  117         if rv != 0:
  118            return False
  119 
  120         return p.stderr == ""
  121 
  122 
  123 ###################################################################################################
  124 # path construction
  125 #
  126     @classmethod
  127     def mkpath( cls, path ) :
  128         if not os.path.exists(path):
  129             return os.makedirs(path)
  130 
  131 
  132     @classmethod
  133     def e2e_src( cls, path ) :
  134         return os.path.join(cls.E2E_DIR, path)
  135 
  136 ###################################################################################################
  137 # command execution
  138 #
  139     @classmethod
  140     def run( cls, args, input=None ) :
  141         print("execute: %s" % " ".join(args))
  142         p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
  143         rv = p.returncode
  144         print("exit code %d, stderr:\n%s" % (rv, p.stderr.decode('utf-8')))
  145         try:
  146             jout = json.loads(p.stdout)
  147         except:
  148             jout = None
  149             print("stdout:\n%s" % p.stdout.decode('utf-8'))
  150         return { 
  151             "rv": rv,
  152             "out" : {
  153                 "raw" : p.stdout,
  154                 "text" : p.stdout.decode('utf-8'),
  155                 "err" : p.stderr.decode('utf-8'),
  156                 "json" : jout
  157             } 
  158         }
  159 
  160     @classmethod
  161     def mkurl( cls, scheme, hostname, path='/' ) :
  162         port = cls.HTTPS_PORT if scheme == 'https' else cls.HTTP_PORT
  163         return "%s://%s.%s:%s%s" % (scheme, hostname, cls.HTTP_TLD, port, path)
  164 
  165 ###################################################################################################
  166 # http methods
  167 #
  168     @classmethod
  169     def is_live( cls, url, timeout ) :
  170         s = requests.Session()
  171         try_until = time.time() + timeout
  172         print("checking reachability of %s" % url)
  173         while time.time() < try_until:
  174             try:
  175                 req = requests.Request('HEAD', url).prepare()
  176                 resp = s.send(req, verify=cls.VERIFY_CERTIFICATES, timeout=timeout)
  177                 return True
  178             except IOError:
  179                 print("connect error: %s" % sys.exc_info()[0])
  180                 time.sleep(.2)
  181             except:
  182                 print("Unexpected error: %s" % sys.exc_info()[0])
  183                 time.sleep(.2)
  184         print("Unable to contact '%s' after %d sec" % (url, timeout))
  185         return False
  186 
  187     @classmethod
  188     def is_dead( cls, url, timeout ) :
  189         s = requests.Session()
  190         try_until = time.time() + timeout
  191         print("checking reachability of %s" % url)
  192         while time.time() < try_until:
  193             try:
  194                 req = requests.Request('HEAD', url).prepare()
  195                 resp = s.send(req, verify=cls.VERIFY_CERTIFICATES, timeout=timeout)
  196                 time.sleep(.2)
  197             except IOError:
  198                 return True
  199             except:
  200                 return True
  201         print("Server still responding after %d sec" % timeout)
  202         return False
  203 
  204 ###################################################################################################
  205 # apachectl
  206 #
  207     @classmethod
  208     def apachectl( cls, cmd, conf=None, check_live=True ) :
  209         if conf:
  210             cls.install_test_conf(conf)
  211         args = [cls.APACHECTL, "-d", cls.WEBROOT, "-k", cmd]
  212         print("execute: %s" % " ".join(args))
  213         cls.apachectl_stderr = ""
  214         p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)
  215         sys.stderr.write(p.stderr)
  216         rv = p.returncode
  217         if rv == 0:
  218             if check_live:
  219                 rv = 0 if cls.is_live(cls.HTTP_URL, 10) else -1
  220             else:
  221                 rv = 0 if cls.is_dead(cls.HTTP_URL, 10) else -1
  222                 print("waited for a apache.is_dead, rv=%d" % rv)
  223         return rv
  224 
  225     @classmethod
  226     def apache_restart( cls ) :
  227         return cls.apachectl( "graceful" )
  228         
  229     @classmethod
  230     def apache_start( cls ) :
  231         return cls.apachectl( "start" )
  232 
  233     @classmethod
  234     def apache_stop( cls ) :
  235         return cls.apachectl( "stop", check_live=False )
  236 
  237     @classmethod
  238     def apache_fail( cls ) :
  239         rv = cls.apachectl( "graceful", check_live=False )
  240         if rv != 0:
  241             print("check, if dead: %s" % cls.HTTPD_CHECK_URL)
  242             return 0 if cls.is_dead(cls.HTTPD_CHECK_URL, 5) else -1
  243         return rv
  244         
  245     @classmethod
  246     def install_test_conf( cls, conf=None) :
  247         if conf is None:
  248             conf_src = os.path.join("conf", "test.conf")
  249         elif os.path.isabs(conf):
  250             conf_src = conf
  251         else:
  252             conf_src = os.path.join("data", conf + ".conf")
  253         copyfile(conf_src, cls.HTTPD_TEST_CONF)
  254 
  255 ###################################################################################################
  256 # curl
  257 #
  258 
  259     @classmethod
  260     def curl_complete_args( cls, urls, timeout, options ):
  261         if not isinstance(urls, list):
  262             urls = [ urls ]
  263         u = urlparse(urls[0])
  264         headerfile = ("%s/curl.headers" % cls.GEN_DIR)
  265         if os.path.isfile(headerfile):
  266             os.remove(headerfile)
  267 
  268         args = [ 
  269             cls.CURL,
  270             "-ks", "-D", headerfile, 
  271             "--resolve", ("%s:%s:%s" % (u.hostname, u.port, cls.HTTPD_ADDR)),
  272             "--connect-timeout", ("%d" % timeout) 
  273         ]
  274         if options:
  275             args.extend(options)
  276         args += urls
  277         return args, headerfile
  278 
  279     @classmethod
  280     def curl_raw( cls, urls, timeout, options ):
  281         args, headerfile = cls.curl_complete_args(urls, timeout, options)
  282         r = cls.run( args )
  283         if r["rv"] == 0:
  284             lines = open(headerfile).readlines()
  285             exp_stat = True
  286             header = {}
  287             for line in lines:
  288                 if exp_stat:
  289                     print("reading 1st response line: %s" % line)
  290                     m = re.match(r'^(\S+) (\d+) (.*)$', line)
  291                     assert m
  292                     prev = r["response"] if "response" in r else None
  293                     r["response"] = {
  294                         "protocol"    : m.group(1), 
  295                         "status"      : int(m.group(2)), 
  296                         "description" : m.group(3),
  297                         "body"        : r["out"]["raw"]
  298                     }
  299                     if prev:
  300                         r["response"]["previous"] = prev
  301                     exp_stat = False
  302                     header = {}
  303                 elif re.match(r'^$', line):
  304                     exp_stat = True
  305                 else:
  306                     print("reading header line: %s" % line)
  307                     m = re.match(r'^([^:]+):\s*(.*)$', line)
  308                     assert m
  309                     header[ m.group(1).lower() ] = m.group(2)
  310             r["response"]["header"] = header
  311             if r["out"]["json"]:
  312                 r["response"]["json"] = r["out"]["json"] 
  313         return r
  314 
  315     @classmethod
  316     def curl_get( cls, url, timeout=5, options=None ) :
  317         return cls.curl_raw( url, timeout=timeout, options=options )
  318 
  319     @classmethod
  320     def curl_upload( cls, url, fpath, timeout=5, options=None ) :
  321         fname = os.path.basename(fpath)
  322         if not options:
  323             options = []
  324         options.extend([
  325             "--form", ("file=@%s" % (fpath))
  326         ])
  327         return cls.curl_raw( url, timeout, options )
  328 
  329     @classmethod
  330     def curl_post_data( cls, url, data="", timeout=5, options=None ) :
  331         if not options:
  332             options = []
  333         options.extend([ "--data", "%s" % data ])
  334         return cls.curl_raw( url, timeout, options )
  335 
  336     @classmethod
  337     def curl_post_value( cls, url, key, value, timeout=5, options=None ) :
  338         if not options:
  339             options = []
  340         options.extend([ "--form", "{0}={1}".format(key, value) ])
  341         return cls.curl_raw( url, timeout, options )
  342 
  343     @classmethod
  344     def curl_protocol_version( cls, url, timeout=5, options=None ) :
  345         if not options:
  346             options = []
  347         options.extend([ "-w", "%{http_version}\n", "-o", "/dev/null" ])
  348         r = cls.curl_raw( url, timeout=timeout, options=options )
  349         if r["rv"] == 0 and "response" in r:
  350             return r["response"]["body"].decode('utf-8').rstrip()
  351         return -1
  352         
  353 ###################################################################################################
  354 # nghttp
  355 #
  356     @classmethod
  357     def nghttp( cls ) :
  358         return Nghttp( cls.NGHTTP, connect_addr=cls.HTTPD_ADDR, tmp_dir=cls.GEN_DIR )
  359 
  360 
  361 ###################################################################################################
  362 # h2load
  363 #
  364     @classmethod
  365     def h2load_status( cls, run ) :
  366         m = re.search(r'requests: (\d+) total, (\d+) started, (\d+) done, (\d+) succeeded, (\d+) failed, (\d+) errored, (\d+) timeout', run["out"]["text"])
  367         if m:
  368             run["h2load"] = {
  369                 "requests" : {
  370                     "total" : int(m.group(1)),
  371                     "started" : int(m.group(2)),
  372                     "done" : int(m.group(3)),
  373                     "succeeded" : int(m.group(4)) 
  374                 }
  375             }
  376             m = re.search(r'status codes: (\d+) 2xx, (\d+) 3xx, (\d+) 4xx, (\d+) 5xx', run["out"]["text"])
  377             if m:
  378                 run["h2load"]["status"] = {
  379                     "2xx" : int(m.group(1)),
  380                     "3xx" : int(m.group(2)),
  381                     "4xx" : int(m.group(3)),
  382                     "5xx" : int(m.group(4))
  383                 }
  384         return run
  385 
  386 
  387 ###################################################################################################
  388 # generate some test data
  389 #
  390     @classmethod
  391     def setup_data_1k_1m( cls ):
  392         s100="012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678\n"
  393         with open(os.path.join(cls.GEN_DIR, "data-1k"), 'w') as f:
  394             for i in range(10):
  395                 f.write(s100)
  396         with open(os.path.join(cls.GEN_DIR, "data-10k"), 'w') as f:
  397             for i in range(100):
  398                 f.write(s100)
  399         with open(os.path.join(cls.GEN_DIR, "data-100k"), 'w') as f:
  400             for i in range(1000):
  401                 f.write(s100)
  402         with open(os.path.join(cls.GEN_DIR, "data-1m"), 'w') as f:
  403             for i in range(10000):
  404                 f.write(s100)
  405         
  406