"Fossies" - the Fresh Open Source Software Archive

Member "mod_http2-1.15.17/test/e2e/TestNghttp.py" (3 Nov 2020, 11827 Bytes) of package /linux/www/apache_httpd_modules/mod_http2-1.15.17.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "TestNghttp.py": 1.15.16_vs_1.15.17.

    1 ###################################################################################################
    2 # Utility class for calling and analysing nghttp calls
    3 #
    4 # (c) 2019 greenbytes GmbH, Stefan Eissing
    5 ###################################################################################################
    6 
    7 import json
    8 import pytest
    9 import re
   10 import os
   11 import shutil
   12 import subprocess
   13 import sys
   14 import string
   15 import time
   16 import requests
   17 
   18 from datetime import datetime
   19 from datetime import tzinfo
   20 from datetime import timedelta
   21 from shutil import copyfile
   22 from urllib.parse import urlparse
   23 
   24 def _get_path(x):
   25     return x["path"]
   26     
   27 class Nghttp:
   28 
   29     def __init__( self, path, connect_addr=None, tmp_dir="/tmp" ) :
   30         self.NGHTTP = path
   31         self.CONNECT_ADDR = connect_addr
   32         self.TMP_DIR = tmp_dir
   33 
   34     def get_stream( cls, streams, sid ) :
   35         sid = int(sid)
   36         if not sid in streams:
   37             streams[sid] = {
   38                     "id" : sid,
   39                     "header" : {},
   40                     "request" : {
   41                         "id" : sid,
   42                         "body" : b'' 
   43                     },
   44                     "response" : {
   45                         "id" : sid, 
   46                         "body" : b''
   47                     },
   48                     "paddings" : [],
   49                     "promises" : []
   50             }
   51         return streams[sid] if sid in streams else None
   52 
   53     def run( self, urls, timeout, options ) :
   54         return self._baserun(urls, timeout, options)
   55 
   56     def complete_args(self, url, timeout, options: [str]) -> [str]:
   57         if not isinstance(url, list):
   58             url = [url]
   59         u = urlparse(url[0])
   60         args = [self.NGHTTP]
   61         if self.CONNECT_ADDR:
   62             connect_host = self.CONNECT_ADDR
   63             args.append("--header=host: %s:%s" % (u.hostname, u.port))
   64         else:
   65             connect_host = u.hostname
   66         if options:
   67             args.extend(options)
   68         for xurl in url:
   69             xu = urlparse(xurl)
   70             nurl = "%s://%s:%s/%s" % (u.scheme, connect_host, xu.port, xu.path)
   71             if xu.query:
   72                 nurl = "%s?%s" % (nurl, xu.query)
   73             args.append(nurl)
   74         return args
   75 
   76     def _baserun(self, url, timeout, options):
   77         return self._run(self.complete_args(url, timeout, options))
   78     
   79     def parse_output(self, btext):
   80         # getting meta data and response body out of nghttp's output
   81         # is a bit tricky. Without '-v' we just get the body. With '-v' meta
   82         # data and timings in both directions are listed. 
   83         # We rely on response :status: to be unique and on 
   84         # response body not starting with space.
   85         # Something not good enough for general purpose, but for these tests.
   86         output = {}
   87         body = ''
   88         stream = 0
   89         streams = {}
   90         skip_indents = True
   91         # chunk output into lines. nghttp mixes text
   92         # meta output with bytes from the response body.
   93         lines = [l.decode() for l in btext.split(b'\n')]
   94         for lidx, l in enumerate(lines):
   95             if len(l) == 0:
   96                 body += '\n'
   97                 continue
   98             m = re.match(r'\[.*\] recv \(stream_id=(\d+)\) (\S+): (\S*)', l)
   99             if m:
  100                 s = self.get_stream( streams, m.group(1) )
  101                 hname = m.group(2)
  102                 hval = m.group(3)
  103                 print("stream %d header %s: %s" % (s["id"], hname, hval))
  104                 header = s["header"]
  105                 if hname in header: 
  106                     header[hname] += ", %s" % hval
  107                 else:
  108                     header[hname] = hval
  109                 body = ''
  110                 continue
  111 
  112             m = re.match(r'\[.*\] recv HEADERS frame <.* stream_id=(\d+)>', l)
  113             if m:
  114                 s = self.get_stream( streams, m.group(1) )
  115                 if s:
  116                     print("stream %d: recv %d header" % (s["id"], len(s["header"]))) 
  117                     response = s["response"]
  118                     hkey = "header"
  119                     if "header" in response:
  120                         h = response["header"]
  121                         if ":status" in h and int(h[":status"]) >= 200:
  122                             hkey = "trailer"
  123                         else:
  124                             prev = {
  125                                 "header" : h
  126                             }
  127                             if "previous" in response:
  128                                 prev["previous"] = response["previous"]
  129                             response["previous"] = prev
  130                     response[hkey] = s["header"]
  131                     s["header"] = {} 
  132                 body = ''
  133                 continue
  134             
  135             m = re.match(r'(.*)\[.*\] recv DATA frame <length=(\d+), .*stream_id=(\d+)>', l)
  136             if m:
  137                 s = self.get_stream( streams, m.group(3) )
  138                 body += m.group(1)
  139                 blen = int(m.group(2))
  140                 if s:
  141                     print("stream %d: %d DATA bytes added" % (s["id"], blen))
  142                     padlen = 0
  143                     if len(lines) > lidx + 2:
  144                         mpad = re.match(r' +\(padlen=(\d+)\)', lines[lidx+2])
  145                         if mpad: 
  146                             padlen = int(mpad.group(1))
  147                     s["paddings"].append(padlen)
  148                     blen -= padlen
  149                     s["response"]["body"] += body[-blen:].encode()
  150                 body = ''
  151                 skip_indents = True
  152                 continue
  153                 
  154             m = re.match(r'\[.*\] recv PUSH_PROMISE frame <.* stream_id=(\d+)>', l)
  155             if m:
  156                 s = self.get_stream( streams, m.group(1) )
  157                 if s:
  158                     # headers we have are request headers for the PUSHed stream
  159                     # these have been received on the originating stream, the promised
  160                     # stream id it mentioned in the following lines
  161                     print("stream %d: %d PUSH_PROMISE header" % (s["id"], len(s["header"])))
  162                     if len(lines) > lidx+2:
  163                         m2 = re.match(r'\s+\(.*promised_stream_id=(\d+)\)', lines[lidx+2])
  164                         if m2:
  165                             s2 = self.get_stream( streams, m2.group(1) )
  166                             s2["request"]["header"] = s["header"]
  167                             s["promises"].append(s2)
  168                     s["header"] = {} 
  169                 continue
  170                     
  171             m = re.match(r'(.*)\[.*\] recv (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
  172             if m:
  173                 print("recv frame %s on stream %s" % (m.group(2), m.group(4)))
  174                 body += m.group(1)
  175                 skip_indents = True
  176                 continue
  177                 
  178             m = re.match(r'(.*)\[.*\] send (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
  179             if m:
  180                 print("send frame %s on stream %s" % (m.group(2), m.group(4)))
  181                 body += m.group(1)
  182                 skip_indents = True
  183                 continue
  184                 
  185             if skip_indents and l.startswith('      '):
  186                 continue
  187             
  188             if '[' != l[0]:
  189                 skip_indents = None
  190                 body += l + '\n' 
  191                 
  192         # the main request is done on the lowest odd numbered id
  193         main_stream = 99999999999
  194         for sid in streams:
  195             s = streams[sid]
  196             if ":status" in s["response"]["header"]:
  197                 s["response"]["status"] = int(s["response"]["header"][":status"])
  198             if (sid % 2) == 1 and sid < main_stream:
  199                 main_stream = sid
  200         
  201         output["streams"] = streams
  202         if main_stream in streams:
  203             output["response"] = streams[main_stream]["response"]
  204             output["paddings"] = streams[main_stream]["paddings"]
  205         return output
  206     
  207     def _raw( self, url, timeout, options ) :
  208         args = [ "-v" ]
  209         if options:
  210             args.extend(options)
  211         r = self._baserun( url, timeout, args )
  212         if 0 == r["rv"]:
  213             o = self.parse_output(r["out"]["raw"])
  214             for name in o:
  215                 r[name] = o[name] 
  216         return r
  217 
  218     def get( self, url, timeout=5, options=None ) :
  219         return self._raw( url, timeout, options )
  220 
  221     def assets( self, url, timeout=5, options=None ) :
  222         if not options:
  223             options = []
  224         options.extend([ "-ans" ]) 
  225         r = self._baserun( url, timeout, options )
  226         assets = []
  227         if 0 == r["rv"]:
  228             lines = re.findall(r'[^\n]*\n', r["out"]["text"], re.MULTILINE)
  229             for lidx, l in enumerate(lines):
  230                 m = re.match(r'\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+/(.*)', l)
  231                 if m:
  232                     assets.append({
  233                         "path" : m.group(7),
  234                         "status" : int(m.group(5)),
  235                         "size" : m.group(6)
  236                     })
  237         assets.sort(key=_get_path)
  238         r["assets"] = assets
  239         return r
  240 
  241     def post_data( self, url, data, timeout=5, options=None ) :
  242         reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
  243         with open(reqbody, 'wb') as f:
  244             f.write(data.encode('utf-8'))
  245         if not options:
  246             options = []
  247         options.extend([ "--data=%s" % reqbody ])
  248         return self._raw( url, timeout, options )
  249 
  250     def post_name( self, url, name, timeout=5, options=None ) :
  251         reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
  252         with open(reqbody, 'w') as f:
  253             f.write("--DSAJKcd9876\n")
  254             f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\n")
  255             f.write("Content-Type: text/plain\n")
  256             f.write("\n%s\n" % name)
  257             f.write("--DSAJKcd9876\n")
  258         if not options:
  259             options = []
  260         options.extend([ "--data=%s" % reqbody ])
  261         return self._raw( url, timeout, options )
  262 
  263     def upload( self, url, fpath, timeout=5, options=None ) :
  264         if not options:
  265             options = []
  266         options.extend([ "--data=%s" % fpath ])
  267         return self._raw( url, timeout, options )
  268 
  269     def upload_file( self, url, fpath, timeout=5, options=None ) :
  270         fname = os.path.basename(fpath)
  271         reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
  272         with open(fpath, 'rb') as fin:
  273             with open(reqbody, 'wb') as f:
  274                 f.write(("""--DSAJKcd9876
  275 Content-Disposition: form-data; name="xxx"; filename="xxxxx"
  276 Content-Type: text/plain
  277 
  278 testing mod_h2
  279 --DSAJKcd9876
  280 Content-Disposition: form-data; name="file"; filename="%s"
  281 Content-Type: application/octet-stream
  282 Content-Transfer-Encoding: binary
  283 
  284 """ % (fname)).encode('utf-8'))
  285                 f.write(fin.read())
  286                 f.write("""
  287 --DSAJKcd9876""".encode('utf-8'))
  288         if not options:
  289             options = []
  290         options.extend([ 
  291             "--data=%s" % reqbody, 
  292             "--expect-continue", 
  293             "-HContent-Type: multipart/form-data; boundary=DSAJKcd9876" ])
  294         return self._raw( url, timeout, options )
  295 
  296     def _run( self, args, input=None ) :
  297         print(("execute: %s" % " ".join(args)))
  298         p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
  299         rv = p.returncode
  300         print("stderr: %s" % p.stderr)
  301         try:
  302             jout = json.loads(p.stdout)
  303         except:
  304             jout = None
  305             print("stdout: %s" % p.stdout)
  306         return { 
  307             "rv": rv,
  308             "out" : {
  309                 "raw" : p.stdout,
  310                 "text" : p.stdout.decode('utf-8'),
  311                 "err" : p.stderr.decode('utf-8'),
  312                 "json" : jout
  313             } 
  314         }
  315 
  316 
  317