"Fossies" - the Fresh Open Source Software Archive 
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style:
standard) with prefixed line numbers.
Alternatively you can here
view or
download the uninterpreted source code file.
See also the latest
Fossies "Diffs" side-by-side code changes report for "TestNghttp.py":
1.15.16_vs_1.15.17.
1 ###################################################################################################
2 # Utility class for calling and analysing nghttp calls
3 #
4 # (c) 2019 greenbytes GmbH, Stefan Eissing
5 ###################################################################################################
6
7 import json
8 import pytest
9 import re
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import string
15 import time
16 import requests
17
18 from datetime import datetime
19 from datetime import tzinfo
20 from datetime import timedelta
21 from shutil import copyfile
22 from urllib.parse import urlparse
23
24 def _get_path(x):
25 return x["path"]
26
27 class Nghttp:
28
29 def __init__( self, path, connect_addr=None, tmp_dir="/tmp" ) :
30 self.NGHTTP = path
31 self.CONNECT_ADDR = connect_addr
32 self.TMP_DIR = tmp_dir
33
34 def get_stream( cls, streams, sid ) :
35 sid = int(sid)
36 if not sid in streams:
37 streams[sid] = {
38 "id" : sid,
39 "header" : {},
40 "request" : {
41 "id" : sid,
42 "body" : b''
43 },
44 "response" : {
45 "id" : sid,
46 "body" : b''
47 },
48 "paddings" : [],
49 "promises" : []
50 }
51 return streams[sid] if sid in streams else None
52
53 def run( self, urls, timeout, options ) :
54 return self._baserun(urls, timeout, options)
55
56 def complete_args(self, url, timeout, options: [str]) -> [str]:
57 if not isinstance(url, list):
58 url = [url]
59 u = urlparse(url[0])
60 args = [self.NGHTTP]
61 if self.CONNECT_ADDR:
62 connect_host = self.CONNECT_ADDR
63 args.append("--header=host: %s:%s" % (u.hostname, u.port))
64 else:
65 connect_host = u.hostname
66 if options:
67 args.extend(options)
68 for xurl in url:
69 xu = urlparse(xurl)
70 nurl = "%s://%s:%s/%s" % (u.scheme, connect_host, xu.port, xu.path)
71 if xu.query:
72 nurl = "%s?%s" % (nurl, xu.query)
73 args.append(nurl)
74 return args
75
76 def _baserun(self, url, timeout, options):
77 return self._run(self.complete_args(url, timeout, options))
78
79 def parse_output(self, btext):
80 # getting meta data and response body out of nghttp's output
81 # is a bit tricky. Without '-v' we just get the body. With '-v' meta
82 # data and timings in both directions are listed.
83 # We rely on response :status: to be unique and on
84 # response body not starting with space.
85 # Something not good enough for general purpose, but for these tests.
86 output = {}
87 body = ''
88 stream = 0
89 streams = {}
90 skip_indents = True
91 # chunk output into lines. nghttp mixes text
92 # meta output with bytes from the response body.
93 lines = [l.decode() for l in btext.split(b'\n')]
94 for lidx, l in enumerate(lines):
95 if len(l) == 0:
96 body += '\n'
97 continue
98 m = re.match(r'\[.*\] recv \(stream_id=(\d+)\) (\S+): (\S*)', l)
99 if m:
100 s = self.get_stream( streams, m.group(1) )
101 hname = m.group(2)
102 hval = m.group(3)
103 print("stream %d header %s: %s" % (s["id"], hname, hval))
104 header = s["header"]
105 if hname in header:
106 header[hname] += ", %s" % hval
107 else:
108 header[hname] = hval
109 body = ''
110 continue
111
112 m = re.match(r'\[.*\] recv HEADERS frame <.* stream_id=(\d+)>', l)
113 if m:
114 s = self.get_stream( streams, m.group(1) )
115 if s:
116 print("stream %d: recv %d header" % (s["id"], len(s["header"])))
117 response = s["response"]
118 hkey = "header"
119 if "header" in response:
120 h = response["header"]
121 if ":status" in h and int(h[":status"]) >= 200:
122 hkey = "trailer"
123 else:
124 prev = {
125 "header" : h
126 }
127 if "previous" in response:
128 prev["previous"] = response["previous"]
129 response["previous"] = prev
130 response[hkey] = s["header"]
131 s["header"] = {}
132 body = ''
133 continue
134
135 m = re.match(r'(.*)\[.*\] recv DATA frame <length=(\d+), .*stream_id=(\d+)>', l)
136 if m:
137 s = self.get_stream( streams, m.group(3) )
138 body += m.group(1)
139 blen = int(m.group(2))
140 if s:
141 print("stream %d: %d DATA bytes added" % (s["id"], blen))
142 padlen = 0
143 if len(lines) > lidx + 2:
144 mpad = re.match(r' +\(padlen=(\d+)\)', lines[lidx+2])
145 if mpad:
146 padlen = int(mpad.group(1))
147 s["paddings"].append(padlen)
148 blen -= padlen
149 s["response"]["body"] += body[-blen:].encode()
150 body = ''
151 skip_indents = True
152 continue
153
154 m = re.match(r'\[.*\] recv PUSH_PROMISE frame <.* stream_id=(\d+)>', l)
155 if m:
156 s = self.get_stream( streams, m.group(1) )
157 if s:
158 # headers we have are request headers for the PUSHed stream
159 # these have been received on the originating stream, the promised
160 # stream id it mentioned in the following lines
161 print("stream %d: %d PUSH_PROMISE header" % (s["id"], len(s["header"])))
162 if len(lines) > lidx+2:
163 m2 = re.match(r'\s+\(.*promised_stream_id=(\d+)\)', lines[lidx+2])
164 if m2:
165 s2 = self.get_stream( streams, m2.group(1) )
166 s2["request"]["header"] = s["header"]
167 s["promises"].append(s2)
168 s["header"] = {}
169 continue
170
171 m = re.match(r'(.*)\[.*\] recv (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
172 if m:
173 print("recv frame %s on stream %s" % (m.group(2), m.group(4)))
174 body += m.group(1)
175 skip_indents = True
176 continue
177
178 m = re.match(r'(.*)\[.*\] send (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
179 if m:
180 print("send frame %s on stream %s" % (m.group(2), m.group(4)))
181 body += m.group(1)
182 skip_indents = True
183 continue
184
185 if skip_indents and l.startswith(' '):
186 continue
187
188 if '[' != l[0]:
189 skip_indents = None
190 body += l + '\n'
191
192 # the main request is done on the lowest odd numbered id
193 main_stream = 99999999999
194 for sid in streams:
195 s = streams[sid]
196 if ":status" in s["response"]["header"]:
197 s["response"]["status"] = int(s["response"]["header"][":status"])
198 if (sid % 2) == 1 and sid < main_stream:
199 main_stream = sid
200
201 output["streams"] = streams
202 if main_stream in streams:
203 output["response"] = streams[main_stream]["response"]
204 output["paddings"] = streams[main_stream]["paddings"]
205 return output
206
207 def _raw( self, url, timeout, options ) :
208 args = [ "-v" ]
209 if options:
210 args.extend(options)
211 r = self._baserun( url, timeout, args )
212 if 0 == r["rv"]:
213 o = self.parse_output(r["out"]["raw"])
214 for name in o:
215 r[name] = o[name]
216 return r
217
218 def get( self, url, timeout=5, options=None ) :
219 return self._raw( url, timeout, options )
220
221 def assets( self, url, timeout=5, options=None ) :
222 if not options:
223 options = []
224 options.extend([ "-ans" ])
225 r = self._baserun( url, timeout, options )
226 assets = []
227 if 0 == r["rv"]:
228 lines = re.findall(r'[^\n]*\n', r["out"]["text"], re.MULTILINE)
229 for lidx, l in enumerate(lines):
230 m = re.match(r'\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+/(.*)', l)
231 if m:
232 assets.append({
233 "path" : m.group(7),
234 "status" : int(m.group(5)),
235 "size" : m.group(6)
236 })
237 assets.sort(key=_get_path)
238 r["assets"] = assets
239 return r
240
241 def post_data( self, url, data, timeout=5, options=None ) :
242 reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
243 with open(reqbody, 'wb') as f:
244 f.write(data.encode('utf-8'))
245 if not options:
246 options = []
247 options.extend([ "--data=%s" % reqbody ])
248 return self._raw( url, timeout, options )
249
250 def post_name( self, url, name, timeout=5, options=None ) :
251 reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
252 with open(reqbody, 'w') as f:
253 f.write("--DSAJKcd9876\n")
254 f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\n")
255 f.write("Content-Type: text/plain\n")
256 f.write("\n%s\n" % name)
257 f.write("--DSAJKcd9876\n")
258 if not options:
259 options = []
260 options.extend([ "--data=%s" % reqbody ])
261 return self._raw( url, timeout, options )
262
263 def upload( self, url, fpath, timeout=5, options=None ) :
264 if not options:
265 options = []
266 options.extend([ "--data=%s" % fpath ])
267 return self._raw( url, timeout, options )
268
269 def upload_file( self, url, fpath, timeout=5, options=None ) :
270 fname = os.path.basename(fpath)
271 reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
272 with open(fpath, 'rb') as fin:
273 with open(reqbody, 'wb') as f:
274 f.write(("""--DSAJKcd9876
275 Content-Disposition: form-data; name="xxx"; filename="xxxxx"
276 Content-Type: text/plain
277
278 testing mod_h2
279 --DSAJKcd9876
280 Content-Disposition: form-data; name="file"; filename="%s"
281 Content-Type: application/octet-stream
282 Content-Transfer-Encoding: binary
283
284 """ % (fname)).encode('utf-8'))
285 f.write(fin.read())
286 f.write("""
287 --DSAJKcd9876""".encode('utf-8'))
288 if not options:
289 options = []
290 options.extend([
291 "--data=%s" % reqbody,
292 "--expect-continue",
293 "-HContent-Type: multipart/form-data; boundary=DSAJKcd9876" ])
294 return self._raw( url, timeout, options )
295
296 def _run( self, args, input=None ) :
297 print(("execute: %s" % " ".join(args)))
298 p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
299 rv = p.returncode
300 print("stderr: %s" % p.stderr)
301 try:
302 jout = json.loads(p.stdout)
303 except:
304 jout = None
305 print("stdout: %s" % p.stdout)
306 return {
307 "rv": rv,
308 "out" : {
309 "raw" : p.stdout,
310 "text" : p.stdout.decode('utf-8'),
311 "err" : p.stderr.decode('utf-8'),
312 "json" : jout
313 }
314 }
315
316
317