"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.5.1/buildbot/test/unit/test_util_httpclientservice.py" (24 Nov 2019, 16524 Bytes) of package /linux/misc/buildbot-2.5.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 import datetime
   17 import json
   18 import os
   19 
   20 import mock
   21 
   22 from twisted.internet import defer
   23 from twisted.internet import reactor
   24 from twisted.python import components
   25 from twisted.python.compat import intToBytes
   26 from twisted.trial import unittest
   27 from twisted.web import resource
   28 from twisted.web import server
   29 
   30 from buildbot import interfaces
   31 from buildbot.test.fake import httpclientservice as fakehttpclientservice
   32 from buildbot.util import bytes2unicode
   33 from buildbot.util import httpclientservice
   34 from buildbot.util import service
   35 from buildbot.util import unicode2bytes
   36 
   37 try:
   38     from requests.auth import HTTPDigestAuth
   39 except ImportError:
   40     pass
   41 
   42 # There is no way to unregister an adapter, so we have no other option
   43 # than registering it as a module side effect :-(
   44 components.registerAdapter(
   45     lambda m: m,
   46     mock.Mock, interfaces.IHttpResponse)
   47 
   48 
   49 class HTTPClientServiceTestBase(unittest.SynchronousTestCase):
   50 
   51     def setUp(self):
   52         if httpclientservice.txrequests is None or httpclientservice.treq is None:
   53             raise unittest.SkipTest('this test requires txrequests and treq')
   54         self.patch(httpclientservice, 'txrequests', mock.Mock())
   55         self.patch(httpclientservice, 'treq', mock.Mock())
   56         self.parent = service.MasterService()
   57         self.parent.reactor = reactor
   58         self.base_headers = {}
   59         self.successResultOf(self.parent.startService())
   60 
   61 
   62 class HTTPClientServiceTestTxRequest(HTTPClientServiceTestBase):
   63 
   64     def setUp(self):
   65         super().setUp()
   66         self._http = self.successResultOf(
   67             httpclientservice.HTTPClientService.getService(self.parent, 'http://foo',
   68                                                            headers=self.base_headers))
   69 
   70     def test_get(self):
   71         self._http.get('/bar')
   72         self._http._session.request.assert_called_once_with('get', 'http://foo/bar', headers={},
   73                                                             background_callback=mock.ANY)
   74 
   75     def test_put(self):
   76         self._http.put('/bar', json={'foo': 'bar'})
   77         jsonStr = json.dumps(dict(foo='bar'))
   78         jsonBytes = unicode2bytes(jsonStr)
   79         self._http._session.request.assert_called_once_with('put', 'http://foo/bar',
   80                                                             background_callback=mock.ANY,
   81                                                             data=jsonBytes,
   82                                                             headers={'Content-Type': 'application/json'})
   83 
   84     def test_post(self):
   85         self._http.post('/bar', json={'foo': 'bar'})
   86         jsonStr = json.dumps(dict(foo='bar'))
   87         jsonBytes = unicode2bytes(jsonStr)
   88         self._http._session.request.assert_called_once_with('post', 'http://foo/bar',
   89                                                             background_callback=mock.ANY,
   90                                                             data=jsonBytes,
   91                                                             headers={'Content-Type': 'application/json'})
   92 
   93     def test_delete(self):
   94         self._http.delete('/bar')
   95         self._http._session.request.assert_called_once_with('delete', 'http://foo/bar',
   96                                                             background_callback=mock.ANY,
   97                                                             headers={})
   98 
   99     def test_post_headers(self):
  100         self.base_headers.update({'X-TOKEN': 'XXXYYY'})
  101         self._http.post('/bar', json={'foo': 'bar'})
  102         jsonStr = json.dumps(dict(foo='bar'))
  103         jsonBytes = unicode2bytes(jsonStr)
  104         self._http._session.request.assert_called_once_with('post', 'http://foo/bar',
  105                                                             background_callback=mock.ANY,
  106                                                             data=jsonBytes,
  107                                                             headers={
  108                                                                 'X-TOKEN': 'XXXYYY',
  109                                                                 'Content-Type': 'application/json'})
  110 
  111     def test_post_auth(self):
  112         self._http = self.successResultOf(
  113             httpclientservice.HTTPClientService.getService(self.parent, 'http://foo',
  114                                                            auth=('user', 'pa$$')))
  115         self._http.post('/bar', json={'foo': 'bar'})
  116         jsonStr = json.dumps(dict(foo='bar'))
  117         jsonBytes = unicode2bytes(jsonStr)
  118         self._http._session.request.assert_called_once_with('post', 'http://foo/bar',
  119                                                             background_callback=mock.ANY,
  120                                                             data=jsonBytes,
  121                                                             auth=(
  122                                                                 'user', 'pa$$'),
  123                                                             headers={
  124                                                                 'Content-Type': 'application/json'
  125                                                             })
  126 
  127 
  128 class HTTPClientServiceTestTReq(HTTPClientServiceTestBase):
  129 
  130     def setUp(self):
  131         super().setUp()
  132         self.patch(httpclientservice.HTTPClientService, 'PREFER_TREQ', True)
  133         self._http = self.successResultOf(
  134             httpclientservice.HTTPClientService.getService(self.parent, 'http://foo',
  135                                                            headers=self.base_headers))
  136 
  137     def test_get(self):
  138         self._http.get('/bar')
  139         httpclientservice.treq.get.assert_called_once_with('http://foo/bar',
  140                                                            agent=mock.ANY,
  141                                                            headers={})
  142 
  143     def test_put(self):
  144         self._http.put('/bar', json={'foo': 'bar'})
  145         httpclientservice.treq.put.assert_called_once_with('http://foo/bar',
  146                                                            agent=mock.ANY,
  147                                                            data=b'{"foo": "bar"}',
  148                                                            headers={'Content-Type': ['application/json']})
  149 
  150     def test_post(self):
  151         self._http.post('/bar', json={'foo': 'bar'})
  152         httpclientservice.treq.post.assert_called_once_with('http://foo/bar',
  153                                                             agent=mock.ANY,
  154                                                             data=b'{"foo": "bar"}',
  155                                                             headers={'Content-Type': ['application/json']})
  156 
  157     def test_delete(self):
  158         self._http.delete('/bar')
  159         httpclientservice.treq.delete.assert_called_once_with('http://foo/bar',
  160                                                               agent=mock.ANY,
  161                                                               headers={})
  162 
  163     def test_post_headers(self):
  164         self.base_headers.update({'X-TOKEN': 'XXXYYY'})
  165         self._http.post('/bar', json={'foo': 'bar'})
  166         httpclientservice.treq.post.assert_called_once_with('http://foo/bar',
  167                                                             agent=mock.ANY,
  168                                                             data=b'{"foo": "bar"}',
  169                                                             headers={
  170                                                                 'Content-Type': ['application/json'],
  171                                                                 'X-TOKEN': ['XXXYYY']})
  172 
  173     def test_post_auth(self):
  174         self._http = self.successResultOf(
  175             httpclientservice.HTTPClientService.getService(self.parent, 'http://foo',
  176                                                            auth=('user', 'pa$$')))
  177         self._http.post('/bar', json={'foo': 'bar'})
  178         httpclientservice.treq.post.assert_called_once_with('http://foo/bar',
  179                                                             agent=mock.ANY,
  180                                                             data=b'{"foo": "bar"}',
  181                                                             auth=(
  182                                                                 'user', 'pa$$'),
  183                                                             headers={
  184                                                                 'Content-Type': ['application/json'],
  185                                                             })
  186 
  187     def test_post_auth_digest(self):
  188         auth = HTTPDigestAuth('user', 'pa$$')
  189         self._http = self.successResultOf(
  190             httpclientservice.HTTPClientService.getService(self.parent, 'http://foo',
  191                                                            auth=auth))
  192         self._http.post('/bar', data={'foo': 'bar'})
  193         # if digest auth, we don't use treq! we use txrequests
  194         self._http._session.request.assert_called_once_with('post', 'http://foo/bar',
  195                                                             background_callback=mock.ANY,
  196                                                             data=dict(
  197                                                                 foo='bar'),
  198                                                             auth=auth,
  199                                                             headers={
  200                                                             })
  201 
  202 
  203 class MyResource(resource.Resource):
  204     isLeaf = True
  205 
  206     def render_GET(self, request):
  207         def decode(x):
  208             if isinstance(x, bytes):
  209                 return bytes2unicode(x)
  210             elif isinstance(x, (list, tuple)):
  211                 return [bytes2unicode(y) for y in x]
  212             elif isinstance(x, dict):
  213                 newArgs = {}
  214                 for a, b in x.items():
  215                     newArgs[decode(a)] = decode(b)
  216                 return newArgs
  217             return x
  218 
  219         args = decode(request.args)
  220         content_type = request.getHeader(b'content-type')
  221         if content_type == b"application/json":
  222             jsonBytes = request.content.read()
  223             jsonStr = bytes2unicode(jsonBytes)
  224             args['json_received'] = json.loads(jsonStr)
  225 
  226         data = json.dumps(args)
  227         data = unicode2bytes(data)
  228         request.setHeader(b'content-type', b'application/json')
  229         request.setHeader(b'content-length', intToBytes(len(data)))
  230         if request.method == b'HEAD':
  231             return b''
  232         return data
  233     render_HEAD = render_GET
  234     render_POST = render_GET
  235 
  236 
  237 class HTTPClientServiceTestTxRequestE2E(unittest.TestCase):
  238     """The e2e tests must be the same for txrequests and treq
  239 
  240     We just force treq in the other TestCase
  241     """
  242 
  243     def httpFactory(self, parent):
  244         return httpclientservice.HTTPClientService.getService(
  245             parent, 'http://127.0.0.1:{}'.format(self.port))
  246 
  247     def expect(self, *arg, **kwargs):
  248         pass
  249 
  250     @defer.inlineCallbacks
  251     def setUp(self):
  252         if httpclientservice.txrequests is None or httpclientservice.treq is None:
  253             raise unittest.SkipTest('this test requires txrequests and treq')
  254         site = server.Site(MyResource())
  255         self.listenport = reactor.listenTCP(0, site)
  256         self.port = self.listenport.getHost().port
  257         self.parent = parent = service.MasterService()
  258         self.parent.reactor = reactor
  259         yield parent.startService()
  260         self._http = yield self.httpFactory(parent)
  261 
  262     @defer.inlineCallbacks
  263     def tearDown(self):
  264         self.listenport.stopListening()
  265         yield self.parent.stopService()
  266 
  267     @defer.inlineCallbacks
  268     def test_content(self):
  269         self.expect('get', '/', content_json={})
  270         res = yield self._http.get('/')
  271         content = yield res.content()
  272         self.assertEqual(content, b'{}')
  273 
  274     @defer.inlineCallbacks
  275     def test_content_with_params(self):
  276         self.expect('get', '/', params=dict(a='b'), content_json=dict(a=['b']))
  277         res = yield self._http.get('/', params=dict(a='b'))
  278         content = yield res.content()
  279         self.assertEqual(content, b'{"a": ["b"]}')
  280 
  281     @defer.inlineCallbacks
  282     def test_post_content_with_params(self):
  283         self.expect('post', '/', params=dict(a='b'),
  284                     content_json=dict(a=['b']))
  285         res = yield self._http.post('/', params=dict(a='b'))
  286         content = yield res.content()
  287         self.assertEqual(content, b'{"a": ["b"]}')
  288 
  289     @defer.inlineCallbacks
  290     def test_put_content_with_data(self):
  291         self.expect('post', '/', data=dict(a='b'), content_json=dict(a=['b']))
  292         res = yield self._http.post('/', data=dict(a='b'))
  293         content = yield res.content()
  294         self.assertEqual(content, b'{"a": ["b"]}')
  295 
  296     @defer.inlineCallbacks
  297     def test_put_content_with_json(self):
  298         exp_content_json = dict(json_received=dict(a='b'))
  299         self.expect('post', '/', json=dict(a='b'),
  300                     content_json=exp_content_json)
  301         res = yield self._http.post('/', json=dict(a='b'))
  302         content = yield res.content()
  303         content = bytes2unicode(content)
  304         content = json.loads(content)
  305         self.assertEqual(content, exp_content_json)
  306 
  307     @defer.inlineCallbacks
  308     def test_put_content_with_json_datetime(self):
  309         exp_content_json = dict(json_received=dict(a='b', ts=12))
  310         dt = datetime.datetime.utcfromtimestamp(12)
  311         self.expect('post', '/', json=dict(a='b', ts=dt),
  312                     content_json=exp_content_json)
  313         res = yield self._http.post('/', json=dict(a='b', ts=dt))
  314         content = yield res.content()
  315         content = bytes2unicode(content)
  316         content = json.loads(content)
  317         self.assertEqual(content, exp_content_json)
  318 
  319     @defer.inlineCallbacks
  320     def test_json(self):
  321         self.expect('get', '/', content_json={})
  322         res = yield self._http.get('/')
  323         content = yield res.json()
  324         self.assertEqual(content, {})
  325         self.assertEqual(res.code, 200)
  326 
  327     # note that freebsd workers will not like when there are too many parallel connections
  328     # we can change this test via environment variable
  329     NUM_PARALLEL = os.environ.get("BBTEST_NUM_PARALLEL", 5)
  330 
  331     @defer.inlineCallbacks
  332     def test_lots(self):
  333         for i in range(self.NUM_PARALLEL):
  334             self.expect('get', '/', params=dict(a='b'),
  335                         content_json=dict(a=['b']))
  336         # use for benchmarking (txrequests: 3ms per request treq: 1ms per
  337         # request)
  338         for i in range(self.NUM_PARALLEL):
  339             res = yield self._http.get('/', params=dict(a='b'))
  340             content = yield res.content()
  341             self.assertEqual(content, b'{"a": ["b"]}')
  342 
  343     @defer.inlineCallbacks
  344     def test_lots_parallel(self):
  345         for i in range(self.NUM_PARALLEL):
  346             self.expect('get', '/', params=dict(a='b'),
  347                         content_json=dict(a=['b']))
  348 
  349         # use for benchmarking (txrequests: 3ms per request treq: 11ms per
  350         # request (!?))
  351         def oneReq():
  352             d = self._http.get('/', params=dict(a='b'))
  353 
  354             @d.addCallback
  355             def content(res):
  356                 return res.content()
  357 
  358             return d
  359         dl = [oneReq() for i in range(self.NUM_PARALLEL)]
  360         yield defer.gatherResults(dl)
  361 
  362 
  363 class HTTPClientServiceTestTReqE2E(HTTPClientServiceTestTxRequestE2E):
  364 
  365     def setUp(self):
  366         self.patch(httpclientservice.HTTPClientService, 'PREFER_TREQ', True)
  367         return super().setUp()
  368 
  369 
  370 class HTTPClientServiceTestFakeE2E(HTTPClientServiceTestTxRequestE2E):
  371 
  372     def httpFactory(self, parent):
  373         return fakehttpclientservice.HTTPClientService.getService(
  374             parent, 'http://127.0.0.1:{}'.format(self.port))
  375 
  376     def expect(self, *arg, **kwargs):
  377         self._http.expect(*arg, **kwargs)