"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.5.1/buildbot/www/rest.py" (24 Nov 2019, 21060 Bytes) of package /linux/misc/buildbot-2.5.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "rest.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.5.0_vs_2.5.1.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 import cgi
   17 import datetime
   18 import fnmatch
   19 import json
   20 import re
   21 from contextlib import contextmanager
   22 from urllib.parse import urlparse
   23 
   24 from twisted.internet import defer
   25 from twisted.python import log
   26 from twisted.web.error import Error
   27 
   28 from buildbot.data import exceptions
   29 from buildbot.data import resultspec
   30 from buildbot.util import bytes2unicode
   31 from buildbot.util import toJson
   32 from buildbot.util import unicode2bytes
   33 from buildbot.www import resource
   34 from buildbot.www.authz import Forbidden
   35 
   36 
   37 class BadRequest(Exception):
   38     pass
   39 
   40 
   41 class BadJsonRpc2(Exception):
   42 
   43     def __init__(self, message, jsonrpccode):
   44         self.message = message
   45         self.jsonrpccode = jsonrpccode
   46 
   47 
   48 class ContentTypeParser:
   49 
   50     def __init__(self, contenttype):
   51         self.typeheader = contenttype
   52 
   53     def gettype(self):
   54         mimetype, options = cgi.parse_header(
   55             bytes2unicode(self.typeheader))
   56         return mimetype
   57 
   58 
   59 URL_ENCODED = b"application/x-www-form-urlencoded"
   60 JSON_ENCODED = b"application/json"
   61 
   62 
   63 class RestRootResource(resource.Resource):
   64     version_classes = {}
   65 
   66     @classmethod
   67     def addApiVersion(cls, version, version_cls):
   68         cls.version_classes[version] = version_cls
   69         version_cls.apiVersion = version
   70 
   71     def __init__(self, master):
   72         super().__init__(master)
   73 
   74         min_vers = master.config.www.get('rest_minimum_version', 0)
   75         latest = max(list(self.version_classes))
   76         for version, klass in self.version_classes.items():
   77             if version < min_vers:
   78                 continue
   79             child = klass(master)
   80             child_path = 'v{}'.format(version)
   81             child_path = unicode2bytes(child_path)
   82             self.putChild(child_path, child)
   83             if version == latest:
   84                 self.putChild(b'latest', child)
   85 
   86     def render(self, request):
   87         request.setHeader(b"content-type", JSON_ENCODED)
   88         min_vers = self.master.config.www.get('rest_minimum_version', 0)
   89         api_versions = dict(('v%d' % v, '%sapi/v%d' % (self.base_url, v))
   90                             for v in self.version_classes
   91                             if v > min_vers)
   92         data = json.dumps(dict(api_versions=api_versions))
   93         return unicode2bytes(data)
   94 
   95 
   96 JSONRPC_CODES = dict(parse_error=-32700,
   97                      invalid_request=-32600,
   98                      method_not_found=-32601,
   99                      invalid_params=-32602,
  100                      internal_error=-32603)
  101 
  102 
  103 class V2RootResource(resource.Resource):
  104 
  105     # For GETs, this API follows http://jsonapi.org.  The getter API does not
  106     # permit create, update, or delete, so this is limited to reading.
  107     #
  108     # Data API control methods can be invoked via a POST to the appropriate
  109     # URL.  These follow http://www.jsonrpc.org/specification, with a few
  110     # limitations:
  111     # - params as list is not supported
  112     # - rpc call batching is not supported
  113     # - jsonrpc2 notifications are not supported (you always get an answer)
  114 
  115     # rather than construct the entire possible hierarchy of Rest resources,
  116     # this is marked as a leaf node, and any remaining path items are parsed
  117     # during rendering
  118     isLeaf = True
  119 
  120     # enable reconfigResource calls
  121     needsReconfig = True
  122 
  123     @defer.inlineCallbacks
  124     def getEndpoint(self, request, method, params):
  125         # note that trailing slashes are not allowed
  126         request_postpath = tuple(bytes2unicode(p) for p in request.postpath)
  127         yield self.master.www.assertUserAllowed(request, request_postpath,
  128                                                 method, params)
  129         ret = yield self.master.data.getEndpoint(request_postpath)
  130         return ret
  131 
  132     @contextmanager
  133     def handleErrors(self, writeError):
  134         try:
  135             yield
  136         except exceptions.InvalidPathError as e:
  137             msg = unicode2bytes(e.args[0])
  138             writeError(msg or b"invalid path", errcode=404,
  139                        jsonrpccode=JSONRPC_CODES['invalid_request'])
  140             return
  141         except exceptions.InvalidControlException as e:
  142             msg = unicode2bytes(str(e))
  143             writeError(msg or b"invalid control action", errcode=501,
  144                        jsonrpccode=JSONRPC_CODES["method_not_found"])
  145             return
  146         except BadRequest as e:
  147             msg = unicode2bytes(e.args[0])
  148             writeError(msg or b"invalid request", errcode=400,
  149                        jsonrpccode=JSONRPC_CODES["method_not_found"])
  150             return
  151         except BadJsonRpc2 as e:
  152             msg = unicode2bytes(e.message)
  153             writeError(msg, errcode=400, jsonrpccode=e.jsonrpccode)
  154             return
  155         except Forbidden as e:
  156             # There is nothing in jsonrc spec about forbidden error, so pick
  157             # invalid request
  158             msg = unicode2bytes(e.message)
  159             writeError(
  160                 msg, errcode=403, jsonrpccode=JSONRPC_CODES["invalid_request"])
  161             return
  162         except Exception as e:
  163             log.err(_why='while handling API request')
  164             msg = unicode2bytes(repr(e))
  165             writeError(repr(e), errcode=500,
  166                        jsonrpccode=JSONRPC_CODES["internal_error"])
  167             return
  168 
  169     # JSONRPC2 support
  170 
  171     def decodeJsonRPC2(self, request):
  172         # Verify the content-type.  Browsers are easily convinced to send
  173         # POST data to arbitrary URLs via 'form' elements, but they won't
  174         # use the application/json content-type.
  175         if ContentTypeParser(request.getHeader(b'content-type')).gettype() != "application/json":
  176             raise BadJsonRpc2('Invalid content-type (use application/json)',
  177                               JSONRPC_CODES["invalid_request"])
  178 
  179         try:
  180             data = json.loads(bytes2unicode(request.content.read()))
  181         except Exception as e:
  182             raise BadJsonRpc2("JSON parse error: %s" % (str(e),),
  183                               JSONRPC_CODES["parse_error"])
  184 
  185         if isinstance(data, list):
  186             raise BadJsonRpc2("JSONRPC batch requests are not supported",
  187                               JSONRPC_CODES["invalid_request"])
  188         if not isinstance(data, dict):
  189             raise BadJsonRpc2("JSONRPC root object must be an object",
  190                               JSONRPC_CODES["invalid_request"])
  191 
  192         def check(name, types, typename):
  193             if name not in data:
  194                 raise BadJsonRpc2("missing key '%s'" % (name,),
  195                                   JSONRPC_CODES["invalid_request"])
  196             if not isinstance(data[name], types):
  197                 raise BadJsonRpc2("'%s' must be %s" % (name, typename),
  198                                   JSONRPC_CODES["invalid_request"])
  199         check("jsonrpc", (str,), "a string")
  200         check("method", (str,), "a string")
  201         check("id", (str, int, type(None)),
  202               "a string, number, or null")
  203         check("params", (dict,), "an object")
  204         if data['jsonrpc'] != '2.0':
  205             raise BadJsonRpc2("only JSONRPC 2.0 is supported",
  206                               JSONRPC_CODES['invalid_request'])
  207         return data["method"], data["id"], data['params']
  208 
  209     @defer.inlineCallbacks
  210     def renderJsonRpc(self, request):
  211         jsonRpcReply = {'jsonrpc': "2.0"}
  212 
  213         def writeError(msg, errcode=399,
  214                        jsonrpccode=JSONRPC_CODES["internal_error"]):
  215             if isinstance(msg, bytes):
  216                 msg = bytes2unicode(msg)
  217             if self.debug:
  218                 log.msg("JSONRPC error: %s" % (msg,))
  219             request.setResponseCode(errcode)
  220             request.setHeader(b'content-type', JSON_ENCODED)
  221             if "error" not in jsonRpcReply:  # already filled in by caller
  222                 jsonRpcReply['error'] = dict(code=jsonrpccode, message=msg)
  223             data = json.dumps(jsonRpcReply)
  224             data = unicode2bytes(data)
  225             request.write(data)
  226 
  227         with self.handleErrors(writeError):
  228             method, id, params = self.decodeJsonRPC2(request)
  229             jsonRpcReply['id'] = id
  230             ep, kwargs = yield self.getEndpoint(request, method, params)
  231             userinfos = self.master.www.getUserInfos(request)
  232             if 'anonymous' in userinfos and userinfos['anonymous']:
  233                 owner = "anonymous"
  234             else:
  235                 owner = userinfos['email']
  236             params['owner'] = owner
  237 
  238             result = yield ep.control(method, params, kwargs)
  239             jsonRpcReply['result'] = result
  240 
  241             data = json.dumps(jsonRpcReply, default=toJson,
  242                               sort_keys=True, separators=(',', ':'))
  243 
  244             request.setHeader(b'content-type', JSON_ENCODED)
  245             if request.method == b"HEAD":
  246                 request.setHeader(b"content-length", unicode2bytes(str(len(data))))
  247                 request.write(b'')
  248             else:
  249                 data = unicode2bytes(data)
  250                 request.write(data)
  251 
  252     # JSONAPI support
  253     def decodeResultSpec(self, request, endpoint):
  254         reqArgs = request.args
  255 
  256         def checkFields(fields, negOk=False):
  257             for field in fields:
  258                 k = bytes2unicode(field)
  259                 if k[0] == '-' and negOk:
  260                     k = k[1:]
  261                 if k not in entityType.fieldNames:
  262                     raise BadRequest("no such field '{}'".format(k))
  263 
  264         entityType = endpoint.rtype.entityType
  265         limit = offset = order = fields = None
  266         filters, properties = [], []
  267         for arg in reqArgs:
  268             argStr = bytes2unicode(arg)
  269             if arg == b'order':
  270                 order = tuple([bytes2unicode(o) for o in reqArgs[arg]])
  271                 checkFields(order, True)
  272             elif arg == b'field':
  273                 fields = reqArgs[arg]
  274                 checkFields(fields, False)
  275             elif arg == b'limit':
  276                 try:
  277                     limit = int(reqArgs[arg][0])
  278                 except Exception:
  279                     raise BadRequest('invalid limit')
  280             elif arg == b'offset':
  281                 try:
  282                     offset = int(reqArgs[arg][0])
  283                 except Exception:
  284                     raise BadRequest('invalid offset')
  285             elif arg == b'property':
  286                 try:
  287                     props = []
  288                     for v in reqArgs[arg]:
  289                         if not isinstance(v, (bytes, str)):
  290                             raise TypeError(
  291                                 "Invalid type {} for {}".format(type(v), v))
  292                         props.append(bytes2unicode(v))
  293                 except Exception:
  294                     raise BadRequest(
  295                         'invalid property value for {}'.format(arg))
  296                 properties.append(resultspec.Property(arg, 'eq', props))
  297             elif argStr in entityType.fieldNames:
  298                 field = entityType.fields[argStr]
  299                 try:
  300                     values = [field.valueFromString(v) for v in reqArgs[arg]]
  301                 except Exception:
  302                     raise BadRequest(
  303                         'invalid filter value for {}'.format(argStr))
  304 
  305                 filters.append(resultspec.Filter(argStr, 'eq', values))
  306             elif '__' in argStr:
  307                 field, op = argStr.rsplit('__', 1)
  308                 args = reqArgs[arg]
  309                 operators = (resultspec.Filter.singular_operators
  310                              if len(args) == 1
  311                              else resultspec.Filter.plural_operators)
  312                 if op in operators and field in entityType.fieldNames:
  313                     fieldType = entityType.fields[field]
  314                     try:
  315                         values = [fieldType.valueFromString(v)
  316                                   for v in reqArgs[arg]]
  317                     except Exception:
  318                         raise BadRequest(
  319                             'invalid filter value for {}'.format(argStr))
  320                     filters.append(resultspec.Filter(field, op, values))
  321             else:
  322                 raise BadRequest(
  323                     "unrecognized query parameter '{}'".format(argStr))
  324 
  325         # if ordering or filtering is on a field that's not in fields, bail out
  326         if fields:
  327             fields = [bytes2unicode(f) for f in fields]
  328             fieldsSet = set(fields)
  329             if order and {o.lstrip('-') for o in order} - fieldsSet:
  330                 raise BadRequest("cannot order on un-selected fields")
  331             for filter in filters:
  332                 if filter.field not in fieldsSet:
  333                     raise BadRequest("cannot filter on un-selected fields")
  334 
  335         # build the result spec
  336         rspec = resultspec.ResultSpec(fields=fields, limit=limit, offset=offset,
  337                                       order=order, filters=filters, properties=properties)
  338 
  339         # for singular endpoints, only allow fields
  340         if not endpoint.isCollection:
  341             if rspec.filters:
  342                 raise BadRequest("this is not a collection")
  343 
  344         return rspec
  345 
  346     def encodeRaw(self, data, request):
  347         request.setHeader(b"content-type",
  348                           unicode2bytes(data['mime-type']) + b'; charset=utf-8')
  349         request.setHeader(b"content-disposition",
  350                           b'attachment; filename=' + unicode2bytes(data['filename']))
  351         request.write(unicode2bytes(data['raw']))
  352         return
  353 
  354     @defer.inlineCallbacks
  355     def renderRest(self, request):
  356         def writeError(msg, errcode=404, jsonrpccode=None):
  357             if self.debug:
  358                 log.msg("REST error: %s" % (msg,))
  359             request.setResponseCode(errcode)
  360             request.setHeader(b'content-type', b'text/plain; charset=utf-8')
  361             msg = bytes2unicode(msg)
  362             data = json.dumps(dict(error=msg))
  363             data = unicode2bytes(data)
  364             request.write(data)
  365 
  366         with self.handleErrors(writeError):
  367             ep, kwargs = yield self.getEndpoint(request, bytes2unicode(request.method), {})
  368 
  369             rspec = self.decodeResultSpec(request, ep)
  370             data = yield ep.get(rspec, kwargs)
  371             if data is None:
  372                 msg = ("not found while getting from {} with "
  373                        "arguments {} and {}").format(repr(ep), repr(rspec),
  374                                                      str(kwargs))
  375                 msg = unicode2bytes(msg)
  376                 writeError(msg, errcode=404)
  377                 return
  378 
  379             if ep.isRaw:
  380                 self.encodeRaw(data, request)
  381                 return
  382 
  383             # post-process any remaining parts of the resultspec
  384             data = rspec.apply(data)
  385 
  386             # annotate the result with some metadata
  387             meta = {}
  388             if ep.isCollection:
  389                 offset, total = data.offset, data.total
  390                 if offset is None:
  391                     offset = 0
  392 
  393                 # add total, if known
  394                 if total is not None:
  395                     meta['total'] = total
  396 
  397                 # get the real list instance out of the ListResult
  398                 data = data.data
  399             else:
  400                 data = [data]
  401 
  402             typeName = ep.rtype.plural
  403             data = {
  404                 typeName: data,
  405                 'meta': meta
  406             }
  407 
  408             # set up the content type and formatting options; if the request
  409             # accepts text/html or text/plain, the JSON will be rendered in a
  410             # readable, multiline format.
  411 
  412             if b'application/json' in (request.getHeader(b'accept') or b''):
  413                 compact = True
  414                 request.setHeader(b"content-type",
  415                                   b'application/json; charset=utf-8')
  416             else:
  417                 compact = False
  418                 request.setHeader(b"content-type",
  419                                   b'text/plain; charset=utf-8')
  420 
  421             # set up caching
  422             if self.cache_seconds:
  423                 now = datetime.datetime.utcnow()
  424                 expires = now + datetime.timedelta(seconds=self.cache_seconds)
  425                 expiresBytes = unicode2bytes(
  426                     expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
  427                 request.setHeader(b"Expires", expiresBytes)
  428                 request.setHeader(b"Pragma", b"no-cache")
  429 
  430             # filter out blanks if necessary and render the data
  431             if compact:
  432                 data = json.dumps(data, default=toJson,
  433                                   sort_keys=True, separators=(',', ':'))
  434             else:
  435                 data = json.dumps(data, default=toJson,
  436                                   sort_keys=True, indent=2)
  437 
  438             if request.method == b"HEAD":
  439                 request.setHeader(b"content-length", unicode2bytes(str(len(data))))
  440             else:
  441                 data = unicode2bytes(data)
  442                 request.write(data)
  443 
  444     def reconfigResource(self, new_config):
  445         # buildbotURL may contain reverse proxy path, Origin header is just
  446         # scheme + host + port
  447         buildbotURL = urlparse(unicode2bytes(new_config.buildbotURL))
  448         origin_self = buildbotURL.scheme + b"://" + buildbotURL.netloc
  449         # pre-translate the origin entries in the config
  450         self.origins = []
  451         for o in new_config.www.get('allowed_origins', [origin_self]):
  452             origin = bytes2unicode(o).lower()
  453             self.origins.append(re.compile(fnmatch.translate(origin)))
  454 
  455         # and copy some other flags
  456         self.debug = new_config.www.get('debug')
  457         self.cache_seconds = new_config.www.get('json_cache_seconds', 0)
  458 
  459     def render(self, request):
  460         def writeError(msg, errcode=400):
  461             msg = bytes2unicode(msg)
  462             if self.debug:
  463                 log.msg("HTTP error: %s" % (msg,))
  464             request.setResponseCode(errcode)
  465             request.setHeader(b'content-type', b'text/plain; charset=utf-8')
  466             if request.method == b'POST':
  467                 # jsonRPC callers want the error message in error.message
  468                 data = json.dumps(dict(error=dict(message=msg)))
  469                 data = unicode2bytes(data)
  470                 request.write(data)
  471             else:
  472                 data = json.dumps(dict(error=msg))
  473                 data = unicode2bytes(data)
  474                 request.write(data)
  475             request.finish()
  476         return self.asyncRenderHelper(request, self.asyncRender, writeError)
  477 
  478     @defer.inlineCallbacks
  479     def asyncRender(self, request):
  480 
  481         # Handle CORS, if necessary.
  482         origins = self.origins
  483         if origins is not None:
  484             isPreflight = False
  485             reqOrigin = request.getHeader(b'origin')
  486             if reqOrigin:
  487                 err = None
  488                 reqOrigin = reqOrigin.lower()
  489                 if not any(o.match(bytes2unicode(reqOrigin)) for o in self.origins):
  490                     err = b"invalid origin"
  491                 elif request.method == b'OPTIONS':
  492                     preflightMethod = request.getHeader(
  493                         b'access-control-request-method')
  494                     if preflightMethod not in (b'GET', b'POST', b'HEAD'):
  495                         err = b'invalid method'
  496                     isPreflight = True
  497                 if err:
  498                     raise Error(400, err)
  499 
  500                 # If it's OK, then let the browser know we checked it out.  The
  501                 # Content-Type header is included here because CORS considers
  502                 # content types other than form data and text/plain to not be
  503                 # simple.
  504                 request.setHeader(b"access-control-allow-origin", reqOrigin)
  505                 request.setHeader(b"access-control-allow-headers",
  506                                   b"Content-Type")
  507                 request.setHeader(b"access-control-max-age", b'3600')
  508 
  509                 # if this was a preflight request, we're done
  510                 if isPreflight:
  511                     return b""
  512 
  513         # based on the method, this is either JSONRPC or REST
  514         if request.method == b'POST':
  515             res = yield self.renderJsonRpc(request)
  516         elif request.method in (b'GET', b'HEAD'):
  517             res = yield self.renderRest(request)
  518         else:
  519             raise Error(400, b"invalid HTTP method")
  520 
  521         return res
  522 
  523 
  524 RestRootResource.addApiVersion(2, V2RootResource)