"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.3.1/buildbot/test/util/validation.py" (23 May 2019, 20699 Bytes) of package /linux/misc/buildbot-2.3.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "validation.py": 2.1.0_vs_2.2.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 # See "Type Validation" in master/docs/developer/tests.rst
   17 
   18 import datetime
   19 import json
   20 import re
   21 
   22 from buildbot.util import UTC
   23 from buildbot.util import bytes2unicode
   24 
   25 # Base class
   26 
   27 validatorsByName = {}
   28 
   29 
   30 class Validator:
   31 
   32     name = None
   33     hasArgs = False
   34 
   35     def validate(self, name, object):
   36         raise NotImplementedError
   37 
   38     class __metaclass__(type):
   39 
   40         def __new__(mcs, name, bases, attrs):
   41             cls = type.__new__(mcs, name, bases, attrs)
   42             if 'name' in attrs and attrs['name']:
   43                 assert attrs['name'] not in validatorsByName
   44                 validatorsByName[attrs['name']] = cls
   45             return cls
   46 
   47 
   48 # Basic types
   49 
   50 class InstanceValidator(Validator):
   51     types = ()
   52 
   53     def validate(self, name, object):
   54         if not isinstance(object, self.types):
   55             yield "{} ({!r}) is not a {}".format(
   56                 name, object, self.name or repr(self.types))
   57 
   58 
   59 class IntValidator(InstanceValidator):
   60     types = (int,)
   61     name = 'integer'
   62 
   63 
   64 class BooleanValidator(InstanceValidator):
   65     types = (bool,)
   66     name = 'boolean'
   67 
   68 
   69 class StringValidator(InstanceValidator):
   70     # strings must be unicode
   71     types = (str,)
   72     name = 'string'
   73 
   74 
   75 class BinaryValidator(InstanceValidator):
   76     types = (bytes,)
   77     name = 'bytestring'
   78 
   79 
   80 class StrValidator(InstanceValidator):
   81     types = (str,)
   82     name = 'str'
   83 
   84 
   85 class DateTimeValidator(Validator):
   86     types = (datetime.datetime,)
   87     name = 'datetime'
   88 
   89     def validate(self, name, object):
   90         if not isinstance(object, datetime.datetime):
   91             yield "{} - {!r} - is not a datetime".format(name, object)
   92         elif object.tzinfo != UTC:
   93             yield "{} is not a UTC datetime".format(name)
   94 
   95 
   96 class IdentifierValidator(Validator):
   97     types = (str,)
   98     name = 'identifier'
   99     hasArgs = True
  100 
  101     ident_re = re.compile('^[a-zA-Z\u00a0-\U0010ffff_-][a-zA-Z0-9\u00a0-\U0010ffff_-]*$', flags=re.UNICODE)
  102 
  103     def __init__(self, len):
  104         self.len = len
  105 
  106     def validate(self, name, object):
  107         if not isinstance(object, str):
  108             yield "{} - {!r} - is not a unicode string".format(name, object)
  109         elif not self.ident_re.match(object):
  110             yield "{} - {!r} - is not an identifier".format(name, object)
  111         elif not object:
  112             yield "{} - identifiers cannot be an empty string".format(name)
  113         elif len(object) > self.len:
  114             yield "{} - {!r} - is longer than {} characters".format(
  115                 name, object, self.len)
  116 
  117 # Miscellaneous
  118 
  119 
  120 class NoneOk:
  121 
  122     def __init__(self, original):
  123         self.original = original
  124 
  125     def validate(self, name, object):
  126         if object is None:
  127             return
  128         else:
  129             for msg in self.original.validate(name, object):
  130                 yield msg
  131 
  132 
  133 class Any:
  134 
  135     def validate(self, name, object):
  136         return
  137 
  138 # Compound Types
  139 
  140 
  141 class DictValidator(Validator):
  142 
  143     name = 'dict'
  144 
  145     def __init__(self, optionalNames=None, **keys):
  146         if optionalNames is None:
  147             optionalNames = []
  148         self.optionalNames = set(optionalNames)
  149         self.keys = keys
  150         self.expectedNames = set(keys.keys())
  151 
  152     def validate(self, name, object):
  153         # this uses isinstance, allowing dict subclasses as used by the DB API
  154         if not isinstance(object, dict):
  155             yield "{} ({!r}) is not a dictionary (got type {})".format(
  156                 name, object, type(object))
  157             return
  158 
  159         gotNames = set(object.keys())
  160 
  161         unexpected = gotNames - self.expectedNames
  162         if unexpected:
  163             yield "{} has unexpected keys {}".format(name,
  164                                                  ", ".join([repr(n) for n in unexpected]))
  165 
  166         missing = self.expectedNames - self.optionalNames - gotNames
  167         if missing:
  168             yield "{} is missing keys {}".format(name,
  169                                              ", ".join([repr(n) for n in missing]))
  170 
  171         for k in gotNames & self.expectedNames:
  172             for msg in self.keys[k].validate("{}[{!r}]".format(name, k), object[k]):
  173                 yield msg
  174 
  175 
  176 class SequenceValidator(Validator):
  177     type = None
  178 
  179     def __init__(self, elementValidator):
  180         self.elementValidator = elementValidator
  181 
  182     def validate(self, name, object):
  183         if not isinstance(object, self.type):
  184             yield "{} ({!r}) is not a {}".format(name, object, self.name)
  185             return
  186 
  187         for idx, elt in enumerate(object):
  188             for msg in self.elementValidator.validate("{}[{}]".format(name, idx),
  189                                                       elt):
  190                 yield msg
  191 
  192 
  193 class ListValidator(SequenceValidator):
  194     type = list
  195     name = 'list'
  196 
  197 
  198 class TupleValidator(SequenceValidator):
  199     type = tuple
  200     name = 'tuple'
  201 
  202 
  203 class StringListValidator(ListValidator):
  204     name = 'string-list'
  205 
  206     def __init__(self):
  207         super().__init__(StringValidator())
  208 
  209 
  210 class SourcedPropertiesValidator(Validator):
  211 
  212     name = 'sourced-properties'
  213 
  214     def validate(self, name, object):
  215         if not isinstance(object, dict):
  216             yield "{} is not sourced properties (not a dict)".format(name)
  217             return
  218         for k, v in object.items():
  219             if not isinstance(k, str):
  220                 yield "{} property name {!r} is not unicode".format(name, k)
  221             if not isinstance(v, tuple) or len(v) != 2:
  222                 yield "{} property value for '{}' is not a 2-tuple".format(name, k)
  223                 return
  224             propval, propsrc = v
  225             if not isinstance(propsrc, str):
  226                 yield "{}[{}] source {!r} is not unicode".format(name, k, propsrc)
  227             try:
  228                 json.dumps(propval)
  229             except (TypeError, ValueError):
  230                 yield "{}[{!r}] value is not JSON-able".format(name, k)
  231 
  232 
  233 class JsonValidator(Validator):
  234 
  235     name = 'json'
  236 
  237     def validate(self, name, object):
  238         try:
  239             json.dumps(object)
  240         except (TypeError, ValueError):
  241             yield "{}[{!r}] value is not JSON-able".format(name, object)
  242 
  243 
  244 class PatchValidator(Validator):
  245 
  246     name = 'patch'
  247 
  248     validator = DictValidator(
  249         body=NoneOk(BinaryValidator()),
  250         level=NoneOk(IntValidator()),
  251         subdir=NoneOk(StringValidator()),
  252         author=NoneOk(StringValidator()),
  253         comment=NoneOk(StringValidator()),
  254     )
  255 
  256     def validate(self, name, object):
  257         for msg in self.validator.validate(name, object):
  258             yield msg
  259 
  260 
  261 class MessageValidator(Validator):
  262 
  263     routingKeyValidator = TupleValidator(StrValidator())
  264 
  265     def __init__(self, events, messageValidator):
  266         self.events = [bytes2unicode(e) for e in set(events)]
  267         self.messageValidator = messageValidator
  268 
  269     def validate(self, name, routingKey_message):
  270         try:
  271             routingKey, message = routingKey_message
  272         except (TypeError, ValueError) as e:
  273             yield "{!r}: not a routing key and message: {}".format(routingKey_message, e)
  274         routingKeyBad = False
  275         for msg in self.routingKeyValidator.validate("routingKey", routingKey):
  276             yield msg
  277             routingKeyBad = True
  278 
  279         if not routingKeyBad:
  280             event = routingKey[-1]
  281             if event not in self.events:
  282                 yield "routing key event {!r} is not valid".format(event)
  283 
  284         for msg in self.messageValidator.validate("{} message".format(routingKey[0]),
  285                                                   message):
  286             yield msg
  287 
  288 
  289 class Selector(Validator):
  290 
  291     def __init__(self):
  292         self.selectors = []
  293 
  294     def add(self, selector, validator):
  295         self.selectors.append((selector, validator))
  296 
  297     def validate(self, name, arg_object):
  298         try:
  299             arg, object = arg_object
  300         except (TypeError, ValueError) as e:
  301             yield "{!r}: not a not data options and data dict: {}".format(arg_object, e)
  302         for selector, validator in self.selectors:
  303             if selector is None or selector(arg):
  304                 for msg in validator.validate(name, object):
  305                     yield msg
  306                 return
  307         yield "no match for selector argument {!r}".format(arg)
  308 
  309 
  310 # Type definitions
  311 
  312 message = {}
  313 dbdict = {}
  314 
  315 # parse and use a ResourceType class's dataFields into a validator
  316 
  317 # masters
  318 
  319 message['masters'] = Selector()
  320 message['masters'].add(None,
  321                        MessageValidator(
  322                            events=[b'started', b'stopped'],
  323                            messageValidator=DictValidator(
  324                                masterid=IntValidator(),
  325                                name=StringValidator(),
  326                                active=BooleanValidator(),
  327                                # last_active is not included
  328                            )))
  329 
  330 dbdict['masterdict'] = DictValidator(
  331     id=IntValidator(),
  332     name=StringValidator(),
  333     active=BooleanValidator(),
  334     last_active=DateTimeValidator(),
  335 )
  336 
  337 # sourcestamp
  338 
  339 _sourcestamp = dict(
  340     ssid=IntValidator(),
  341     branch=NoneOk(StringValidator()),
  342     revision=NoneOk(StringValidator()),
  343     repository=StringValidator(),
  344     project=StringValidator(),
  345     codebase=StringValidator(),
  346     created_at=DateTimeValidator(),
  347     patch=NoneOk(DictValidator(
  348         body=NoneOk(BinaryValidator()),
  349         level=NoneOk(IntValidator()),
  350         subdir=NoneOk(StringValidator()),
  351         author=NoneOk(StringValidator()),
  352         comment=NoneOk(StringValidator()))),
  353 )
  354 
  355 message['sourcestamps'] = Selector()
  356 message['sourcestamps'].add(None,
  357                             DictValidator(
  358                                 **_sourcestamp
  359                             ))
  360 
  361 dbdict['ssdict'] = DictValidator(
  362     ssid=IntValidator(),
  363     branch=NoneOk(StringValidator()),
  364     revision=NoneOk(StringValidator()),
  365     patchid=NoneOk(IntValidator()),
  366     patch_body=NoneOk(BinaryValidator()),
  367     patch_level=NoneOk(IntValidator()),
  368     patch_subdir=NoneOk(StringValidator()),
  369     patch_author=NoneOk(StringValidator()),
  370     patch_comment=NoneOk(StringValidator()),
  371     codebase=StringValidator(),
  372     repository=StringValidator(),
  373     project=StringValidator(),
  374     created_at=DateTimeValidator(),
  375 )
  376 
  377 # builder
  378 
  379 message['builders'] = Selector()
  380 message['builders'].add(None,
  381                         MessageValidator(
  382                             events=[b'started', b'stopped'],
  383                             messageValidator=DictValidator(
  384                                 builderid=IntValidator(),
  385                                 masterid=IntValidator(),
  386                                 name=StringValidator(),
  387                             )))
  388 
  389 dbdict['builderdict'] = DictValidator(
  390     id=IntValidator(),
  391     masterids=ListValidator(IntValidator()),
  392     name=StringValidator(),
  393     description=NoneOk(StringValidator()),
  394     tags=ListValidator(StringValidator()),
  395 )
  396 
  397 # worker
  398 
  399 dbdict['workerdict'] = DictValidator(
  400     id=IntValidator(),
  401     name=StringValidator(),
  402     configured_on=ListValidator(
  403         DictValidator(
  404             masterid=IntValidator(),
  405             builderid=IntValidator(),
  406         )
  407     ),
  408     paused=BooleanValidator(),
  409     graceful=BooleanValidator(),
  410     connected_to=ListValidator(IntValidator()),
  411     workerinfo=JsonValidator(),
  412 )
  413 
  414 # buildset
  415 
  416 _buildset = dict(
  417     bsid=IntValidator(),
  418     external_idstring=NoneOk(StringValidator()),
  419     reason=StringValidator(),
  420     submitted_at=IntValidator(),
  421     complete=BooleanValidator(),
  422     complete_at=NoneOk(IntValidator()),
  423     results=NoneOk(IntValidator()),
  424     parent_buildid=NoneOk(IntValidator()),
  425     parent_relationship=NoneOk(StringValidator()),
  426 )
  427 _buildsetEvents = [b'new', b'complete']
  428 
  429 message['buildsets'] = Selector()
  430 message['buildsets'].add(lambda k: k[-1] == 'new',
  431                          MessageValidator(
  432                              events=_buildsetEvents,
  433                              messageValidator=DictValidator(
  434                                  scheduler=StringValidator(),  # only for 'new'
  435                                  sourcestamps=ListValidator(
  436                                      DictValidator(
  437                                          **_sourcestamp
  438                                      )),
  439                                  **_buildset
  440                              )))
  441 message['buildsets'].add(None,
  442                          MessageValidator(
  443                              events=_buildsetEvents,
  444                              messageValidator=DictValidator(
  445                                  sourcestamps=ListValidator(
  446                                      DictValidator(
  447                                          **_sourcestamp
  448                                      )),
  449                                  **_buildset
  450                              )))
  451 
  452 dbdict['bsdict'] = DictValidator(
  453     bsid=IntValidator(),
  454     external_idstring=NoneOk(StringValidator()),
  455     reason=StringValidator(),
  456     sourcestamps=ListValidator(IntValidator()),
  457     submitted_at=DateTimeValidator(),
  458     complete=BooleanValidator(),
  459     complete_at=NoneOk(DateTimeValidator()),
  460     results=NoneOk(IntValidator()),
  461     parent_buildid=NoneOk(IntValidator()),
  462     parent_relationship=NoneOk(StringValidator()),
  463 )
  464 
  465 # buildrequest
  466 
  467 message['buildrequests'] = Selector()
  468 message['buildrequests'].add(None,
  469                              MessageValidator(
  470                                  events=[b'new', b'claimed', b'unclaimed'],
  471                                  messageValidator=DictValidator(
  472                                      # TODO: probably wrong!
  473                                      brid=IntValidator(),
  474                                      builderid=IntValidator(),
  475                                      bsid=IntValidator(),
  476                                      buildername=StringValidator(),
  477                                  )))
  478 
  479 # change
  480 
  481 message['changes'] = Selector()
  482 message['changes'].add(None,
  483                        MessageValidator(
  484                            events=[b'new'],
  485                            messageValidator=DictValidator(
  486                                changeid=IntValidator(),
  487                                parent_changeids=ListValidator(IntValidator()),
  488                                author=StringValidator(),
  489                                files=ListValidator(StringValidator()),
  490                                comments=StringValidator(),
  491                                revision=NoneOk(StringValidator()),
  492                                when_timestamp=IntValidator(),
  493                                branch=NoneOk(StringValidator()),
  494                                category=NoneOk(StringValidator()),
  495                                revlink=NoneOk(StringValidator()),
  496                                properties=SourcedPropertiesValidator(),
  497                                repository=StringValidator(),
  498                                project=StringValidator(),
  499                                codebase=StringValidator(),
  500                                sourcestamp=DictValidator(
  501                                    **_sourcestamp
  502                                ),
  503                            )))
  504 
  505 dbdict['chdict'] = DictValidator(
  506     changeid=IntValidator(),
  507     author=StringValidator(),
  508     files=ListValidator(StringValidator()),
  509     comments=StringValidator(),
  510     revision=NoneOk(StringValidator()),
  511     when_timestamp=DateTimeValidator(),
  512     branch=NoneOk(StringValidator()),
  513     category=NoneOk(StringValidator()),
  514     revlink=NoneOk(StringValidator()),
  515     properties=SourcedPropertiesValidator(),
  516     repository=StringValidator(),
  517     project=StringValidator(),
  518     codebase=StringValidator(),
  519     sourcestampid=IntValidator(),
  520     parent_changeids=ListValidator(IntValidator()),
  521 )
  522 
  523 # changesources
  524 
  525 dbdict['changesourcedict'] = DictValidator(
  526     id=IntValidator(),
  527     name=StringValidator(),
  528     masterid=NoneOk(IntValidator()),
  529 )
  530 
  531 # schedulers
  532 
  533 dbdict['schedulerdict'] = DictValidator(
  534     id=IntValidator(),
  535     name=StringValidator(),
  536     masterid=NoneOk(IntValidator()),
  537     enabled=BooleanValidator(),
  538 )
  539 
  540 # builds
  541 
  542 _build = dict(
  543     buildid=IntValidator(),
  544     number=IntValidator(),
  545     builderid=IntValidator(),
  546     buildrequestid=IntValidator(),
  547     workerid=IntValidator(),
  548     masterid=IntValidator(),
  549     started_at=IntValidator(),
  550     complete=BooleanValidator(),
  551     complete_at=NoneOk(IntValidator()),
  552     state_string=StringValidator(),
  553     results=NoneOk(IntValidator()),
  554 )
  555 _buildEvents = [b'new', b'complete']
  556 
  557 message['builds'] = Selector()
  558 message['builds'].add(None,
  559                       MessageValidator(
  560                           events=_buildEvents,
  561                           messageValidator=DictValidator(
  562                               **_build
  563                           )))
  564 
  565 # As build's properties are fetched at DATA API level,
  566 # a distinction shall be made as both are not equal.
  567 # Validates DB layer
  568 dbdict['dbbuilddict'] = buildbase = DictValidator(
  569     id=IntValidator(),
  570     number=IntValidator(),
  571     builderid=IntValidator(),
  572     buildrequestid=IntValidator(),
  573     workerid=IntValidator(),
  574     masterid=IntValidator(),
  575     started_at=DateTimeValidator(),
  576     complete_at=NoneOk(DateTimeValidator()),
  577     state_string=StringValidator(),
  578     results=NoneOk(IntValidator()),
  579 )
  580 
  581 # Validates DATA API layer
  582 dbdict['builddict'] = DictValidator(
  583     properties=NoneOk(SourcedPropertiesValidator()), **buildbase.keys)
  584 
  585 # steps
  586 
  587 _step = dict(
  588     stepid=IntValidator(),
  589     number=IntValidator(),
  590     name=IdentifierValidator(50),
  591     buildid=IntValidator(),
  592     started_at=IntValidator(),
  593     complete=BooleanValidator(),
  594     complete_at=NoneOk(IntValidator()),
  595     state_string=StringValidator(),
  596     results=NoneOk(IntValidator()),
  597     urls=ListValidator(StringValidator()),
  598     hidden=BooleanValidator(),
  599 )
  600 _stepEvents = [b'new', b'complete']
  601 
  602 message['steps'] = Selector()
  603 message['steps'].add(None,
  604                      MessageValidator(
  605                          events=_stepEvents,
  606                          messageValidator=DictValidator(
  607                              **_step
  608                          )))
  609 
  610 dbdict['stepdict'] = DictValidator(
  611     id=IntValidator(),
  612     number=IntValidator(),
  613     name=IdentifierValidator(50),
  614     buildid=IntValidator(),
  615     started_at=DateTimeValidator(),
  616     complete_at=NoneOk(DateTimeValidator()),
  617     state_string=StringValidator(),
  618     results=NoneOk(IntValidator()),
  619     urls=ListValidator(StringValidator()),
  620     hidden=BooleanValidator(),
  621 )
  622 
  623 # logs
  624 
  625 _log = dict(
  626     logid=IntValidator(),
  627     name=IdentifierValidator(50),
  628     stepid=IntValidator(),
  629     complete=BooleanValidator(),
  630     num_lines=IntValidator(),
  631     type=IdentifierValidator(1))
  632 _logEvents = ['new', 'complete', 'appended']
  633 
  634 # message['log']
  635 
  636 dbdict['logdict'] = DictValidator(
  637     id=IntValidator(),
  638     stepid=IntValidator(),
  639     name=StringValidator(),
  640     slug=IdentifierValidator(50),
  641     complete=BooleanValidator(),
  642     num_lines=IntValidator(),
  643     type=IdentifierValidator(1))
  644 
  645 
  646 # external functions
  647 
  648 def _verify(testcase, validator, name, object):
  649     msgs = list(validator.validate(name, object))
  650     if msgs:
  651         msg = "; ".join(msgs)
  652         if testcase:
  653             testcase.fail(msg)
  654         else:
  655             raise AssertionError(msg)
  656 
  657 
  658 def verifyMessage(testcase, routingKey, message_):
  659     # the validator is a Selector wrapping a MessageValidator, so we need to
  660     # pass (arg, (routingKey, message)), where the routing key is the arg
  661     # the "type" of the message is identified by last path name
  662     # -1 being the event, and -2 the id.
  663 
  664     validator = message[bytes2unicode(routingKey[-3])]
  665     _verify(testcase, validator, '',
  666             (routingKey, (routingKey, message_)))
  667 
  668 
  669 def verifyDbDict(testcase, type, value):
  670     _verify(testcase, dbdict[type], type, value)
  671 
  672 
  673 def verifyData(testcase, entityType, options, value):
  674     _verify(testcase, entityType, entityType.name, value)
  675 
  676 
  677 def verifyType(testcase, name, value, validator):
  678     _verify(testcase, validator, name, value)