"Fossies" - the Fresh Open Source Software Archive

Member "cheetah3-3.2.6.post2/Cheetah/Parser.py" (20 Apr 2021, 105269 Bytes) of package /linux/www/cheetah3-3.2.6.post2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "Parser.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 3-3.2.1_vs_3-3.2.2.

    1 """
    2 Parser classes for Cheetah's Compiler
    3 
    4 Classes:
    5   ParseError( Exception )
    6   _LowLevelParser( Cheetah.SourceReader.SourceReader ), basically a lexer
    7   _HighLevelParser( _LowLevelParser )
    8   Parser === _HighLevelParser (an alias)
    9 """
   10 
   11 import sys
   12 import re
   13 import types
   14 import inspect
   15 
   16 from Cheetah.SourceReader import SourceReader
   17 from Cheetah.Unspecified import Unspecified
   18 from Cheetah.Macros.I18n import I18n
   19 from Cheetah.compat import PY2, string_type, unicode
   20 if PY2:
   21     from tokenize import pseudoprog
   22 else:
   23     from tokenize import _compile, PseudoToken
   24     pseudoprog = _compile(PseudoToken)
   25     del _compile, PseudoToken
   26 
   27 # re tools
   28 _regexCache = {}
   29 
   30 
   31 def cachedRegex(pattern):
   32     if pattern not in _regexCache:
   33         _regexCache[pattern] = re.compile(pattern)
   34     return _regexCache[pattern]
   35 
   36 
   37 def escapeRegexChars(txt,
   38                      escapeRE=re.compile(r'([\$\^\*\+\.\?\{\}\[\]\(\)\|\\])')):
   39 
   40     """Return a txt with all special regular expressions chars escaped."""
   41 
   42     return escapeRE.sub(r'\\\1', txt)
   43 
   44 
   45 def group(*choices):
   46     return '(' + '|'.join(choices) + ')'
   47 
   48 
   49 def nongroup(*choices):
   50     return '(?:' + '|'.join(choices) + ')'
   51 
   52 
   53 def namedGroup(name, *choices):
   54     return '(P:<' + name + '>' + '|'.join(choices) + ')'
   55 
   56 
   57 def any(*choices):
   58     return group(*choices) + '*'
   59 
   60 
   61 def maybe(*choices):
   62     return group(*choices) + '?'
   63 
   64 ##################################################
   65 # CONSTANTS & GLOBALS ##
   66 
   67 
   68 NO_CACHE = 0
   69 STATIC_CACHE = 1
   70 REFRESH_CACHE = 2
   71 
   72 SET_LOCAL = 0
   73 SET_GLOBAL = 1
   74 SET_MODULE = 2
   75 
   76 ##################################################
   77 # Tokens for the parser ##
   78 
   79 # generic
   80 identchars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"
   81 namechars = identchars + "0123456789"
   82 
   83 # operators
   84 powerOp = '**'
   85 unaryArithOps = ('+', '-', '~')
   86 binaryArithOps = ('+', '-', '/', '//', '%')
   87 shiftOps = ('>>', '<<')
   88 bitwiseOps = ('&', '|', '^')
   89 assignOp = '='
   90 augAssignOps = ('+=', '-=', '/=', '*=', '**=', '^=', '%=',
   91                 '>>=', '<<=', '&=', '|=', )
   92 assignmentOps = (assignOp,) + augAssignOps
   93 
   94 compOps = ('<', '>', '==', '!=', '<=', '>=', '<>', 'is', 'in',)
   95 booleanOps = ('and', 'or', 'not')
   96 operators = (powerOp,) + unaryArithOps + binaryArithOps \
   97     + shiftOps + bitwiseOps + assignmentOps \
   98     + compOps + booleanOps
   99 
  100 delimeters = ('(', ')', '{', '}', '[', ']',
  101               ',', '.', ':', ';', '=', '`') + augAssignOps
  102 
  103 
  104 keywords = ('and',       'del',       'for',       'is',        'raise',   # noqa: E241,E501 multiple spaces after ','
  105             'assert',    'elif',      'from',      'lambda',    'return',  # noqa: E241,E501 multiple spaces after ','
  106             'break',     'else',      'global',    'not',       'try',     # noqa: E241,E501 multiple spaces after ','
  107             'class',     'except',    'if',        'or',        'while',   # noqa: E241,E501 multiple spaces after ','
  108             'continue',  'exec',      'import',    'pass',                 # noqa: E241,E501 multiple spaces after ','
  109             'def',       'finally',   'in',        'print',                # noqa: E241,E501 multiple spaces after ','
  110             )
  111 
  112 single3 = "'''"
  113 double3 = '"""'
  114 
  115 tripleQuotedStringStarts = ("'''", '"""',
  116                             "r'''", 'r"""', "R'''", 'R"""',
  117                             "u'''", 'u"""', "U'''", 'U"""',
  118                             "ur'''", 'ur"""', "Ur'''", 'Ur"""',
  119                             "uR'''", 'uR"""', "UR'''", 'UR"""')
  120 
  121 tripleQuotedStringPairs = {"'''": single3, '"""': double3,
  122                            "r'''": single3, 'r"""': double3,
  123                            "u'''": single3, 'u"""': double3,
  124                            "ur'''": single3, 'ur"""': double3,
  125                            "R'''": single3, 'R"""': double3,
  126                            "U'''": single3, 'U"""': double3,
  127                            "uR'''": single3, 'uR"""': double3,
  128                            "Ur'''": single3, 'Ur"""': double3,
  129                            "UR'''": single3, 'UR"""': double3,
  130                            }
  131 
  132 closurePairs = {')': '(', ']': '[', '}': '{'}
  133 closurePairsRev = {'(': ')', '[': ']', '{': '}'}
  134 
  135 ##################################################
  136 # Regex chunks for the parser ##
  137 
  138 tripleQuotedStringREs = {}
  139 
  140 
  141 def makeTripleQuoteRe(start, end):
  142     start = escapeRegexChars(start)
  143     end = escapeRegexChars(end)
  144     return re.compile(r'(?:' + start + r').*?' + r'(?:' + end + r')',
  145                       re.DOTALL)
  146 
  147 
  148 for start, end in tripleQuotedStringPairs.items():
  149     tripleQuotedStringREs[start] = makeTripleQuoteRe(start, end)
  150 
  151 WS = r'[ \f\t]*'
  152 EOL = r'\r\n|\n|\r'
  153 EOLZ = EOL + r'|\Z'
  154 escCharLookBehind = nongroup(r'(?<=\A)', r'(?<!\\)')
  155 nameCharLookAhead = r'(?=[A-Za-z_])'
  156 identRE = re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*')
  157 EOLre = re.compile(r'(?:\r\n|\r|\n)')
  158 
  159 specialVarRE = re.compile(r'([a-zA-z_]+)@')  # for matching specialVar comments
  160 # e.g. ##author@ Tavis Rudd
  161 
  162 unicodeDirectiveRE = re.compile(
  163     r'(?:^|\r\n|\r|\n)\s*#\s{0,5}unicode[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)',
  164     re.MULTILINE)
  165 encodingDirectiveRE = re.compile(
  166     r'(?:^|\r\n|\r|\n)\s*#\s{0,5}encoding[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)',
  167     re.MULTILINE)
  168 
  169 escapedNewlineRE = re.compile(r'(?<!\\)((\\\\)*)\\(n|012)')
  170 
  171 directiveNamesAndParsers = {
  172     # importing and inheritance
  173     'import': None,
  174     'from': None,
  175     'extends': 'eatExtends',
  176     'implements': 'eatImplements',
  177     'super': 'eatSuper',
  178 
  179     # output, filtering, and caching
  180     'slurp': 'eatSlurp',
  181     'raw': 'eatRaw',
  182     'include': 'eatInclude',
  183     'cache': 'eatCache',
  184     'filter': 'eatFilter',
  185     'echo': None,
  186     'silent': None,
  187     'transform': 'eatTransform',
  188 
  189     'call': 'eatCall',
  190     'arg': 'eatCallArg',
  191 
  192     'capture': 'eatCapture',
  193 
  194     # declaration, assignment, and deletion
  195     'attr': 'eatAttr',
  196     'def': 'eatDef',
  197     'block': 'eatBlock',
  198     '@': 'eatDecorator',
  199     'defmacro': 'eatDefMacro',
  200 
  201     'closure': 'eatClosure',
  202 
  203     'set': 'eatSet',
  204     'del': None,
  205 
  206     # flow control
  207     'if': 'eatIf',
  208     'while': None,
  209     'for': None,
  210     'else': None,
  211     'elif': None,
  212     'pass': None,
  213     'break': None,
  214     'continue': None,
  215     'stop': None,
  216     'return': None,
  217     'yield': None,
  218 
  219     # little wrappers
  220     'repeat': None,
  221     'unless': None,
  222 
  223     # error handling
  224     'assert': None,
  225     'raise': None,
  226     'try': None,
  227     'except': None,
  228     'finally': None,
  229     'errorCatcher': 'eatErrorCatcher',
  230 
  231     # intructions to the parser and compiler
  232     'breakpoint': 'eatBreakPoint',
  233     'compiler': 'eatCompiler',
  234     'compiler-settings': 'eatCompilerSettings',
  235 
  236     # misc
  237     'shBang': 'eatShbang',
  238     'encoding': 'eatEncoding',
  239 
  240     'end': 'eatEndDirective',
  241 }
  242 
  243 endDirectiveNamesAndHandlers = {
  244     'def': 'handleEndDef',      # has short-form
  245     'block': None,              # has short-form
  246     'closure': None,            # has short-form
  247     'cache': None,              # has short-form
  248     'call': None,               # has short-form
  249     'capture': None,            # has short-form
  250     'filter': None,
  251     'errorCatcher': None,
  252     'while': None,              # has short-form
  253     'for': None,                # has short-form
  254     'if': None,                 # has short-form
  255     'try': None,                # has short-form
  256     'repeat': None,             # has short-form
  257     'unless': None,             # has short-form
  258 }
  259 
  260 ##################################################
  261 # CLASSES ##
  262 
  263 # @@TR: SyntaxError doesn't call exception.__str__ for some reason!
  264 # class ParseError(SyntaxError):
  265 
  266 
  267 class ParseError(ValueError):
  268     def __init__(self, stream, msg='Invalid Syntax', extMsg='',
  269                  lineno=None, col=None):
  270         self.stream = stream
  271         if stream.pos() >= len(stream):
  272             stream.setPos(len(stream) - 1)
  273         self.msg = msg
  274         self.extMsg = extMsg
  275         self.lineno = lineno
  276         self.col = col
  277 
  278     def __str__(self):
  279         return self.report()
  280 
  281     def report(self):
  282         stream = self.stream
  283         if stream.filename():
  284             f = " in file %s" % stream.filename()
  285         else:
  286             f = ''
  287         report = ''
  288         if self.lineno:
  289             lineno = self.lineno
  290             row, col, line = (lineno, (self.col or 0),
  291                               self.stream.splitlines()[lineno - 1])
  292         else:
  293             row, col, line = self.stream.getRowColLine()
  294 
  295         # get the surrounding lines
  296         lines = stream.splitlines()
  297         prevLines = []                  # (rowNum, content)
  298         for i in range(1, 4):
  299             if row - 1 - i <= 0:
  300                 break
  301             prevLines.append((row - i, lines[row - 1 - i]))
  302 
  303         nextLines = []                  # (rowNum, content)
  304         for i in range(1, 4):
  305             if not row - 1 + i < len(lines):
  306                 break
  307             nextLines.append((row + i, lines[row - 1 + i]))
  308         nextLines.reverse()
  309 
  310         # print the main message
  311         report += "\n\n%s\n" % self.msg
  312         report += "Line %i, column %i%s\n\n" % (row, col, f)
  313         report += 'Line|Cheetah Code\n'
  314         report += '----|-----------------------------' \
  315             '--------------------------------\n'
  316         while prevLines:
  317             lineInfo = prevLines.pop()
  318             report += "%(row)-4d|%(line)s\n" \
  319                 % {'row': lineInfo[0], 'line': lineInfo[1]}
  320         report += "%(row)-4d|%(line)s\n" % {'row': row, 'line': line}
  321         report += ' '*5 + ' '*(col - 1) + "^\n"  # noqa: E226,E501 missing whitespace around operator
  322 
  323         while nextLines:
  324             lineInfo = nextLines.pop()
  325             report += "%(row)-4d|%(line)s\n" \
  326                 % {'row': lineInfo[0], 'line': lineInfo[1]}
  327         # add the extra msg
  328         if self.extMsg:
  329             report += self.extMsg + '\n'
  330 
  331         return report
  332 
  333 
  334 class ForbiddenSyntax(ParseError):
  335     pass
  336 
  337 
  338 class ForbiddenExpression(ForbiddenSyntax):
  339     pass
  340 
  341 
  342 class ForbiddenDirective(ForbiddenSyntax):
  343     pass
  344 
  345 
  346 class CheetahVariable(object):
  347     def __init__(self, nameChunks, useNameMapper=True, cacheToken=None,
  348                  rawSource=None):
  349         self.nameChunks = nameChunks
  350         self.useNameMapper = useNameMapper
  351         self.cacheToken = cacheToken
  352         self.rawSource = rawSource
  353 
  354 
  355 class Placeholder(CheetahVariable):
  356     pass
  357 
  358 
  359 class ArgList(object):
  360     """Used by _LowLevelParser.getArgList()"""
  361 
  362     def __init__(self):
  363         self.arguments = []
  364         self.defaults = []
  365         self.count = 0
  366 
  367     def add_argument(self, name):
  368         self.arguments.append(name)
  369         self.defaults.append(None)
  370 
  371     def next(self):
  372         self.count += 1
  373 
  374     def add_default(self, token):
  375         count = self.count
  376         if self.defaults[count] is None:
  377             self.defaults[count] = ''
  378         self.defaults[count] += token
  379 
  380     def merge(self):
  381         defaults = (isinstance(d, string_type) and d.strip() or None
  382                     for d in self.defaults)
  383         arguments = (a.strip() for a in self.arguments)
  384         if PY2:
  385             return list(map(None, arguments, defaults))
  386         else:
  387             from itertools import zip_longest
  388             return list(zip_longest(arguments, defaults))
  389 
  390     def __str__(self):
  391         return str(self.merge())
  392 
  393 
  394 class _LowLevelParser(SourceReader):
  395     """This class implements the methods to match or extract ('get*') the basic
  396     elements of Cheetah's grammar.  It does NOT handle any code generation or
  397     state management.
  398     """
  399 
  400     _settingsManager = None
  401 
  402     def setSettingsManager(self, settingsManager):
  403         self._settingsManager = settingsManager
  404 
  405     def setting(self, key, default=Unspecified):
  406         if default is Unspecified:
  407             return self._settingsManager.setting(key)
  408         else:
  409             return self._settingsManager.setting(key, default=default)
  410 
  411     def setSetting(self, key, val):
  412         self._settingsManager.setSetting(key, val)
  413 
  414     def settings(self):
  415         return self._settingsManager.settings()
  416 
  417     def updateSettings(self, settings):
  418         self._settingsManager.updateSettings(settings)
  419 
  420     def _initializeSettings(self):
  421         self._settingsManager._initializeSettings()
  422 
  423     def configureParser(self):
  424         """Is called by the Compiler instance after the parser has had a
  425         settingsManager assigned with self.setSettingsManager()
  426         """
  427         self._makeCheetahVarREs()
  428         self._makeCommentREs()
  429         self._makeDirectiveREs()
  430         self._makePspREs()
  431         self._possibleNonStrConstantChars = (
  432             self.setting('commentStartToken')[0]
  433             + self.setting('multiLineCommentStartToken')[0]
  434             + self.setting('cheetahVarStartToken')[0]
  435             + self.setting('directiveStartToken')[0]
  436             + self.setting('PSPStartToken')[0])
  437         self._nonStrConstMatchers = [
  438             self.matchCommentStartToken,
  439             self.matchMultiLineCommentStartToken,
  440             self.matchVariablePlaceholderStart,
  441             self.matchExpressionPlaceholderStart,
  442             self.matchDirective,
  443             self.matchPSPStartToken,
  444             self.matchEOLSlurpToken,
  445         ]
  446 
  447     # regex setup ##
  448 
  449     def _makeCheetahVarREs(self):
  450 
  451         """Setup the regexs for Cheetah $var parsing."""
  452 
  453         num = r'[0-9\.]+'
  454         interval = (r'(?P<interval>'
  455                     + num + r's|'
  456                     + num + r'm|'
  457                     + num + r'h|'
  458                     + num + r'd|'
  459                     + num + r'w|'
  460                     + num + ')'
  461                     )
  462 
  463         cacheToken = (r'(?:'
  464                       + r'(?P<REFRESH_CACHE>\*' + interval + '\*)'
  465                       + '|'
  466                       + r'(?P<STATIC_CACHE>\*)'
  467                       + '|'
  468                       + r'(?P<NO_CACHE>)'
  469                       + ')')
  470         self.cacheTokenRE = cachedRegex(cacheToken)
  471 
  472         silentPlaceholderToken = (r'(?:'
  473                                   + r'(?P<SILENT>'
  474                                   + escapeRegexChars('!') + ')'
  475                                   + '|'
  476                                   + r'(?P<NOT_SILENT>)'
  477                                   + ')')
  478         self.silentPlaceholderTokenRE = cachedRegex(silentPlaceholderToken)
  479 
  480         self.cheetahVarStartRE = cachedRegex(
  481             escCharLookBehind + r'(?P<startToken>'
  482             + escapeRegexChars(self.setting('cheetahVarStartToken')) + ')'
  483             + r'(?P<silenceToken>' + silentPlaceholderToken + ')'
  484             + r'(?P<cacheToken>' + cacheToken + ')'
  485             # allow WS after enclosure
  486             + r'(?P<enclosure>|(?:(?:\{|\(|\[)[ \t\f]*))' + r'(?=[A-Za-z_])')
  487         validCharsLookAhead = r'(?=[A-Za-z_\*!\{\(\[])'
  488         self.cheetahVarStartToken = self.setting('cheetahVarStartToken')
  489         self.cheetahVarStartTokenRE = cachedRegex(
  490             escCharLookBehind
  491             + escapeRegexChars(self.setting('cheetahVarStartToken'))
  492             + validCharsLookAhead
  493         )
  494 
  495         self.cheetahVarInExpressionStartTokenRE = cachedRegex(
  496             escapeRegexChars(self.setting('cheetahVarStartToken'))
  497             + r'(?=[A-Za-z_])'
  498         )
  499 
  500         self.expressionPlaceholderStartRE = cachedRegex(
  501             escCharLookBehind + r'(?P<startToken>'
  502             + escapeRegexChars(self.setting('cheetahVarStartToken'))
  503             + ')'
  504             + r'(?P<cacheToken>' + cacheToken + ')'
  505             # r'\[[ \t\f]*'
  506             + r'(?:\{|\(|\[)[ \t\f]*' + r'(?=[^\)\}\]])')
  507 
  508         if self.setting('EOLSlurpToken'):
  509             self.EOLSlurpRE = cachedRegex(
  510                 escapeRegexChars(self.setting('EOLSlurpToken'))
  511                 + r'[ \t\f]*'
  512                 + r'(?:' + EOL + ')'
  513             )
  514         else:
  515             self.EOLSlurpRE = None
  516 
  517     def _makeCommentREs(self):
  518         """Construct the regex bits that are used in comment parsing."""
  519         startTokenEsc = escapeRegexChars(self.setting('commentStartToken'))
  520         self.commentStartTokenRE = cachedRegex(
  521             escCharLookBehind + startTokenEsc)
  522         del startTokenEsc
  523 
  524         startTokenEsc = escapeRegexChars(
  525             self.setting('multiLineCommentStartToken'))
  526         endTokenEsc = escapeRegexChars(
  527             self.setting('multiLineCommentEndToken'))
  528         self.multiLineCommentTokenStartRE = cachedRegex(
  529             escCharLookBehind + startTokenEsc)
  530         self.multiLineCommentEndTokenRE = cachedRegex(
  531             escCharLookBehind + endTokenEsc)
  532 
  533     def _makeDirectiveREs(self):
  534         """Construct the regexs that are used in directive parsing."""
  535         startToken = self.setting('directiveStartToken')
  536         endToken = self.setting('directiveEndToken')
  537         startTokenEsc = escapeRegexChars(startToken)
  538         endTokenEsc = escapeRegexChars(endToken)
  539         validSecondCharsLookAhead = r'(?=[A-Za-z_@])'
  540         reParts = [escCharLookBehind, startTokenEsc]
  541         if self.setting('allowWhitespaceAfterDirectiveStartToken'):
  542             reParts.append('[ \t]*')
  543         reParts.append(validSecondCharsLookAhead)
  544         self.directiveStartTokenRE = cachedRegex(''.join(reParts))
  545         self.directiveEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc)
  546 
  547     def _makePspREs(self):
  548         """Setup the regexs for PSP parsing."""
  549         startToken = self.setting('PSPStartToken')
  550         startTokenEsc = escapeRegexChars(startToken)
  551         self.PSPStartTokenRE = cachedRegex(escCharLookBehind + startTokenEsc)
  552         endToken = self.setting('PSPEndToken')
  553         endTokenEsc = escapeRegexChars(endToken)
  554         self.PSPEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc)
  555 
  556     def _unescapeCheetahVars(self, theString):
  557         """Unescape any escaped Cheetah \$vars in the string.
  558         """
  559 
  560         token = self.setting('cheetahVarStartToken')
  561         return theString.replace('\\' + token, token)
  562 
  563     def _unescapeDirectives(self, theString):
  564         """Unescape any escaped Cheetah directives in the string.
  565         """
  566 
  567         token = self.setting('directiveStartToken')
  568         return theString.replace('\\' + token, token)
  569 
  570     def isLineClearToStartToken(self, pos=None):
  571         return self.isLineClearToPos(pos)
  572 
  573     def matchTopLevelToken(self):
  574         """Returns the first match found from the following methods:
  575             self.matchCommentStartToken
  576             self.matchMultiLineCommentStartToken
  577             self.matchVariablePlaceholderStart
  578             self.matchExpressionPlaceholderStart
  579             self.matchDirective
  580             self.matchPSPStartToken
  581             self.matchEOLSlurpToken
  582 
  583         Returns None if no match.
  584         """
  585         match = None
  586         if self.peek() in self._possibleNonStrConstantChars:
  587             for matcher in self._nonStrConstMatchers:
  588                 match = matcher()
  589                 if match:
  590                     break
  591         return match
  592 
  593     def matchPyToken(self):
  594         match = pseudoprog.match(self.src(), self.pos())
  595 
  596         if match and match.group() in tripleQuotedStringStarts:
  597             TQSmatch = tripleQuotedStringREs[match.group()].match(self.src(),
  598                                                                   self.pos())
  599             if TQSmatch:
  600                 return TQSmatch
  601         return match
  602 
  603     def getPyToken(self):
  604         match = self.matchPyToken()
  605         if match is None:
  606             raise ParseError(self)
  607         elif match.group() in tripleQuotedStringStarts:
  608             raise ParseError(self, msg='Malformed triple-quoted string')
  609         return self.readTo(match.end())
  610 
  611     def matchEOLSlurpToken(self):
  612         if self.EOLSlurpRE:
  613             return self.EOLSlurpRE.match(self.src(), self.pos())
  614 
  615     def getEOLSlurpToken(self):
  616         match = self.matchEOLSlurpToken()
  617         if not match:
  618             raise ParseError(self, msg='Invalid EOL slurp token')
  619         return self.readTo(match.end())
  620 
  621     def matchCommentStartToken(self):
  622         return self.commentStartTokenRE.match(self.src(), self.pos())
  623 
  624     def getCommentStartToken(self):
  625         match = self.matchCommentStartToken()
  626         if not match:
  627             raise ParseError(self,
  628                              msg='Invalid single-line comment start token')
  629         return self.readTo(match.end())
  630 
  631     def matchMultiLineCommentStartToken(self):
  632         return self.multiLineCommentTokenStartRE.match(self.src(), self.pos())
  633 
  634     def getMultiLineCommentStartToken(self):
  635         match = self.matchMultiLineCommentStartToken()
  636         if not match:
  637             raise ParseError(self,
  638                              msg='Invalid multi-line comment start token')
  639         return self.readTo(match.end())
  640 
  641     def matchMultiLineCommentEndToken(self):
  642         return self.multiLineCommentEndTokenRE.match(self.src(), self.pos())
  643 
  644     def getMultiLineCommentEndToken(self):
  645         match = self.matchMultiLineCommentEndToken()
  646         if not match:
  647             raise ParseError(self, msg='Invalid multi-line comment end token')
  648         return self.readTo(match.end())
  649 
  650     def getCommaSeparatedSymbols(self):
  651         """
  652             Loosely based on getDottedName to pull out comma separated
  653             named chunks
  654         """
  655         srcLen = len(self)
  656         pieces = []
  657         nameChunks = []
  658 
  659         if not self.peek() in identchars:
  660             raise ParseError(self)
  661 
  662         while self.pos() < srcLen:
  663             c = self.peek()
  664             if c in namechars:
  665                 nameChunk = self.getIdentifier()
  666                 nameChunks.append(nameChunk)
  667             elif c == '.':
  668                 if self.pos() + 1 < srcLen and self.peek(1) in identchars:
  669                     nameChunks.append(self.getc())
  670                 else:
  671                     break
  672             elif c == ',':
  673                 self.getc()
  674                 pieces.append(''.join(nameChunks))
  675                 nameChunks = []
  676             elif c in (' ', '\t'):
  677                 self.getc()
  678             else:
  679                 break
  680 
  681         if nameChunks:
  682             pieces.append(''.join(nameChunks))
  683 
  684         return pieces
  685 
  686     def getDottedName(self):
  687         srcLen = len(self)
  688         nameChunks = []
  689 
  690         if not self.peek() in identchars:
  691             raise ParseError(self)
  692 
  693         while self.pos() < srcLen:
  694             c = self.peek()
  695             if c in namechars:
  696                 nameChunk = self.getIdentifier()
  697                 nameChunks.append(nameChunk)
  698             elif c == '.':
  699                 if self.pos() + 1 < srcLen and self.peek(1) in identchars:
  700                     nameChunks.append(self.getc())
  701                 else:
  702                     break
  703             else:
  704                 break
  705 
  706         return ''.join(nameChunks)
  707 
  708     def matchIdentifier(self):
  709         return identRE.match(self.src(), self.pos())
  710 
  711     def getIdentifier(self):
  712         match = self.matchIdentifier()
  713         if not match:
  714             raise ParseError(self, msg='Invalid identifier')
  715         return self.readTo(match.end())
  716 
  717     def matchOperator(self):
  718         match = self.matchPyToken()
  719         if match and match.group() not in operators:
  720             match = None
  721         return match
  722 
  723     def getOperator(self):
  724         match = self.matchOperator()
  725         if not match:
  726             raise ParseError(self, msg='Expected operator')
  727         return self.readTo(match.end())
  728 
  729     def matchAssignmentOperator(self):
  730         match = self.matchPyToken()
  731         if match and match.group() not in assignmentOps:
  732             match = None
  733         return match
  734 
  735     def getAssignmentOperator(self):
  736         match = self.matchAssignmentOperator()
  737         if not match:
  738             raise ParseError(self, msg='Expected assignment operator')
  739         return self.readTo(match.end())
  740 
  741     def matchDirective(self):
  742         """Returns False or the name of the directive matched.
  743         """
  744         startPos = self.pos()
  745         if not self.matchDirectiveStartToken():
  746             return False
  747         self.getDirectiveStartToken()
  748         directiveName = self.matchDirectiveName()
  749         self.setPos(startPos)
  750         return directiveName
  751 
  752     def matchDirectiveName(self,
  753                            directiveNameChars=identchars + '0123456789-@'):
  754         startPos = self.pos()
  755         possibleMatches = self._directiveNamesAndParsers.keys()
  756         name = ''
  757         match = None
  758 
  759         while not self.atEnd():
  760             c = self.getc()
  761             if c not in directiveNameChars:
  762                 break
  763             name += c
  764             if name == '@':
  765                 if not self.atEnd() and self.peek() in identchars:
  766                     match = '@'
  767                 break
  768             possibleMatches = [dn for dn in possibleMatches
  769                                if dn.startswith(name)]
  770             if not possibleMatches:
  771                 break
  772             elif (name in possibleMatches and (
  773                     self.atEnd() or self.peek() not in directiveNameChars)):
  774                 match = name
  775                 break
  776 
  777         self.setPos(startPos)
  778         return match
  779 
  780     def matchDirectiveStartToken(self):
  781         return self.directiveStartTokenRE.match(self.src(), self.pos())
  782 
  783     def getDirectiveStartToken(self):
  784         match = self.matchDirectiveStartToken()
  785         if not match:
  786             raise ParseError(self, msg='Invalid directive start token')
  787         return self.readTo(match.end())
  788 
  789     def matchDirectiveEndToken(self):
  790         return self.directiveEndTokenRE.match(self.src(), self.pos())
  791 
  792     def getDirectiveEndToken(self):
  793         match = self.matchDirectiveEndToken()
  794         if not match:
  795             raise ParseError(self, msg='Invalid directive end token')
  796         return self.readTo(match.end())
  797 
  798     def matchColonForSingleLineShortFormDirective(self):
  799         if not self.atEnd() and self.peek() == ':':
  800             restOfLine = self[self.pos()+1:self.findEOL()]  # noqa: E226,E501 missing whitespace around operator
  801             restOfLine = restOfLine.strip()
  802             if not restOfLine:
  803                 return False
  804             elif self.commentStartTokenRE.match(restOfLine):
  805                 return False
  806             else:  # non-whitespace, non-commment chars found
  807                 return True
  808         return False
  809 
  810     def matchPSPStartToken(self):
  811         return self.PSPStartTokenRE.match(self.src(), self.pos())
  812 
  813     def matchPSPEndToken(self):
  814         return self.PSPEndTokenRE.match(self.src(), self.pos())
  815 
  816     def getPSPStartToken(self):
  817         match = self.matchPSPStartToken()
  818         if not match:
  819             raise ParseError(self, msg='Invalid psp start token')
  820         return self.readTo(match.end())
  821 
  822     def getPSPEndToken(self):
  823         match = self.matchPSPEndToken()
  824         if not match:
  825             raise ParseError(self, msg='Invalid psp end token')
  826         return self.readTo(match.end())
  827 
  828     def matchCheetahVarStart(self):
  829         """includes the enclosure and cache token"""
  830         return self.cheetahVarStartRE.match(self.src(), self.pos())
  831 
  832     def matchCheetahVarStartToken(self):
  833         """includes the enclosure and cache token"""
  834         return self.cheetahVarStartTokenRE.match(self.src(), self.pos())
  835 
  836     def matchCheetahVarInExpressionStartToken(self):
  837         """no enclosures or cache tokens allowed"""
  838         return self.cheetahVarInExpressionStartTokenRE.match(self.src(),
  839                                                              self.pos())
  840 
  841     def matchVariablePlaceholderStart(self):
  842         """includes the enclosure and cache token"""
  843         return self.cheetahVarStartRE.match(self.src(), self.pos())
  844 
  845     def matchExpressionPlaceholderStart(self):
  846         """includes the enclosure and cache token"""
  847         return self.expressionPlaceholderStartRE.match(self.src(), self.pos())
  848 
  849     def getCheetahVarStartToken(self):
  850         """just the start token, not the enclosure or cache token"""
  851         match = self.matchCheetahVarStartToken()
  852         if not match:
  853             raise ParseError(self, msg='Expected Cheetah $var start token')
  854         return self.readTo(match.end())
  855 
  856     def getCacheToken(self):
  857         try:
  858             token = self.cacheTokenRE.match(self.src(), self.pos())
  859             self.setPos(token.end())
  860             return token.group()
  861         except Exception:
  862             raise ParseError(self, msg='Expected cache token')
  863 
  864     def getSilentPlaceholderToken(self):
  865         try:
  866             token = self.silentPlaceholderTokenRE.match(self.src(), self.pos())
  867             self.setPos(token.end())
  868             return token.group()
  869         except Exception:
  870             raise ParseError(self, msg='Expected silent placeholder token')
  871 
  872     def getTargetVarsList(self):
  873         varnames = []
  874         while not self.atEnd():
  875             if self.peek() in ' \t\f':
  876                 self.getWhiteSpace()
  877             elif self.peek() in '\r\n':
  878                 break
  879             elif self.startswith(','):
  880                 self.advance()
  881             elif self.startswith('in ') or self.startswith('in\t'):
  882                 break
  883             # elif self.matchCheetahVarStart():
  884             elif self.matchCheetahVarInExpressionStartToken():
  885                 self.getCheetahVarStartToken()
  886                 self.getSilentPlaceholderToken()
  887                 self.getCacheToken()
  888                 varnames.append(self.getDottedName())
  889             elif self.matchIdentifier():
  890                 varnames.append(self.getDottedName())
  891             else:
  892                 break
  893         return varnames
  894 
  895     def getCheetahVar(self, plain=False, skipStartToken=False):
  896         """This is called when parsing inside expressions. Cache tokens are only
  897         valid in placeholders so this method discards any cache tokens found.
  898         """
  899         if not skipStartToken:
  900             self.getCheetahVarStartToken()
  901         self.getSilentPlaceholderToken()
  902         self.getCacheToken()
  903         return self.getCheetahVarBody(plain=plain)
  904 
  905     def getCheetahVarBody(self, plain=False):
  906         # @@TR: this should be in the compiler
  907         return self._compiler.genCheetahVar(self.getCheetahVarNameChunks(),
  908                                             plain=plain)
  909 
  910     def getCheetahVarNameChunks(self):
  911 
  912         """
  913         nameChunks = list of Cheetah $var subcomponents represented as tuples
  914           [ (namemapperPart,autoCall,restOfName),
  915           ]
  916         where:
  917           namemapperPart = the dottedName base
  918           autocall = where NameMapper should use autocalling on namemapperPart
  919           restOfName = any arglist, index, or slice
  920 
  921         If restOfName contains a call arglist (e.g. '(1234)') then autocall is
  922         False, otherwise it defaults to True.
  923 
  924         EXAMPLE
  925         ------------------------------------------------------------------------
  926 
  927         if the raw CheetahVar is
  928           $a.b.c[1].d().x.y.z
  929 
  930         nameChunks is the list
  931           [ ('a.b.c',True,'[1]'),
  932             ('d',False,'()'),
  933             ('x.y.z',True,''),
  934           ]
  935 
  936         """
  937 
  938         chunks = []
  939         while self.pos() < len(self):
  940             rest = ''
  941             autoCall = True
  942             if not self.peek() in identchars + '.':
  943                 break
  944             elif self.peek() == '.':
  945 
  946                 if self.pos() + 1 < len(self) and self.peek(1) in identchars:
  947                     # discard the period as it isn't needed with NameMapper
  948                     self.advance()
  949                 else:
  950                     break
  951 
  952             dottedName = self.getDottedName()
  953             if not self.atEnd() and self.peek() in '([':
  954                 if self.peek() == '(':
  955                     rest = self.getCallArgString()
  956                 else:
  957                     rest = self.getExpression(enclosed=True)
  958 
  959                 period = max(dottedName.rfind('.'), 0)
  960                 if period:
  961                     chunks.append((dottedName[:period], autoCall, ''))
  962                     dottedName = dottedName[period+1:]  # noqa: E226,E501 missing whitespace around operator
  963                 if rest and rest[0] == '(':
  964                     autoCall = False
  965             chunks.append((dottedName, autoCall, rest))
  966 
  967         return chunks
  968 
  969     def getCallArgString(self,
  970                          # list of tuples (char, pos), where char is ( { or [
  971                          enclosures=[],
  972                          useNameMapper=Unspecified):
  973 
  974         """ Get a method/function call argument string.
  975 
  976         This method understands *arg, and **kw
  977         """
  978 
  979         # @@TR: this settings mangling should be removed
  980         if useNameMapper is not Unspecified:
  981             useNameMapper_orig = self.setting('useNameMapper')
  982             self.setSetting('useNameMapper', useNameMapper)
  983 
  984         if enclosures:
  985             pass
  986         else:
  987             if not self.peek() == '(':
  988                 raise ParseError(self, msg="Expected '('")
  989             startPos = self.pos()
  990             self.getc()
  991             enclosures = [('(', startPos),
  992                           ]
  993 
  994         argStringBits = ['(']
  995         addBit = argStringBits.append
  996 
  997         while True:
  998             if self.atEnd():
  999                 open = enclosures[-1][0]
 1000                 close = closurePairsRev[open]
 1001                 self.setPos(enclosures[-1][1])
 1002                 raise ParseError(
 1003                     self, msg="EOF was reached before a matching '" + close
 1004                     + "' was found for the '" + open + "'")
 1005 
 1006             c = self.peek()
 1007             if c in ")}]":  # get the ending enclosure and break
 1008                 if not enclosures:
 1009                     raise ParseError(self)
 1010                 c = self.getc()
 1011                 open = closurePairs[c]
 1012                 if enclosures[-1][0] == open:
 1013                     enclosures.pop()
 1014                     addBit(')')
 1015                     break
 1016                 else:
 1017                     raise ParseError(self)
 1018             elif c in " \t\f\r\n":
 1019                 addBit(self.getc())
 1020             elif self.matchCheetahVarInExpressionStartToken():
 1021                 startPos = self.pos()
 1022                 codeFor1stToken = self.getCheetahVar()
 1023                 WS = self.getWhiteSpace()
 1024                 if not self.atEnd() and self.peek() == '=':
 1025                     nextToken = self.getPyToken()
 1026                     if nextToken == '=':
 1027                         endPos = self.pos()
 1028                         self.setPos(startPos)
 1029                         codeFor1stToken = self.getCheetahVar(plain=True)
 1030                         self.setPos(endPos)
 1031 
 1032                     # finally
 1033                     addBit(codeFor1stToken + WS + nextToken)
 1034                 else:
 1035                     addBit(codeFor1stToken + WS)
 1036             elif self.matchCheetahVarStart():
 1037                 # it has syntax that is only valid at the top level
 1038                 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
 1039             else:
 1040                 beforeTokenPos = self.pos()
 1041                 token = self.getPyToken()
 1042                 if token in ('{', '(', '['):
 1043                     self.rev()
 1044                     token = self.getExpression(enclosed=True)
 1045                 token = self.transformToken(token, beforeTokenPos)
 1046                 addBit(token)
 1047 
 1048         if useNameMapper is not Unspecified:
 1049             # @@TR: see comment above
 1050             self.setSetting('useNameMapper', useNameMapper_orig)
 1051 
 1052         return ''.join(argStringBits)
 1053 
 1054     def getDefArgList(self, exitPos=None, useNameMapper=False):
 1055 
 1056         """Get an argument list. Can be used for method/function definition
 1057         argument lists or for #directive argument lists. Returns a list of
 1058         tuples in the form (argName, defVal=None) with one tuple for each arg
 1059         name.
 1060 
 1061         These defVals are always strings, so (argName, defVal=None) is safe
 1062         even with a case like (arg1, arg2=None, arg3=1234*2),
 1063         which would be returned as
 1064 
 1065         [('arg1', None),
 1066          ('arg2', 'None'),
 1067          ('arg3', '1234*2'),
 1068         ]
 1069 
 1070         This method understands *arg, and **kw.
 1071         """
 1072         if self.peek() == '(':
 1073             self.advance()
 1074         else:
 1075             exitPos = self.findEOL()  # it's a directive so break at the EOL
 1076         argList = ArgList()
 1077         onDefVal = False
 1078 
 1079         # @@TR: this settings mangling should be removed
 1080         useNameMapper_orig = self.setting('useNameMapper')
 1081         self.setSetting('useNameMapper', useNameMapper)
 1082 
 1083         while True:
 1084             if self.atEnd():
 1085                 raise ParseError(
 1086                     self, msg="EOF was reached before a matching ')'"
 1087                     + " was found for the '('")
 1088 
 1089             if self.pos() == exitPos:
 1090                 break
 1091 
 1092             c = self.peek()
 1093             if c == ")" or self.matchDirectiveEndToken():
 1094                 break
 1095             elif c == ":":
 1096                 break
 1097             elif c in " \t\f\r\n":
 1098                 if onDefVal:
 1099                     argList.add_default(c)
 1100                 self.advance()
 1101             elif c == '=':
 1102                 onDefVal = True
 1103                 self.advance()
 1104             elif c == ",":
 1105                 argList.next()
 1106                 onDefVal = False
 1107                 self.advance()
 1108             elif self.startswith(self.cheetahVarStartToken) and not onDefVal:
 1109                 self.advance(len(self.cheetahVarStartToken))
 1110             elif self.matchIdentifier() and not onDefVal:
 1111                 argList.add_argument(self.getIdentifier())
 1112             elif onDefVal:
 1113                 if self.matchCheetahVarInExpressionStartToken():
 1114                     token = self.getCheetahVar()
 1115                 elif self.matchCheetahVarStart():
 1116                     # it has syntax that is only valid at the top level
 1117                     self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
 1118                 else:
 1119                     beforeTokenPos = self.pos()
 1120                     token = self.getPyToken()
 1121                     if token in ('{', '(', '['):
 1122                         self.rev()
 1123                         token = self.getExpression(enclosed=True)
 1124                     token = self.transformToken(token, beforeTokenPos)
 1125                 argList.add_default(token)
 1126             elif c == '*' and not onDefVal:
 1127                 varName = self.getc()
 1128                 if self.peek() == '*':
 1129                     varName += self.getc()
 1130                 if not self.matchIdentifier():
 1131                     raise ParseError(self)
 1132                 varName += self.getIdentifier()
 1133                 argList.add_argument(varName)
 1134             else:
 1135                 raise ParseError(self)
 1136 
 1137         # @@TR: see comment above
 1138         self.setSetting('useNameMapper', useNameMapper_orig)
 1139         return argList.merge()
 1140 
 1141     def getExpressionParts(self,
 1142                            enclosed=False,
 1143                            # list of tuples (char, pos), where char is ( { or [
 1144                            enclosures=None,
 1145                            # only works if not enclosed
 1146                            pyTokensToBreakAt=None,
 1147                            useNameMapper=Unspecified,
 1148                            ):
 1149 
 1150         """ Get a Cheetah expression that includes $CheetahVars and break at
 1151         directive end tokens, the end of an enclosure, or at a specified
 1152         pyToken.
 1153         """
 1154 
 1155         if useNameMapper is not Unspecified:
 1156             useNameMapper_orig = self.setting('useNameMapper')
 1157             self.setSetting('useNameMapper', useNameMapper)
 1158 
 1159         if enclosures is None:
 1160             enclosures = []
 1161 
 1162         srcLen = len(self)
 1163         exprBits = []
 1164         while True:
 1165             if self.atEnd():
 1166                 if enclosures:
 1167                     open = enclosures[-1][0]
 1168                     close = closurePairsRev[open]
 1169                     self.setPos(enclosures[-1][1])
 1170                     raise ParseError(
 1171                         self,
 1172                         msg="EOF was reached before a matching '" + close
 1173                         + "' was found for the '" + open + "'")
 1174                 else:
 1175                     break
 1176 
 1177             c = self.peek()
 1178             if c in "{([":
 1179                 exprBits.append(c)
 1180                 enclosures.append((c, self.pos()))
 1181                 self.advance()
 1182             elif enclosed and not enclosures:
 1183                 break
 1184             elif c in "])}":
 1185                 if not enclosures:
 1186                     raise ParseError(self)
 1187                 open = closurePairs[c]
 1188                 if enclosures[-1][0] == open:
 1189                     enclosures.pop()
 1190                     exprBits.append(c)
 1191                 else:
 1192                     open = enclosures[-1][0]
 1193                     close = closurePairsRev[open]
 1194                     row, col = self.getRowCol()
 1195                     self.setPos(enclosures[-1][1])
 1196                     raise ParseError(
 1197                         self,
 1198                         msg="A '" + c + "' was found at line " + str(row)
 1199                         + ", col " + str(col)
 1200                         + " before a matching '" + close
 1201                         + "' was found\nfor the '" + open + "'")
 1202                 self.advance()
 1203 
 1204             elif c in " \f\t":
 1205                 exprBits.append(self.getWhiteSpace())
 1206             elif self.matchDirectiveEndToken() and not enclosures:
 1207                 break
 1208             elif c == "\\" and self.pos() + 1 < srcLen:
 1209                 eolMatch = EOLre.match(self.src(), self.pos() + 1)
 1210                 if not eolMatch:
 1211                     self.advance()
 1212                     raise ParseError(self, msg='Line ending expected')
 1213                 self.setPos(eolMatch.end())
 1214             elif c in '\r\n':
 1215                 if enclosures:
 1216                     self.advance()
 1217                 else:
 1218                     break
 1219             elif self.matchCheetahVarInExpressionStartToken():
 1220                 expr = self.getCheetahVar()
 1221                 exprBits.append(expr)
 1222             elif self.matchCheetahVarStart():
 1223                 # it has syntax that is only valid at the top level
 1224                 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
 1225             else:
 1226                 beforeTokenPos = self.pos()
 1227                 token = self.getPyToken()
 1228                 if (not enclosures and pyTokensToBreakAt
 1229                         and token in pyTokensToBreakAt):
 1230                     self.setPos(beforeTokenPos)
 1231                     break
 1232 
 1233                 token = self.transformToken(token, beforeTokenPos)
 1234 
 1235                 exprBits.append(token)
 1236                 if identRE.match(token):
 1237                     if token == 'for':
 1238                         expr = self.getExpression(
 1239                             useNameMapper=False, pyTokensToBreakAt=['in'])
 1240                         exprBits.append(expr)
 1241                     else:
 1242                         exprBits.append(self.getWhiteSpace())
 1243                         if not self.atEnd() and self.peek() == '(':
 1244                             exprBits.append(self.getCallArgString())
 1245         ##
 1246         if useNameMapper is not Unspecified:
 1247             # @@TR: see comment above
 1248             self.setSetting('useNameMapper', useNameMapper_orig)
 1249         return exprBits
 1250 
 1251     def getExpression(self,
 1252                       enclosed=False,
 1253                       # list of tuples (char, pos), where # char is ( { or [
 1254                       enclosures=None,
 1255                       pyTokensToBreakAt=None,
 1256                       useNameMapper=Unspecified,
 1257                       ):
 1258         """Returns the output of self.getExpressionParts() as a concatenated
 1259         string rather than as a list.
 1260         """
 1261         return ''.join(self.getExpressionParts(
 1262             enclosed=enclosed, enclosures=enclosures,
 1263             pyTokensToBreakAt=pyTokensToBreakAt,
 1264             useNameMapper=useNameMapper))
 1265 
 1266     def transformToken(self, token, beforeTokenPos):
 1267         """Takes a token from the expression being parsed and performs and
 1268         special transformations required by Cheetah.
 1269 
 1270         At the moment only Cheetah's c'$placeholder strings' are transformed.
 1271         """
 1272         if token == 'c' and not self.atEnd() and self.peek() in '\'"':
 1273             nextToken = self.getPyToken()
 1274             token = nextToken.upper()
 1275             theStr = eval(token)
 1276             endPos = self.pos()
 1277             if not theStr:
 1278                 return
 1279 
 1280             if token.startswith(single3) or token.startswith(double3):
 1281                 startPosIdx = 3
 1282             else:
 1283                 startPosIdx = 1
 1284             self.setPos(beforeTokenPos + startPosIdx + 1)
 1285             outputExprs = []
 1286             strConst = ''
 1287             while self.pos() < (endPos - startPosIdx):
 1288                 if self.matchCheetahVarStart() \
 1289                         or self.matchExpressionPlaceholderStart():
 1290                     if strConst:
 1291                         outputExprs.append(repr(strConst))
 1292                         strConst = ''
 1293                     placeholderExpr = self.getPlaceholder()
 1294                     outputExprs.append('str(' + placeholderExpr + ')')
 1295                 else:
 1296                     strConst += self.getc()
 1297             self.setPos(endPos)
 1298             if strConst:
 1299                 outputExprs.append(repr(strConst))
 1300             token = "''.join([" + ','.join(outputExprs) + "])"
 1301         return token
 1302 
 1303     def _raiseErrorAboutInvalidCheetahVarSyntaxInExpr(self):
 1304         match = self.matchCheetahVarStart()
 1305         groupdict = match.groupdict()
 1306         if groupdict.get('cacheToken'):
 1307             raise ParseError(
 1308                 self,
 1309                 msg='Cache tokens are not valid inside expressions. '
 1310                 'Use them in top-level $placeholders only.')
 1311         elif groupdict.get('enclosure'):
 1312             raise ParseError(
 1313                 self,
 1314                 msg='Long-form placeholders - ${}, $(), $[], etc. '
 1315                 'are not valid inside expressions. '
 1316                 'Use them in top-level $placeholders only.')
 1317         else:
 1318             raise ParseError(
 1319                 self,
 1320                 msg='This form of $placeholder syntax is not valid here.')
 1321 
 1322     def getPlaceholder(self, allowCacheTokens=False,
 1323                        plain=False, returnEverything=False):
 1324         # filtered
 1325         for callback in self.setting('preparsePlaceholderHooks'):
 1326             callback(parser=self)
 1327 
 1328         startPos = self.pos()
 1329         lineCol = self.getRowCol(startPos)
 1330         startToken = self.getCheetahVarStartToken()  # noqa: F841
 1331         silentPlaceholderToken = self.getSilentPlaceholderToken()
 1332         if silentPlaceholderToken:
 1333             isSilentPlaceholder = True
 1334         else:
 1335             isSilentPlaceholder = False
 1336 
 1337         if allowCacheTokens:
 1338             cacheToken = self.getCacheToken()
 1339             cacheTokenParts = self.cacheTokenRE.match(cacheToken).groupdict()
 1340         else:
 1341             cacheTokenParts = {}
 1342 
 1343         if self.peek() in '({[':
 1344             pos = self.pos()
 1345             enclosureOpenChar = self.getc()
 1346             enclosures = [(enclosureOpenChar, pos)]
 1347             self.getWhiteSpace()
 1348         else:
 1349             enclosures = []
 1350 
 1351         filterArgs = None
 1352         if self.matchIdentifier():
 1353             nameChunks = self.getCheetahVarNameChunks()
 1354             expr = self._compiler.genCheetahVar(nameChunks[:], plain=plain)
 1355             restOfExpr = None
 1356             if enclosures:
 1357                 WS = self.getWhiteSpace()
 1358                 expr += WS
 1359                 if self.setting('allowPlaceholderFilterArgs') \
 1360                         and self.peek() == ',':
 1361                     filterArgs = \
 1362                         self.getCallArgString(enclosures=enclosures)[1:-1]
 1363                 else:
 1364                     if self.peek() == closurePairsRev[enclosureOpenChar]:
 1365                         self.getc()
 1366                     else:
 1367                         restOfExpr = self.getExpression(
 1368                             enclosed=True, enclosures=enclosures)
 1369                         if restOfExpr[-1] == \
 1370                                 closurePairsRev[enclosureOpenChar]:
 1371                             restOfExpr = restOfExpr[:-1]
 1372                         expr += restOfExpr
 1373             rawPlaceholder = self[startPos: self.pos()]
 1374         else:
 1375             expr = self.getExpression(enclosed=True, enclosures=enclosures)
 1376             if expr[-1] == closurePairsRev[enclosureOpenChar]:
 1377                 expr = expr[:-1]
 1378             rawPlaceholder = self[startPos:self.pos()]
 1379 
 1380         expr = self._applyExpressionFilters(expr, 'placeholder',
 1381                                             rawExpr=rawPlaceholder,
 1382                                             startPos=startPos)
 1383         for callback in self.setting('postparsePlaceholderHooks'):
 1384             callback(parser=self)
 1385 
 1386         if returnEverything:
 1387             return (expr, rawPlaceholder, lineCol, cacheTokenParts,
 1388                     filterArgs, isSilentPlaceholder)
 1389         else:
 1390             return expr
 1391 
 1392 
 1393 class _HighLevelParser(_LowLevelParser):
 1394     """This class is a StateMachine for parsing Cheetah source and
 1395     sending state dependent code generation commands to
 1396     Cheetah.Compiler.Compiler.
 1397     """
 1398     def __init__(self, src, filename=None, breakPoint=None, compiler=None):
 1399         super(_HighLevelParser, self).__init__(
 1400             src, filename=filename, breakPoint=breakPoint)
 1401         self.setSettingsManager(compiler)
 1402         self._compiler = compiler
 1403         self.setupState()
 1404         self.configureParser()
 1405 
 1406     def setupState(self):
 1407         self._macros = {}
 1408         self._macroDetails = {}
 1409         self._openDirectivesStack = []
 1410 
 1411     def cleanup(self):
 1412         """Cleanup to remove any possible reference cycles
 1413         """
 1414         self._macros.clear()
 1415         for macroname, macroDetails in self._macroDetails.items():
 1416             macroDetails.template.shutdown()
 1417             del macroDetails.template
 1418         self._macroDetails.clear()
 1419 
 1420     def configureParser(self):
 1421         super(_HighLevelParser, self).configureParser()
 1422         self._initDirectives()
 1423 
 1424     def _initDirectives(self):
 1425         def normalizeParserVal(val):
 1426             if isinstance(val, (str, unicode)):
 1427                 handler = getattr(self, val)
 1428             elif isinstance(val, type):
 1429                 handler = val(self)
 1430             elif callable(val):
 1431                 handler = val
 1432             elif val is None:
 1433                 handler = val
 1434             else:
 1435                 raise Exception('Invalid parser/handler value %r for %s'
 1436                                 % (val, name))
 1437             return handler
 1438 
 1439         normalizeHandlerVal = normalizeParserVal
 1440 
 1441         _directiveNamesAndParsers = directiveNamesAndParsers.copy()
 1442         customNamesAndParsers = self.setting('directiveNamesAndParsers', {})
 1443         _directiveNamesAndParsers.update(customNamesAndParsers)
 1444 
 1445         _endDirectiveNamesAndHandlers = endDirectiveNamesAndHandlers.copy()
 1446         customNamesAndHandlers = self.setting('endDirectiveNamesAndHandlers',
 1447                                               {})
 1448         _endDirectiveNamesAndHandlers.update(customNamesAndHandlers)
 1449 
 1450         self._directiveNamesAndParsers = {}
 1451         for name, val in _directiveNamesAndParsers.items():
 1452             if val in (False, 0):
 1453                 continue
 1454             self._directiveNamesAndParsers[name] = normalizeParserVal(val)
 1455 
 1456         self._endDirectiveNamesAndHandlers = {}
 1457         for name, val in _endDirectiveNamesAndHandlers.items():
 1458             if val in (False, 0):
 1459                 continue
 1460             self._endDirectiveNamesAndHandlers[name] = normalizeHandlerVal(val)
 1461 
 1462         self._closeableDirectives = ['def', 'block', 'closure', 'defmacro',
 1463                                      'call',
 1464                                      'capture',
 1465                                      'cache',
 1466                                      'filter',
 1467                                      'if', 'unless',
 1468                                      'for', 'while', 'repeat',
 1469                                      'try',
 1470                                      ]
 1471         for directiveName in self.setting('closeableDirectives', []):
 1472             self._closeableDirectives.append(directiveName)
 1473 
 1474         macroDirectives = self.setting('macroDirectives', {})
 1475         macroDirectives['i18n'] = I18n
 1476 
 1477         for macroName, callback in macroDirectives.items():
 1478             if isinstance(callback, type):
 1479                 callback = callback(parser=self)
 1480             assert callback
 1481             self._macros[macroName] = callback
 1482             self._directiveNamesAndParsers[macroName] = self.eatMacroCall
 1483 
 1484     def _applyExpressionFilters(self, expr, exprType,
 1485                                 rawExpr=None, startPos=None):
 1486         """Pipes cheetah expressions through a set of optional filter hooks.
 1487 
 1488         The filters are functions which may modify the expressions or raise
 1489         a ForbiddenExpression exception if the expression is not allowed.  They
 1490         are defined in the compiler setting 'expressionFilterHooks'.
 1491 
 1492         Some intended use cases:
 1493 
 1494          - to implement 'restricted execution' safeguards in cases where you
 1495            can't trust the author of the template.
 1496 
 1497          - to enforce style guidelines
 1498 
 1499         Filter call signature:
 1500         (parser, expr, exprType, rawExpr=None, startPos=None)
 1501          - parser is the Cheetah parser.
 1502          - expr is the expression to filter.  In some cases the parser
 1503            will have already modified it from the original source code form.
 1504            For example, placeholders will have been translated
 1505            into namemapper calls.
 1506            If you need to work with the original source, see rawExpr.
 1507          - exprType is the name of the directive, 'psp', or 'placeholder'. All
 1508            lowercase.  @@TR: These will eventually be replaced with a set of
 1509            constants.
 1510          - rawExpr is the original source string that Cheetah parsed.  This
 1511            might be None in some cases.
 1512          - startPos is the character position in the source string/file
 1513            where the parser started parsing the current expression.
 1514 
 1515         @@TR: I realize this use of the term 'expression' is a bit wonky
 1516         as many of the 'expressions' are actually statements,
 1517         but I haven't thought of a better name yet.  Suggestions?
 1518         """
 1519         for callback in self.setting('expressionFilterHooks'):
 1520             expr = callback(parser=self, expr=expr, exprType=exprType,
 1521                             rawExpr=rawExpr, startPos=startPos)
 1522         return expr
 1523 
 1524     def _filterDisabledDirectives(self, directiveName):
 1525         directiveName = directiveName.lower()
 1526         if (directiveName in self.setting('disabledDirectives')
 1527             or (self.setting('enabledDirectives')
 1528                 and directiveName not in self.setting('enabledDirectives'))):
 1529             for callback in self.setting('disabledDirectiveHooks'):
 1530                 callback(parser=self, directiveName=directiveName)
 1531             raise ForbiddenDirective(
 1532                 self, msg='This %r directive is disabled' % directiveName)
 1533 
 1534     # main parse loop
 1535 
 1536     def parse(self, breakPoint=None, assertEmptyStack=True):
 1537         if breakPoint:
 1538             origBP = self.breakPoint()
 1539             self.setBreakPoint(breakPoint)
 1540             assertEmptyStack = False
 1541 
 1542         while not self.atEnd():
 1543             if self.matchCommentStartToken():
 1544                 self.eatComment()
 1545             elif self.matchMultiLineCommentStartToken():
 1546                 self.eatMultiLineComment()
 1547             elif self.matchVariablePlaceholderStart():
 1548                 self.eatPlaceholder()
 1549             elif self.matchExpressionPlaceholderStart():
 1550                 self.eatPlaceholder()
 1551             elif self.matchDirective():
 1552                 self.eatDirective()
 1553             elif self.matchPSPStartToken():
 1554                 self.eatPSP()
 1555             elif self.matchEOLSlurpToken():
 1556                 self.eatEOLSlurpToken()
 1557             else:
 1558                 self.eatPlainText()
 1559         if assertEmptyStack:
 1560             self.assertEmptyOpenDirectivesStack()
 1561         if breakPoint:
 1562             self.setBreakPoint(origBP)
 1563 
 1564     # non-directive eat methods
 1565 
 1566     def eatPlainText(self):
 1567         startPos = self.pos()
 1568         match = None
 1569         while not self.atEnd():
 1570             match = self.matchTopLevelToken()
 1571             if match:
 1572                 break
 1573             else:
 1574                 self.advance()
 1575         strConst = self.readTo(self.pos(), start=startPos)
 1576         strConst = self._unescapeCheetahVars(strConst)
 1577         strConst = self._unescapeDirectives(strConst)
 1578         self._compiler.addStrConst(strConst)
 1579         return match
 1580 
 1581     def eatComment(self):
 1582         isLineClearToStartToken = self.isLineClearToStartToken()
 1583         if isLineClearToStartToken:
 1584             self._compiler.handleWSBeforeDirective()
 1585         self.getCommentStartToken()
 1586         comm = self.readToEOL(gobble=isLineClearToStartToken)
 1587         self._compiler.addComment(comm)
 1588 
 1589     def eatMultiLineComment(self):
 1590         isLineClearToStartToken = self.isLineClearToStartToken()
 1591         endOfFirstLine = self.findEOL()
 1592 
 1593         self.getMultiLineCommentStartToken()
 1594         endPos = startPos = self.pos()
 1595         level = 1
 1596         while True:
 1597             endPos = self.pos()
 1598             if self.atEnd():
 1599                 break
 1600             if self.matchMultiLineCommentStartToken():
 1601                 self.getMultiLineCommentStartToken()
 1602                 level += 1
 1603             elif self.matchMultiLineCommentEndToken():
 1604                 self.getMultiLineCommentEndToken()
 1605                 level -= 1
 1606             if not level:
 1607                 break
 1608             self.advance()
 1609         comm = self.readTo(endPos, start=startPos)
 1610 
 1611         if not self.atEnd():
 1612             self.getMultiLineCommentEndToken()
 1613 
 1614         if (not self.atEnd()) and \
 1615                 self.setting('gobbleWhitespaceAroundMultiLineComments'):
 1616             restOfLine = self[self.pos():self.findEOL()]
 1617             if not restOfLine.strip():  # WS only to EOL
 1618                 self.readToEOL(gobble=isLineClearToStartToken)
 1619 
 1620             if isLineClearToStartToken and \
 1621                     (self.atEnd() or self.pos() > endOfFirstLine):
 1622                 self._compiler.handleWSBeforeDirective()
 1623 
 1624         self._compiler.addComment(comm)
 1625 
 1626     def eatPlaceholder(self):
 1627         (expr, rawPlaceholder,
 1628          lineCol, cacheTokenParts,
 1629          filterArgs, isSilentPlaceholder) = self.getPlaceholder(
 1630             allowCacheTokens=True, returnEverything=True)
 1631 
 1632         self._compiler.addPlaceholder(
 1633             expr,
 1634             filterArgs=filterArgs,
 1635             rawPlaceholder=rawPlaceholder,
 1636             cacheTokenParts=cacheTokenParts,
 1637             lineCol=lineCol,
 1638             silentMode=isSilentPlaceholder)
 1639         return
 1640 
 1641     def eatPSP(self):
 1642         # filtered
 1643         self._filterDisabledDirectives(directiveName='psp')
 1644         self.getPSPStartToken()
 1645         endToken = self.setting('PSPEndToken')
 1646         startPos = self.pos()
 1647         while not self.atEnd():
 1648             if self.peek() == endToken[0]:
 1649                 if self.matchPSPEndToken():
 1650                     break
 1651             self.advance()
 1652         pspString = self.readTo(self.pos(), start=startPos).strip()
 1653         pspString = self._applyExpressionFilters(pspString, 'psp',
 1654                                                  startPos=startPos)
 1655         self._compiler.addPSP(pspString)
 1656         self.getPSPEndToken()
 1657 
 1658     # generic directive eat methods
 1659     _simpleIndentingDirectives = '''
 1660     else elif for while repeat unless try except finally'''.split()
 1661     _simpleExprDirectives = '''
 1662     pass continue stop return yield break
 1663     del assert raise
 1664     silent echo
 1665     import from'''.split()
 1666     _directiveHandlerNames = {'import': 'addImportStatement',
 1667                               'from': 'addImportStatement', }
 1668 
 1669     def eatDirective(self):
 1670         directiveName = self.matchDirective()
 1671         self._filterDisabledDirectives(directiveName)
 1672 
 1673         for callback in self.setting('preparseDirectiveHooks'):
 1674             callback(parser=self, directiveName=directiveName)
 1675 
 1676         # subclasses can override the default behaviours here by providing an
 1677         # eater method in self._directiveNamesAndParsers[directiveName]
 1678         directiveParser = self._directiveNamesAndParsers.get(directiveName)
 1679         if directiveParser:
 1680             directiveParser()
 1681         elif directiveName in self._simpleIndentingDirectives:
 1682             handlerName = self._directiveHandlerNames.get(directiveName)
 1683             if not handlerName:
 1684                 handlerName = 'add' + directiveName.capitalize()
 1685             handler = getattr(self._compiler, handlerName)
 1686             self.eatSimpleIndentingDirective(directiveName, callback=handler)
 1687         elif directiveName in self._simpleExprDirectives:
 1688             handlerName = self._directiveHandlerNames.get(directiveName)
 1689             if not handlerName:
 1690                 handlerName = 'add' + directiveName.capitalize()
 1691             handler = getattr(self._compiler, handlerName)
 1692             if directiveName in ('silent', 'echo'):
 1693                 includeDirectiveNameInExpr = False
 1694             else:
 1695                 includeDirectiveNameInExpr = True
 1696             expr = self.eatSimpleExprDirective(
 1697                 directiveName,
 1698                 includeDirectiveNameInExpr=includeDirectiveNameInExpr)
 1699             handler(expr)
 1700         ##
 1701         for callback in self.setting('postparseDirectiveHooks'):
 1702             callback(parser=self, directiveName=directiveName)
 1703 
 1704     def _eatRestOfDirectiveTag(self, isLineClearToStartToken,
 1705                                endOfFirstLinePos):
 1706         foundComment = False
 1707         if self.matchCommentStartToken():
 1708             pos = self.pos()
 1709             self.advance()
 1710             if not self.matchDirective():
 1711                 self.setPos(pos)
 1712                 foundComment = True
 1713                 self.eatComment()  # this won't gobble the EOL
 1714             else:
 1715                 self.setPos(pos)
 1716 
 1717         if not foundComment and self.matchDirectiveEndToken():
 1718             self.getDirectiveEndToken()
 1719         elif isLineClearToStartToken and (not self.atEnd()) \
 1720                 and self.peek() in '\r\n':
 1721             # still gobble the EOL if a comment was found.
 1722             self.readToEOL(gobble=True)
 1723 
 1724         if isLineClearToStartToken and (
 1725                 self.atEnd() or self.pos() > endOfFirstLinePos):
 1726             self._compiler.handleWSBeforeDirective()
 1727 
 1728     def _eatToThisEndDirective(self, directiveName):
 1729         finalPos = endRawPos = startPos = self.pos()
 1730         directiveChar = self.setting('directiveStartToken')[0]
 1731         isLineClearToStartToken = False
 1732         while not self.atEnd():
 1733             if self.peek() == directiveChar:
 1734                 if self.matchDirective() == 'end':
 1735                     endRawPos = self.pos()
 1736                     self.getDirectiveStartToken()
 1737                     self.advance(len('end'))
 1738                     self.getWhiteSpace()
 1739                     if self.startswith(directiveName):
 1740                         if self.isLineClearToStartToken(endRawPos):
 1741                             isLineClearToStartToken = True
 1742                             endRawPos = self.findBOL(endRawPos)
 1743                         # to the end of directiveName
 1744                         self.advance(len(directiveName))
 1745                         self.getWhiteSpace()
 1746                         finalPos = self.pos()
 1747                         break
 1748             self.advance()
 1749             finalPos = endRawPos = self.pos()
 1750 
 1751         textEaten = self.readTo(endRawPos, start=startPos)
 1752         self.setPos(finalPos)
 1753 
 1754         endOfFirstLinePos = self.findEOL()
 1755 
 1756         if self.matchDirectiveEndToken():
 1757             self.getDirectiveEndToken()
 1758         elif isLineClearToStartToken and (not self.atEnd()) \
 1759                 and self.peek() in '\r\n':
 1760             self.readToEOL(gobble=True)
 1761 
 1762         if isLineClearToStartToken and self.pos() > endOfFirstLinePos:
 1763             self._compiler.handleWSBeforeDirective()
 1764         return textEaten
 1765 
 1766     def eatSimpleExprDirective(self, directiveName,
 1767                                includeDirectiveNameInExpr=True):
 1768         # filtered
 1769         isLineClearToStartToken = self.isLineClearToStartToken()
 1770         endOfFirstLine = self.findEOL()
 1771         self.getDirectiveStartToken()
 1772         if not includeDirectiveNameInExpr:
 1773             self.advance(len(directiveName))
 1774         startPos = self.pos()
 1775         expr = self.getExpression().strip()
 1776         directiveName = expr.split()[0]
 1777         expr = self._applyExpressionFilters(expr, directiveName,
 1778                                             startPos=startPos)
 1779         if directiveName in self._closeableDirectives:
 1780             self.pushToOpenDirectivesStack(directiveName)
 1781         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
 1782         return expr
 1783 
 1784     def eatSimpleIndentingDirective(self, directiveName, callback,
 1785                                     includeDirectiveNameInExpr=False):
 1786         # filtered
 1787         isLineClearToStartToken = self.isLineClearToStartToken()
 1788         endOfFirstLinePos = self.findEOL()
 1789         lineCol = self.getRowCol()
 1790         self.getDirectiveStartToken()
 1791         if directiveName not in \
 1792                 'else elif for while try except finally'.split():
 1793             self.advance(len(directiveName))
 1794         startPos = self.pos()
 1795 
 1796         self.getWhiteSpace()
 1797 
 1798         expr = self.getExpression(pyTokensToBreakAt=[':'])
 1799         expr = self._applyExpressionFilters(expr, directiveName,
 1800                                             startPos=startPos)
 1801         if self.matchColonForSingleLineShortFormDirective():
 1802             self.advance()  # skip over :
 1803             if directiveName in 'else elif except finally'.split():
 1804                 callback(expr, dedent=False, lineCol=lineCol)
 1805             else:
 1806                 callback(expr, lineCol=lineCol)
 1807 
 1808             self.getWhiteSpace(max=1)
 1809             self.parse(breakPoint=self.findEOL(gobble=True))
 1810             self._compiler.commitStrConst()
 1811             self._compiler.dedent()
 1812         else:
 1813             if self.peek() == ':':
 1814                 self.advance()
 1815             self.getWhiteSpace()
 1816             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 1817                                         endOfFirstLinePos)
 1818             if directiveName in self._closeableDirectives:
 1819                 self.pushToOpenDirectivesStack(directiveName)
 1820             callback(expr, lineCol=lineCol)
 1821 
 1822     def eatEndDirective(self):
 1823         isLineClearToStartToken = self.isLineClearToStartToken()
 1824         self.getDirectiveStartToken()
 1825         self.advance(3)                 # to end of 'end'
 1826         self.getWhiteSpace()
 1827         pos = self.pos()
 1828         directiveName = False
 1829         for key in self._endDirectiveNamesAndHandlers.keys():
 1830             if self.find(key, pos) == pos:
 1831                 directiveName = key
 1832                 break
 1833         if not directiveName:
 1834             raise ParseError(self, msg='Invalid end directive')
 1835 
 1836         endOfFirstLinePos = self.findEOL()
 1837         self.getExpression()  # eat in any extra comment-like crap
 1838         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
 1839         if directiveName in self._closeableDirectives:
 1840             self.popFromOpenDirectivesStack(directiveName)
 1841 
 1842         # subclasses can override the default behaviours here
 1843         # by providing an end-directive handler
 1844         # in self._endDirectiveNamesAndHandlers[directiveName]
 1845         if self._endDirectiveNamesAndHandlers.get(directiveName):
 1846             handler = self._endDirectiveNamesAndHandlers[directiveName]
 1847             handler()
 1848         elif directiveName in \
 1849                 'block capture cache call filter errorCatcher'.split():
 1850             if key == 'block':
 1851                 self._compiler.closeBlock()
 1852             elif key == 'capture':
 1853                 self._compiler.endCaptureRegion()
 1854             elif key == 'cache':
 1855                 self._compiler.endCacheRegion()
 1856             elif key == 'call':
 1857                 self._compiler.endCallRegion()
 1858             elif key == 'filter':
 1859                 self._compiler.closeFilterBlock()
 1860             elif key == 'errorCatcher':
 1861                 self._compiler.turnErrorCatcherOff()
 1862         elif directiveName in 'while for if try repeat unless'.split():
 1863             self._compiler.commitStrConst()
 1864             self._compiler.dedent()
 1865         elif directiveName == 'closure':
 1866             self._compiler.commitStrConst()
 1867             self._compiler.dedent()
 1868             # @@TR: temporary hack of useSearchList
 1869             self.setSetting('useSearchList', self._useSearchList_orig)
 1870 
 1871     # specific directive eat methods
 1872 
 1873     def eatBreakPoint(self):
 1874         """Tells the parser to stop parsing at this point and completely ignore
 1875         everything else.
 1876 
 1877         This is a debugging tool.
 1878         """
 1879         self.setBreakPoint(self.pos())
 1880 
 1881     def eatShbang(self):
 1882         # filtered
 1883         self.getDirectiveStartToken()
 1884         self.advance(len('shBang'))
 1885         self.getWhiteSpace()
 1886         startPos = self.pos()
 1887         shBang = self.readToEOL()
 1888         shBang = self._applyExpressionFilters(shBang, 'shbang',
 1889                                               startPos=startPos)
 1890         self._compiler.setShBang(shBang.strip())
 1891 
 1892     def eatEncoding(self):
 1893         # filtered
 1894         self.getDirectiveStartToken()
 1895         self.advance(len('encoding'))
 1896         self.getWhiteSpace()
 1897         startPos = self.pos()
 1898         encoding = self.readToEOL()
 1899         encoding = self._applyExpressionFilters(encoding, 'encoding',
 1900                                                 startPos=startPos)
 1901         self._compiler.setModuleEncoding(encoding.strip())
 1902 
 1903     def eatCompiler(self):
 1904         # filtered
 1905         isLineClearToStartToken = self.isLineClearToStartToken()
 1906         endOfFirstLine = self.findEOL()
 1907         startPos = self.pos()
 1908         self.getDirectiveStartToken()
 1909         self.advance(len('compiler'))   # to end of 'compiler'
 1910         self.getWhiteSpace()
 1911 
 1912         startPos = self.pos()
 1913         settingName = self.getIdentifier()
 1914 
 1915         if settingName.lower() == 'reset':
 1916             self.getExpression()  # gobble whitespace & junk
 1917             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 1918                                         endOfFirstLine)
 1919             self._initializeSettings()
 1920             self.configureParser()
 1921             return
 1922 
 1923         self.getWhiteSpace()
 1924         if self.peek() == '=':
 1925             self.advance()
 1926         else:
 1927             raise ParseError(self)
 1928         valueExpr = self.getExpression()
 1929         endPos = self.pos()
 1930 
 1931         # @@TR: it's unlikely that anyone apply filters would have left this
 1932         # directive enabled:
 1933         # @@TR: fix up filtering, regardless
 1934         self._applyExpressionFilters('%s=%r' % (settingName, valueExpr),
 1935                                      'compiler', startPos=startPos)
 1936 
 1937         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
 1938         try:
 1939             self._compiler.setCompilerSetting(settingName, valueExpr)
 1940         except Exception:
 1941             sys.stderr.write(
 1942                 'An error occurred while processing '
 1943                 'the following #compiler directive.\n')
 1944             sys.stderr.write(
 1945                 '------------------------------------'
 1946                 '----------------------------------\n')
 1947             sys.stderr.write('%s\n' % self[startPos:endPos])
 1948             sys.stderr.write(
 1949                 '------------------------------------'
 1950                 '----------------------------------\n')
 1951             sys.stderr.write('Please check the syntax of these settings.\n\n')
 1952             raise
 1953 
 1954     def eatCompilerSettings(self):
 1955         # filtered
 1956         isLineClearToStartToken = self.isLineClearToStartToken()
 1957         endOfFirstLine = self.findEOL()
 1958         self.getDirectiveStartToken()
 1959         self.advance(len('compiler-settings'))   # to end of 'settings'
 1960 
 1961         keywords = self.getTargetVarsList()
 1962         self.getExpression()            # gobble any garbage
 1963 
 1964         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
 1965 
 1966         if 'reset' in keywords:
 1967             self._compiler._initializeSettings()
 1968             self.configureParser()
 1969             # @@TR: this implies a single-line #compiler-settings directive,
 1970             # and thus we should parse forward for an end directive.
 1971             # Subject to change in the future.
 1972             return
 1973         startPos = self.pos()
 1974         settingsStr = self._eatToThisEndDirective('compiler-settings')
 1975         settingsStr = self._applyExpressionFilters(
 1976             settingsStr, 'compilerSettings', startPos=startPos)
 1977         try:
 1978             self._compiler.setCompilerSettings(keywords=keywords,
 1979                                                settingsStr=settingsStr)
 1980         except Exception:
 1981             sys.stderr.write(
 1982                 'An error occurred while processing '
 1983                 'the following compiler settings.\n')
 1984             sys.stderr.write(
 1985                 '------------------------------------'
 1986                 '----------------------------------\n')
 1987             sys.stderr.write('%s\n' % settingsStr.strip())
 1988             sys.stderr.write(
 1989                 '------------------------------------'
 1990                 '----------------------------------\n')
 1991             sys.stderr.write('Please check the syntax of these settings.\n\n')
 1992             raise
 1993 
 1994     def eatAttr(self):
 1995         # filtered
 1996         isLineClearToStartToken = self.isLineClearToStartToken()
 1997         endOfFirstLinePos = self.findEOL()
 1998         startPos = self.pos()
 1999         self.getDirectiveStartToken()
 2000         self.advance(len('attr'))
 2001         self.getWhiteSpace()
 2002         startPos = self.pos()
 2003         if self.matchCheetahVarStart():
 2004             self.getCheetahVarStartToken()
 2005         attribName = self.getIdentifier()
 2006         self.getWhiteSpace()
 2007         self.getAssignmentOperator()
 2008         expr = self.getExpression()
 2009         expr = self._applyExpressionFilters(expr, 'attr', startPos=startPos)
 2010         self._compiler.addAttribute(attribName, expr)
 2011         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
 2012 
 2013     def eatDecorator(self):
 2014         isLineClearToStartToken = self.isLineClearToStartToken()
 2015         endOfFirstLinePos = self.findEOL()
 2016         startPos = self.pos()
 2017         self.getDirectiveStartToken()
 2018         # self.advance() # eat @
 2019         startPos = self.pos()
 2020         decoratorExpr = self.getExpression()
 2021         decoratorExpr = self._applyExpressionFilters(
 2022             decoratorExpr, 'decorator', startPos=startPos)
 2023         self._compiler.addDecorator(decoratorExpr)
 2024         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
 2025         self.getWhiteSpace()
 2026 
 2027         directiveName = self.matchDirective()
 2028         if not directiveName or \
 2029                 directiveName not in ('def', 'block', 'closure', '@'):
 2030             raise ParseError(
 2031                 self, msg='Expected #def, #block, #closure '
 2032                 'or another @decorator')
 2033         self.eatDirective()
 2034 
 2035     def eatDef(self):
 2036         # filtered
 2037         self._eatDefOrBlock('def')
 2038 
 2039     def eatBlock(self):
 2040         # filtered
 2041         startPos = self.pos()
 2042         methodName, rawSignature = self._eatDefOrBlock('block')
 2043         self._compiler._blockMetaData[methodName] = {
 2044             'raw': rawSignature,
 2045             'lineCol': self.getRowCol(startPos),
 2046         }
 2047 
 2048     def eatClosure(self):
 2049         # filtered
 2050         self._eatDefOrBlock('closure')
 2051 
 2052     def _eatDefOrBlock(self, directiveName):
 2053         # filtered
 2054         assert directiveName in ('def', 'block', 'closure')
 2055         isLineClearToStartToken = self.isLineClearToStartToken()
 2056         endOfFirstLinePos = self.findEOL()
 2057         startPos = self.pos()
 2058         self.getDirectiveStartToken()
 2059         self.advance(len(directiveName))
 2060         self.getWhiteSpace()
 2061         if self.matchCheetahVarStart():
 2062             self.getCheetahVarStartToken()
 2063         methodName = self.getIdentifier()
 2064         self.getWhiteSpace()
 2065         if self.peek() == '(':
 2066             argsList = self.getDefArgList()
 2067             self.advance()              # past the closing ')'
 2068             if argsList and argsList[0][0] == 'self':
 2069                 del argsList[0]
 2070         else:
 2071             argsList = []
 2072 
 2073         def includeBlockMarkers():
 2074             if self.setting('includeBlockMarkers'):
 2075                 startMarker = self.setting('blockMarkerStart')
 2076                 self._compiler.addStrConst(
 2077                     startMarker[0] + methodName + startMarker[1])
 2078 
 2079         # @@TR: fix up filtering
 2080         self._applyExpressionFilters(self[startPos:self.pos()], 'def',
 2081                                      startPos=startPos)
 2082 
 2083         if self.matchColonForSingleLineShortFormDirective():
 2084             isNestedDef = (self.setting('allowNestedDefScopes')
 2085                            and [name for name in self._openDirectivesStack
 2086                                 if name == 'def'])
 2087             self.getc()
 2088             rawSignature = self[startPos:endOfFirstLinePos]
 2089             self._eatSingleLineDef(directiveName=directiveName,
 2090                                    methodName=methodName,
 2091                                    argsList=argsList,
 2092                                    startPos=startPos,
 2093                                    endPos=endOfFirstLinePos)
 2094             if directiveName == 'def' and not isNestedDef:
 2095                 # @@TR: must come before _eatRestOfDirectiveTag...
 2096                 # ...for some reason.
 2097                 self._compiler.closeDef()
 2098             elif directiveName == 'block':
 2099                 includeBlockMarkers()
 2100                 self._compiler.closeBlock()
 2101             elif directiveName == 'closure' or isNestedDef:
 2102                 self._compiler.dedent()
 2103 
 2104             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2105                                         endOfFirstLinePos)
 2106         else:
 2107             if self.peek() == ':':
 2108                 self.getc()
 2109             self.pushToOpenDirectivesStack(directiveName)
 2110             rawSignature = self[startPos:self.pos()]
 2111             self._eatMultiLineDef(
 2112                 directiveName=directiveName, methodName=methodName,
 2113                 argsList=argsList, startPos=startPos,
 2114                 isLineClearToStartToken=isLineClearToStartToken)
 2115             if directiveName == 'block':
 2116                 includeBlockMarkers()
 2117 
 2118         return methodName, rawSignature
 2119 
 2120     def _eatMultiLineDef(self, directiveName, methodName, argsList, startPos,
 2121                          isLineClearToStartToken=False):
 2122         # filtered in calling method
 2123         self.getExpression()            # slurp up any garbage left at the end
 2124         signature = self[startPos:self.pos()]
 2125         endOfFirstLinePos = self.findEOL()
 2126         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
 2127         signature = ' '.join([line.strip() for line in signature.splitlines()])
 2128         parserComment = ('## CHEETAH: generated from ' + signature
 2129                          + ' at line %s, col %s' % self.getRowCol(startPos)
 2130                          + '.')
 2131 
 2132         isNestedDef = (self.setting('allowNestedDefScopes')
 2133                        and len([name for name in self._openDirectivesStack
 2134                                 if name == 'def']) > 1)
 2135         if directiveName == 'block' or (
 2136                 directiveName == 'def' and not isNestedDef):
 2137             self._compiler.startMethodDef(methodName, argsList, parserComment)
 2138         else:  # closure
 2139             self._useSearchList_orig = self.setting('useSearchList')
 2140             self.setSetting('useSearchList', False)
 2141             self._compiler.addClosure(methodName, argsList, parserComment)
 2142 
 2143         return methodName
 2144 
 2145     def _eatSingleLineDef(self, directiveName, methodName,
 2146                           argsList, startPos, endPos):
 2147         # filtered in calling method
 2148         fullSignature = self[startPos:endPos]
 2149         parserComment = ('## Generated from ' + fullSignature
 2150                          + ' at line %s, col %s' % self.getRowCol(startPos)
 2151                          + '.')
 2152         isNestedDef = (self.setting('allowNestedDefScopes')
 2153                        and [name for name in self._openDirectivesStack
 2154                             if name == 'def'])
 2155         if directiveName == 'block' or (
 2156                 directiveName == 'def' and not isNestedDef):
 2157             self._compiler.startMethodDef(methodName, argsList, parserComment)
 2158         else:  # closure
 2159             # @@TR: temporary hack of useSearchList
 2160             useSearchList_orig = self.setting('useSearchList')
 2161             self.setSetting('useSearchList', False)
 2162             self._compiler.addClosure(methodName, argsList, parserComment)
 2163 
 2164         self.getWhiteSpace(max=1)
 2165         self.parse(breakPoint=endPos)
 2166         if directiveName == 'closure' or isNestedDef:
 2167             # @@TR: temporary hack of useSearchList
 2168             self.setSetting('useSearchList', useSearchList_orig)
 2169 
 2170     def eatExtends(self):
 2171         # filtered
 2172         isLineClearToStartToken = self.isLineClearToStartToken()
 2173         endOfFirstLine = self.findEOL()
 2174         self.getDirectiveStartToken()
 2175         self.advance(len('extends'))
 2176         self.getWhiteSpace()
 2177         startPos = self.pos()
 2178         if self.setting('allowExpressionsInExtendsDirective'):
 2179             baseName = self.getExpression()
 2180         else:
 2181             baseName = self.getCommaSeparatedSymbols()
 2182             baseName = ', '.join(baseName)
 2183 
 2184         baseName = self._applyExpressionFilters(baseName, 'extends',
 2185                                                 startPos=startPos)
 2186         self._compiler.setBaseClass(baseName)  # in compiler
 2187         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
 2188 
 2189     def eatImplements(self):
 2190         # filtered
 2191         isLineClearToStartToken = self.isLineClearToStartToken()
 2192         endOfFirstLine = self.findEOL()
 2193         self.getDirectiveStartToken()
 2194         self.advance(len('implements'))
 2195         self.getWhiteSpace()
 2196         startPos = self.pos()
 2197         methodName = self.getIdentifier()
 2198         if not self.atEnd() and self.peek() == '(':
 2199             argsList = self.getDefArgList()
 2200             self.advance()              # past the closing ')'
 2201             if argsList and argsList[0][0] == 'self':
 2202                 del argsList[0]
 2203         else:
 2204             argsList = []
 2205 
 2206         # @@TR: need to split up filtering of the methodname and the args
 2207         # methodName = self._applyExpressionFilters(methodName, 'implements',
 2208         #                                           startPos=startPos)
 2209         self._applyExpressionFilters(self[startPos:self.pos()], 'implements',
 2210                                      startPos=startPos)
 2211 
 2212         self._compiler.setMainMethodName(methodName)
 2213         self._compiler.setMainMethodArgs(argsList)
 2214 
 2215         self.getExpression()  # throw away and unwanted crap that got added in
 2216         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
 2217 
 2218     def eatSuper(self):
 2219         # filtered
 2220         isLineClearToStartToken = self.isLineClearToStartToken()
 2221         endOfFirstLine = self.findEOL()
 2222         self.getDirectiveStartToken()
 2223         self.advance(len('super'))
 2224         self.getWhiteSpace()
 2225         startPos = self.pos()
 2226         if not self.atEnd() and self.peek() == '(':
 2227             argsList = self.getDefArgList()
 2228             self.advance()              # past the closing ')'
 2229             if argsList and argsList[0][0] == 'self':
 2230                 del argsList[0]
 2231         else:
 2232             argsList = []
 2233 
 2234         self._applyExpressionFilters(self[startPos:self.pos()], 'super',
 2235                                      startPos=startPos)
 2236 
 2237         # parserComment = ('## CHEETAH: generated from ' + signature +
 2238         #                 ' at line %s, col %s' % self.getRowCol(startPos)
 2239         #                 + '.')
 2240 
 2241         self.getExpression()  # throw away and unwanted crap that got added in
 2242         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
 2243         self._compiler.addSuper(argsList)
 2244 
 2245     def eatSet(self):
 2246         # filtered
 2247         isLineClearToStartToken = self.isLineClearToStartToken()
 2248         endOfFirstLine = self.findEOL()
 2249         self.getDirectiveStartToken()
 2250         self.advance(3)
 2251         self.getWhiteSpace()
 2252         style = SET_LOCAL
 2253         if self.startswith('local'):
 2254             self.getIdentifier()
 2255             self.getWhiteSpace()
 2256         elif self.startswith('global'):
 2257             self.getIdentifier()
 2258             self.getWhiteSpace()
 2259             style = SET_GLOBAL
 2260         elif self.startswith('module'):
 2261             self.getIdentifier()
 2262             self.getWhiteSpace()
 2263             style = SET_MODULE
 2264 
 2265         startPos = self.pos()
 2266         LVALUE = self.getExpression(pyTokensToBreakAt=assignmentOps,
 2267                                     useNameMapper=False).strip()
 2268         OP = self.getAssignmentOperator()
 2269         RVALUE = self.getExpression()
 2270         expr = LVALUE + ' ' + OP + ' ' + RVALUE.strip()
 2271 
 2272         expr = self._applyExpressionFilters(expr, 'set', startPos=startPos)
 2273         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
 2274 
 2275         # used for 'set global'
 2276         class Components:
 2277             pass
 2278         exprComponents = Components()
 2279         exprComponents.LVALUE = LVALUE
 2280         exprComponents.OP = OP
 2281         exprComponents.RVALUE = RVALUE
 2282         self._compiler.addSet(expr, exprComponents, style)
 2283 
 2284     def eatSlurp(self):
 2285         if self.isLineClearToStartToken():
 2286             self._compiler.handleWSBeforeDirective()
 2287         self._compiler.commitStrConst()
 2288         self.readToEOL(gobble=True)
 2289 
 2290     def eatEOLSlurpToken(self):
 2291         if self.isLineClearToStartToken():
 2292             self._compiler.handleWSBeforeDirective()
 2293         self._compiler.commitStrConst()
 2294         self.readToEOL(gobble=True)
 2295 
 2296     def eatRaw(self):
 2297         isLineClearToStartToken = self.isLineClearToStartToken()
 2298         endOfFirstLinePos = self.findEOL()
 2299         self.getDirectiveStartToken()
 2300         self.advance(len('raw'))
 2301         self.getWhiteSpace()
 2302         if self.matchColonForSingleLineShortFormDirective():
 2303             self.advance()  # skip over :
 2304             self.getWhiteSpace(max=1)
 2305             rawBlock = self.readToEOL(gobble=False)
 2306         else:
 2307             if self.peek() == ':':
 2308                 self.advance()
 2309             self.getWhiteSpace()
 2310             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2311                                         endOfFirstLinePos)
 2312             rawBlock = self._eatToThisEndDirective('raw')
 2313         self._compiler.addRawText(rawBlock)
 2314 
 2315     def eatInclude(self):
 2316         # filtered
 2317         isLineClearToStartToken = self.isLineClearToStartToken()
 2318         endOfFirstLinePos = self.findEOL()
 2319         self.getDirectiveStartToken()
 2320         self.advance(len('include'))
 2321 
 2322         self.getWhiteSpace()
 2323         includeFrom = 'file'
 2324         isRaw = False
 2325         if self.startswith('raw'):
 2326             self.advance(3)
 2327             isRaw = True
 2328 
 2329         self.getWhiteSpace()
 2330         if self.startswith('source'):
 2331             self.advance(len('source'))
 2332             includeFrom = 'str'
 2333             self.getWhiteSpace()
 2334             if not self.peek() == '=':
 2335                 raise ParseError(self)
 2336             self.advance()
 2337         startPos = self.pos()
 2338         sourceExpr = self.getExpression()
 2339         sourceExpr = self._applyExpressionFilters(sourceExpr, 'include',
 2340                                                   startPos=startPos)
 2341         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
 2342         self._compiler.addInclude(sourceExpr, includeFrom, isRaw)
 2343 
 2344     def eatDefMacro(self):
 2345         # @@TR: not filtered yet
 2346         isLineClearToStartToken = self.isLineClearToStartToken()
 2347         endOfFirstLinePos = self.findEOL()
 2348         self.getDirectiveStartToken()
 2349         self.advance(len('defmacro'))
 2350 
 2351         self.getWhiteSpace()
 2352         if self.matchCheetahVarStart():
 2353             self.getCheetahVarStartToken()
 2354         macroName = self.getIdentifier()
 2355         self.getWhiteSpace()
 2356         if self.peek() == '(':
 2357             argsList = self.getDefArgList(useNameMapper=False)
 2358             self.advance()              # past the closing ')'
 2359             if argsList and argsList[0][0] == 'self':
 2360                 del argsList[0]
 2361         else:
 2362             argsList = []
 2363 
 2364         assert macroName not in self._directiveNamesAndParsers
 2365         argsList.insert(0, ('src', None))
 2366         argsList.append(('parser', 'None'))
 2367         argsList.append(('macros', 'None'))
 2368         argsList.append(('compilerSettings', 'None'))
 2369         argsList.append(('isShortForm', 'None'))
 2370         argsList.append(('EOLCharsInShortForm', 'None'))
 2371         argsList.append(('startPos', 'None'))
 2372         argsList.append(('endPos', 'None'))
 2373 
 2374         if self.matchColonForSingleLineShortFormDirective():
 2375             self.advance()  # skip over :
 2376             self.getWhiteSpace(max=1)
 2377             macroSrc = self.readToEOL(gobble=False)
 2378             self.readToEOL(gobble=True)
 2379         else:
 2380             if self.peek() == ':':
 2381                 self.advance()
 2382             self.getWhiteSpace()
 2383             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2384                                         endOfFirstLinePos)
 2385             macroSrc = self._eatToThisEndDirective('defmacro')
 2386 
 2387         normalizedMacroSrc = ''.join(
 2388             ['%def callMacro(' + ','.join([defv and '%s=%s' % (n, defv) or n
 2389                                            for n, defv in argsList]) + ')\n',
 2390              macroSrc,
 2391              '%end def'])
 2392 
 2393         from Cheetah.Template import Template
 2394         templateAPIClass = self.setting('templateAPIClassForDefMacro',
 2395                                         default=Template)
 2396         compilerSettings = self.setting('compilerSettingsForDefMacro',
 2397                                         default={})
 2398         searchListForMacros = self.setting('searchListForDefMacro', default=[])
 2399         # copy to avoid mutation bugs
 2400         searchListForMacros = list(searchListForMacros)
 2401         searchListForMacros.append({'macros': self._macros,
 2402                                     'parser': self,
 2403                                     'compilerSettings': self.settings(),
 2404                                     })
 2405 
 2406         templateAPIClass._updateSettingsWithPreprocessTokens(
 2407             compilerSettings, placeholderToken='@', directiveToken='%')
 2408         macroTemplateClass = templateAPIClass.compile(
 2409             source=normalizedMacroSrc, compilerSettings=compilerSettings)
 2410         # t = macroTemplateClass()
 2411 
 2412         class MacroDetails:
 2413             pass
 2414         macroDetails = MacroDetails()
 2415         macroDetails.macroSrc = macroSrc
 2416         macroDetails.argsList = argsList
 2417         macroDetails.template = macroTemplateClass(
 2418             searchList=searchListForMacros)
 2419 
 2420         self._macroDetails[macroName] = macroDetails
 2421         self._macros[macroName] = macroDetails.template.callMacro
 2422         self._directiveNamesAndParsers[macroName] = self.eatMacroCall
 2423 
 2424     def eatMacroCall(self):
 2425         isLineClearToStartToken = self.isLineClearToStartToken()
 2426         endOfFirstLinePos = self.findEOL()
 2427         startPos = self.pos()
 2428         self.getDirectiveStartToken()
 2429         macroName = self.getIdentifier()
 2430         macro = self._macros[macroName]
 2431         if hasattr(macro, 'parse'):
 2432             return macro.parse(parser=self, startPos=startPos)
 2433 
 2434         if hasattr(macro, 'parseArgs'):
 2435             args = macro.parseArgs(parser=self, startPos=startPos)
 2436         else:
 2437             self.getWhiteSpace()
 2438             args = self.getExpression(useNameMapper=False,
 2439                                       pyTokensToBreakAt=[':']).strip()
 2440 
 2441         if self.matchColonForSingleLineShortFormDirective():
 2442             isShortForm = True
 2443             self.advance()  # skip over :
 2444             self.getWhiteSpace(max=1)
 2445             srcBlock = self.readToEOL(gobble=False)
 2446             EOLCharsInShortForm = self.readToEOL(gobble=True)
 2447             # self.readToEOL(gobble=False)
 2448         else:
 2449             isShortForm = False
 2450             if self.peek() == ':':
 2451                 self.advance()
 2452             self.getWhiteSpace()
 2453             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2454                                         endOfFirstLinePos)
 2455             srcBlock = self._eatToThisEndDirective(macroName)
 2456 
 2457         if hasattr(macro, 'convertArgStrToDict'):
 2458             kwArgs = macro.convertArgStrToDict(args, parser=self,
 2459                                                startPos=startPos)
 2460         else:
 2461             def getArgs(*pargs, **kws):
 2462                 return kws
 2463             kwArgs = eval('getArgs(%(args)s)' % locals())
 2464 
 2465         assert 'src' not in kwArgs
 2466         kwArgs['src'] = srcBlock
 2467 
 2468         if isinstance(macro, types.MethodType):
 2469             co = macro.__func__.__code__
 2470         elif (hasattr(macro, '__call__')
 2471               and hasattr(macro.__call__, '__func__')):
 2472             co = macro.__call__.__func__.__code__
 2473         else:
 2474             co = macro.__code__
 2475         availableKwArgs = inspect.getargs(co)[0]
 2476 
 2477         if 'parser' in availableKwArgs:
 2478             kwArgs['parser'] = self
 2479         if 'macros' in availableKwArgs:
 2480             kwArgs['macros'] = self._macros
 2481         if 'compilerSettings' in availableKwArgs:
 2482             kwArgs['compilerSettings'] = self.settings()
 2483         if 'isShortForm' in availableKwArgs:
 2484             kwArgs['isShortForm'] = isShortForm
 2485         if isShortForm and 'EOLCharsInShortForm' in availableKwArgs:
 2486             kwArgs['EOLCharsInShortForm'] = EOLCharsInShortForm
 2487 
 2488         if 'startPos' in availableKwArgs:
 2489             kwArgs['startPos'] = startPos
 2490         if 'endPos' in availableKwArgs:
 2491             kwArgs['endPos'] = self.pos()
 2492 
 2493         srcFromMacroOutput = macro(**kwArgs)
 2494 
 2495         origParseSrc = self._src
 2496         origBreakPoint = self.breakPoint()
 2497         origPos = self.pos()
 2498         # add a comment to the output about the macro src that is being parsed
 2499         # or add a comment prefix to all the comments added by the compiler
 2500         self._src = srcFromMacroOutput
 2501         self.setPos(0)
 2502         self.setBreakPoint(len(srcFromMacroOutput))
 2503 
 2504         self.parse(assertEmptyStack=False)
 2505 
 2506         self._src = origParseSrc
 2507         self.setBreakPoint(origBreakPoint)
 2508         self.setPos(origPos)
 2509 
 2510         # self._compiler.addRawText('end')
 2511 
 2512     def eatCache(self):
 2513         isLineClearToStartToken = self.isLineClearToStartToken()
 2514         endOfFirstLinePos = self.findEOL()
 2515         lineCol = self.getRowCol()
 2516         self.getDirectiveStartToken()
 2517         self.advance(len('cache'))
 2518 
 2519         startPos = self.pos()
 2520         argList = self.getDefArgList(useNameMapper=True)
 2521         argList = self._applyExpressionFilters(argList, 'cache',
 2522                                                startPos=startPos)
 2523 
 2524         def startCache():
 2525             cacheInfo = self._compiler.genCacheInfoFromArgList(argList)
 2526             self._compiler.startCacheRegion(cacheInfo, lineCol)
 2527 
 2528         if self.matchColonForSingleLineShortFormDirective():
 2529             self.advance()  # skip over :
 2530             self.getWhiteSpace(max=1)
 2531             startCache()
 2532             self.parse(breakPoint=self.findEOL(gobble=True))
 2533             self._compiler.endCacheRegion()
 2534         else:
 2535             if self.peek() == ':':
 2536                 self.advance()
 2537             self.getWhiteSpace()
 2538             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2539                                         endOfFirstLinePos)
 2540             self.pushToOpenDirectivesStack('cache')
 2541             startCache()
 2542 
 2543     def eatCall(self):
 2544         # @@TR: need to enable single line version of this
 2545         isLineClearToStartToken = self.isLineClearToStartToken()
 2546         endOfFirstLinePos = self.findEOL()
 2547         lineCol = self.getRowCol()
 2548         self.getDirectiveStartToken()
 2549         self.advance(len('call'))
 2550         startPos = self.pos()
 2551 
 2552         useAutocallingOrig = self.setting('useAutocalling')
 2553         self.setSetting('useAutocalling', False)
 2554         self.getWhiteSpace()
 2555         if self.matchCheetahVarStart():
 2556             functionName = self.getCheetahVar()
 2557         else:
 2558             functionName = self.getCheetahVar(plain=True, skipStartToken=True)
 2559         self.setSetting('useAutocalling', useAutocallingOrig)
 2560         # @@TR: fix up filtering
 2561         self._applyExpressionFilters(self[startPos:self.pos()], 'call',
 2562                                      startPos=startPos)
 2563 
 2564         self.getWhiteSpace()
 2565         args = self.getExpression(pyTokensToBreakAt=[':']).strip()
 2566         if self.matchColonForSingleLineShortFormDirective():
 2567             self.advance()  # skip over :
 2568             self._compiler.startCallRegion(functionName, args, lineCol)
 2569             self.getWhiteSpace(max=1)
 2570             self.parse(breakPoint=self.findEOL(gobble=False))
 2571             self._compiler.endCallRegion()
 2572         else:
 2573             if self.peek() == ':':
 2574                 self.advance()
 2575             self.getWhiteSpace()
 2576             self.pushToOpenDirectivesStack("call")
 2577             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2578                                         endOfFirstLinePos)
 2579             self._compiler.startCallRegion(functionName, args, lineCol)
 2580 
 2581     def eatCallArg(self):
 2582         isLineClearToStartToken = self.isLineClearToStartToken()
 2583         endOfFirstLinePos = self.findEOL()
 2584         lineCol = self.getRowCol()
 2585         self.getDirectiveStartToken()
 2586 
 2587         self.advance(len('arg'))
 2588         startPos = self.pos()
 2589         self.getWhiteSpace()
 2590         argName = self.getIdentifier()
 2591         self.getWhiteSpace()
 2592         argName = self._applyExpressionFilters(argName, 'arg',
 2593                                                startPos=startPos)
 2594         self._compiler.setCallArg(argName, lineCol)
 2595         if self.peek() == ':':
 2596             self.getc()
 2597         else:
 2598             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2599                                         endOfFirstLinePos)
 2600 
 2601     def eatFilter(self):
 2602         isLineClearToStartToken = self.isLineClearToStartToken()
 2603         endOfFirstLinePos = self.findEOL()
 2604 
 2605         self.getDirectiveStartToken()
 2606         self.advance(len('filter'))
 2607         self.getWhiteSpace()
 2608         startPos = self.pos()
 2609         if self.matchCheetahVarStart():
 2610             isKlass = True
 2611             theFilter = self.getExpression(pyTokensToBreakAt=[':'])
 2612         else:
 2613             isKlass = False
 2614             theFilter = self.getIdentifier()
 2615             self.getWhiteSpace()
 2616         theFilter = self._applyExpressionFilters(theFilter, 'filter',
 2617                                                  startPos=startPos)
 2618 
 2619         if self.matchColonForSingleLineShortFormDirective():
 2620             self.advance()  # skip over :
 2621             self.getWhiteSpace(max=1)
 2622             self._compiler.setFilter(theFilter, isKlass)
 2623             self.parse(breakPoint=self.findEOL(gobble=False))
 2624             self._compiler.closeFilterBlock()
 2625         else:
 2626             if self.peek() == ':':
 2627                 self.advance()
 2628             self.getWhiteSpace()
 2629             self.pushToOpenDirectivesStack("filter")
 2630             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2631                                         endOfFirstLinePos)
 2632             self._compiler.setFilter(theFilter, isKlass)
 2633 
 2634     def eatTransform(self):
 2635         isLineClearToStartToken = self.isLineClearToStartToken()
 2636         endOfFirstLinePos = self.findEOL()
 2637 
 2638         self.getDirectiveStartToken()
 2639         self.advance(len('transform'))
 2640         self.getWhiteSpace()
 2641         startPos = self.pos()
 2642         if self.matchCheetahVarStart():
 2643             isKlass = True
 2644             transformer = self.getExpression(pyTokensToBreakAt=[':'])
 2645         else:
 2646             isKlass = False
 2647             transformer = self.getIdentifier()
 2648             self.getWhiteSpace()
 2649         transformer = self._applyExpressionFilters(transformer, 'transform',
 2650                                                    startPos=startPos)
 2651 
 2652         if self.peek() == ':':
 2653             self.advance()
 2654         self.getWhiteSpace()
 2655         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
 2656         self._compiler.setTransform(transformer, isKlass)
 2657 
 2658     def eatErrorCatcher(self):
 2659         isLineClearToStartToken = self.isLineClearToStartToken()
 2660         endOfFirstLinePos = self.findEOL()
 2661         self.getDirectiveStartToken()
 2662         self.advance(len('errorCatcher'))
 2663         self.getWhiteSpace()
 2664         startPos = self.pos()
 2665         errorCatcherName = self.getIdentifier()
 2666         errorCatcherName = self._applyExpressionFilters(
 2667             errorCatcherName, 'errorcatcher', startPos=startPos)
 2668         self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
 2669         self._compiler.setErrorCatcher(errorCatcherName)
 2670 
 2671     def eatCapture(self):
 2672         # @@TR:  this could be refactored
 2673         # to use the code in eatSimpleIndentingDirective
 2674         # filtered
 2675         isLineClearToStartToken = self.isLineClearToStartToken()
 2676         endOfFirstLinePos = self.findEOL()
 2677         lineCol = self.getRowCol()
 2678 
 2679         self.getDirectiveStartToken()
 2680         self.advance(len('capture'))
 2681         startPos = self.pos()
 2682         self.getWhiteSpace()
 2683 
 2684         expr = self.getExpression(pyTokensToBreakAt=[':'])
 2685         expr = self._applyExpressionFilters(expr, 'capture', startPos=startPos)
 2686         if self.matchColonForSingleLineShortFormDirective():
 2687             self.advance()  # skip over :
 2688             self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
 2689             self.getWhiteSpace(max=1)
 2690             self.parse(breakPoint=self.findEOL(gobble=False))
 2691             self._compiler.endCaptureRegion()
 2692         else:
 2693             if self.peek() == ':':
 2694                 self.advance()
 2695             self.getWhiteSpace()
 2696             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2697                                         endOfFirstLinePos)
 2698             self.pushToOpenDirectivesStack("capture")
 2699             self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
 2700 
 2701     def eatIf(self):
 2702         # filtered
 2703         isLineClearToStartToken = self.isLineClearToStartToken()
 2704         endOfFirstLine = self.findEOL()
 2705         lineCol = self.getRowCol()
 2706         self.getDirectiveStartToken()
 2707         startPos = self.pos()
 2708 
 2709         expressionParts = self.getExpressionParts(pyTokensToBreakAt=[':'])
 2710         expr = ''.join(expressionParts).strip()
 2711         expr = self._applyExpressionFilters(expr, 'if', startPos=startPos)
 2712 
 2713         isTernaryExpr = ('then' in expressionParts
 2714                          and 'else' in expressionParts)
 2715         if isTernaryExpr:
 2716             conditionExpr = []
 2717             trueExpr = []
 2718             falseExpr = []
 2719             currentExpr = conditionExpr
 2720             for part in expressionParts:
 2721                 if part.strip() == 'then':
 2722                     currentExpr = trueExpr
 2723                 elif part.strip() == 'else':
 2724                     currentExpr = falseExpr
 2725                 else:
 2726                     currentExpr.append(part)
 2727 
 2728             conditionExpr = ''.join(conditionExpr)
 2729             trueExpr = ''.join(trueExpr)
 2730             falseExpr = ''.join(falseExpr)
 2731             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2732                                         endOfFirstLine)
 2733             self._compiler.addTernaryExpr(conditionExpr, trueExpr,
 2734                                           falseExpr, lineCol=lineCol)
 2735         elif self.matchColonForSingleLineShortFormDirective():
 2736             self.advance()  # skip over :
 2737             self._compiler.addIf(expr, lineCol=lineCol)
 2738             self.getWhiteSpace(max=1)
 2739             self.parse(breakPoint=self.findEOL(gobble=True))
 2740             self._compiler.commitStrConst()
 2741             self._compiler.dedent()
 2742         else:
 2743             if self.peek() == ':':
 2744                 self.advance()
 2745             self.getWhiteSpace()
 2746             self._eatRestOfDirectiveTag(isLineClearToStartToken,
 2747                                         endOfFirstLine)
 2748             self.pushToOpenDirectivesStack('if')
 2749             self._compiler.addIf(expr, lineCol=lineCol)
 2750 
 2751     # end directive handlers
 2752     def handleEndDef(self):
 2753         isNestedDef = (self.setting('allowNestedDefScopes')
 2754                        and [name for name in self._openDirectivesStack
 2755                             if name == 'def'])
 2756         if not isNestedDef:
 2757             self._compiler.closeDef()
 2758         else:
 2759             # @@TR: temporary hack of useSearchList
 2760             self.setSetting('useSearchList', self._useSearchList_orig)
 2761             self._compiler.commitStrConst()
 2762             self._compiler.dedent()
 2763     ###
 2764 
 2765     def pushToOpenDirectivesStack(self, directiveName):
 2766         assert directiveName in self._closeableDirectives
 2767         self._openDirectivesStack.append(directiveName)
 2768 
 2769     def popFromOpenDirectivesStack(self, directiveName):
 2770         if not self._openDirectivesStack:
 2771             raise ParseError(self, msg="#end found, but nothing to end")
 2772 
 2773         if self._openDirectivesStack[-1] == directiveName:
 2774             del self._openDirectivesStack[-1]
 2775         else:
 2776             raise ParseError(self, msg="#end %s found, expected #end %s" % (
 2777                 directiveName, self._openDirectivesStack[-1]))
 2778 
 2779     def assertEmptyOpenDirectivesStack(self):
 2780         if self._openDirectivesStack:
 2781             errorMsg = (
 2782                 "Some #directives are missing their corresponding #end ___ tag"
 2783                 ": %s" % ', '.join(self._openDirectivesStack))
 2784             raise ParseError(self, msg=errorMsg)
 2785 
 2786 
 2787 ##################################################
 2788 # Make an alias to export
 2789 Parser = _HighLevelParser