"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "tpl/internal/go_templates/texttemplate/parse/lex.go" between
hugo-0.80.0.tar.gz and hugo-0.81.0.tar.gz

About: Hugo is a static site generator that takes a source directory of Markdown files and templates and uses these as input to create a complete website (written in Go).

lex.go  (hugo-0.80.0):lex.go  (hugo-0.81.0)
skipping to change at line 44 skipping to change at line 44
} }
// itemType identifies the type of lex items. // itemType identifies the type of lex items.
type itemType int type itemType int
const ( const (
itemError itemType = iota // error occurred; value is text of erro r itemError itemType = iota // error occurred; value is text of erro r
itemBool // boolean constant itemBool // boolean constant
itemChar // printable ASCII character; grab bag f or comma etc. itemChar // printable ASCII character; grab bag f or comma etc.
itemCharConstant // character constant itemCharConstant // character constant
itemComment // comment text
itemComplex // complex constant (1+2i); imaginary is just a number itemComplex // complex constant (1+2i); imaginary is just a number
itemAssign // equals ('=') introducing an assignmen t itemAssign // equals ('=') introducing an assignmen t
itemDeclare // colon-equals (':=') introducing a dec laration itemDeclare // colon-equals (':=') introducing a dec laration
itemEOF itemEOF
itemField // alphanumeric identifier starting with '.' itemField // alphanumeric identifier starting with '.'
itemIdentifier // alphanumeric identifier not starting with '.' itemIdentifier // alphanumeric identifier not starting with '.'
itemLeftDelim // left action delimiter itemLeftDelim // left action delimiter
itemLeftParen // '(' inside action itemLeftParen // '(' inside action
itemNumber // simple number, including imaginary itemNumber // simple number, including imaginary
itemPipe // pipe symbol itemPipe // pipe symbol
skipping to change at line 94 skipping to change at line 95
"template": itemTemplate, "template": itemTemplate,
"with": itemWith, "with": itemWith,
} }
const eof = -1 const eof = -1
// Trimming spaces. // Trimming spaces.
// If the action begins "{{- " rather than "{{", then all space/tab/newlines // If the action begins "{{- " rather than "{{", then all space/tab/newlines
// preceding the action are trimmed; conversely if it ends " -}}" the // preceding the action are trimmed; conversely if it ends " -}}" the
// leading spaces are trimmed. This is done entirely in the lexer; the // leading spaces are trimmed. This is done entirely in the lexer; the
// parser never sees it happen. We require an ASCII space to be // parser never sees it happen. We require an ASCII space (' ', \t, \r, \n)
// present to avoid ambiguity with things like "{{-3}}". It reads // to be present to avoid ambiguity with things like "{{-3}}". It reads
// better with the space present anyway. For simplicity, only ASCII // better with the space present anyway. For simplicity, only ASCII
// space does the job. // does the job.
const ( const (
spaceChars = " \t\r\n" // These are the space characters defined by spaceChars = " \t\r\n" // These are the space characters defined by G
Go itself. o itself.
leftTrimMarker = "- " // Attached to left delimiter, trims trailing trimMarker = '-' // Attached to left/right delimiter, trims tra
spaces from preceding text. iling spaces from preceding/following text.
rightTrimMarker = " -" // Attached to right delimiter, trims leading trimMarkerLen = Pos(1 + 1) // marker plus space before or after
spaces from following text.
trimMarkerLen = Pos(len(leftTrimMarker))
) )
// stateFn represents the state of the scanner as a function that returns the ne xt state. // stateFn represents the state of the scanner as a function that returns the ne xt state.
type stateFn func(*lexer) stateFn type stateFn func(*lexer) stateFn
// lexer holds the state of the scanner. // lexer holds the state of the scanner.
type lexer struct { type lexer struct {
name string // the name of the input; used only for error re name string // the name of the input; used only for error repor
ports ts
input string // the string being scanned input string // the string being scanned
leftDelim string // start of action leftDelim string // start of action
rightDelim string // end of action rightDelim string // end of action
trimRightDelim string // end of action with trim marker emitComment bool // emit itemComment tokens.
pos Pos // current position in the input pos Pos // current position in the input
start Pos // start position of this item start Pos // start position of this item
width Pos // width of last rune read from input width Pos // width of last rune read from input
items chan item // channel of scanned items items chan item // channel of scanned items
parenDepth int // nesting depth of ( ) exprs parenDepth int // nesting depth of ( ) exprs
line int // 1+number of newlines seen line int // 1+number of newlines seen
startLine int // start line of this item startLine int // start line of this item
} }
// next returns the next rune in the input. // next returns the next rune in the input.
func (l *lexer) next() rune { func (l *lexer) next() rune {
if int(l.pos) >= len(l.input) { if int(l.pos) >= len(l.input) {
l.width = 0 l.width = 0
return eof return eof
} }
r, w := utf8.DecodeRuneInString(l.input[l.pos:]) r, w := utf8.DecodeRuneInString(l.input[l.pos:])
l.width = Pos(w) l.width = Pos(w)
skipping to change at line 206 skipping to change at line 206
} }
// drain drains the output so the lexing goroutine will exit. // drain drains the output so the lexing goroutine will exit.
// Called by the parser, not in the lexing goroutine. // Called by the parser, not in the lexing goroutine.
func (l *lexer) drain() { func (l *lexer) drain() {
for range l.items { for range l.items {
} }
} }
// lex creates a new scanner for the input string. // lex creates a new scanner for the input string.
func lex(name, input, left, right string) *lexer { func lex(name, input, left, right string, emitComment bool) *lexer {
if left == "" { if left == "" {
left = leftDelim left = leftDelim
} }
if right == "" { if right == "" {
right = rightDelim right = rightDelim
} }
l := &lexer{ l := &lexer{
name: name, name: name,
input: input, input: input,
leftDelim: left, leftDelim: left,
rightDelim: right, rightDelim: right,
trimRightDelim: rightTrimMarker + right, emitComment: emitComment,
items: make(chan item), items: make(chan item),
line: 1, line: 1,
startLine: 1, startLine: 1,
} }
go l.run() go l.run()
return l return l
} }
// run runs the state machine for the lexer. // run runs the state machine for the lexer.
func (l *lexer) run() { func (l *lexer) run() {
for state := lexText; state != nil; { for state := lexText; state != nil; {
state = state(l) state = state(l)
} }
skipping to change at line 251 skipping to change at line 251
rightComment = "*/" rightComment = "*/"
) )
// lexText scans until an opening action delimiter, "{{". // lexText scans until an opening action delimiter, "{{".
func lexText(l *lexer) stateFn { func lexText(l *lexer) stateFn {
l.width = 0 l.width = 0
if x := strings.Index(l.input[l.pos:], l.leftDelim); x >= 0 { if x := strings.Index(l.input[l.pos:], l.leftDelim); x >= 0 {
ldn := Pos(len(l.leftDelim)) ldn := Pos(len(l.leftDelim))
l.pos += Pos(x) l.pos += Pos(x)
trimLength := Pos(0) trimLength := Pos(0)
if strings.HasPrefix(l.input[l.pos+ldn:], leftTrimMarker) { if hasLeftTrimMarker(l.input[l.pos+ldn:]) {
trimLength = rightTrimLength(l.input[l.start:l.pos]) trimLength = rightTrimLength(l.input[l.start:l.pos])
} }
l.pos -= trimLength l.pos -= trimLength
if l.pos > l.start { if l.pos > l.start {
l.line += strings.Count(l.input[l.start:l.pos], "\n") l.line += strings.Count(l.input[l.start:l.pos], "\n")
l.emit(itemText) l.emit(itemText)
} }
l.pos += trimLength l.pos += trimLength
l.ignore() l.ignore()
return lexLeftDelim return lexLeftDelim
skipping to change at line 280 skipping to change at line 280
return nil return nil
} }
// rightTrimLength returns the length of the spaces at the end of the string. // rightTrimLength returns the length of the spaces at the end of the string.
func rightTrimLength(s string) Pos { func rightTrimLength(s string) Pos {
return Pos(len(s) - len(strings.TrimRight(s, spaceChars))) return Pos(len(s) - len(strings.TrimRight(s, spaceChars)))
} }
// atRightDelim reports whether the lexer is at a right delimiter, possibly prec eded by a trim marker. // atRightDelim reports whether the lexer is at a right delimiter, possibly prec eded by a trim marker.
func (l *lexer) atRightDelim() (delim, trimSpaces bool) { func (l *lexer) atRightDelim() (delim, trimSpaces bool) {
if strings.HasPrefix(l.input[l.pos:], l.trimRightDelim) { // With trim ma rker. if hasRightTrimMarker(l.input[l.pos:]) && strings.HasPrefix(l.input[l.pos +trimMarkerLen:], l.rightDelim) { // With trim marker.
return true, true return true, true
} }
if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { // Without trim mar ker. if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { // Without trim mar ker.
return true, false return true, false
} }
return false, false return false, false
} }
// leftTrimLength returns the length of the spaces at the beginning of the strin g. // leftTrimLength returns the length of the spaces at the beginning of the strin g.
func leftTrimLength(s string) Pos { func leftTrimLength(s string) Pos {
return Pos(len(s) - len(strings.TrimLeft(s, spaceChars))) return Pos(len(s) - len(strings.TrimLeft(s, spaceChars)))
} }
// lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker. // lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker.
func lexLeftDelim(l *lexer) stateFn { func lexLeftDelim(l *lexer) stateFn {
l.pos += Pos(len(l.leftDelim)) l.pos += Pos(len(l.leftDelim))
trimSpace := strings.HasPrefix(l.input[l.pos:], leftTrimMarker) trimSpace := hasLeftTrimMarker(l.input[l.pos:])
afterMarker := Pos(0) afterMarker := Pos(0)
if trimSpace { if trimSpace {
afterMarker = trimMarkerLen afterMarker = trimMarkerLen
} }
if strings.HasPrefix(l.input[l.pos+afterMarker:], leftComment) { if strings.HasPrefix(l.input[l.pos+afterMarker:], leftComment) {
l.pos += afterMarker l.pos += afterMarker
l.ignore() l.ignore()
return lexComment return lexComment
} }
l.emit(itemLeftDelim) l.emit(itemLeftDelim)
skipping to change at line 326 skipping to change at line 326
l.pos += Pos(len(leftComment)) l.pos += Pos(len(leftComment))
i := strings.Index(l.input[l.pos:], rightComment) i := strings.Index(l.input[l.pos:], rightComment)
if i < 0 { if i < 0 {
return l.errorf("unclosed comment") return l.errorf("unclosed comment")
} }
l.pos += Pos(i + len(rightComment)) l.pos += Pos(i + len(rightComment))
delim, trimSpace := l.atRightDelim() delim, trimSpace := l.atRightDelim()
if !delim { if !delim {
return l.errorf("comment ends before closing delimiter") return l.errorf("comment ends before closing delimiter")
} }
if l.emitComment {
l.emit(itemComment)
}
if trimSpace { if trimSpace {
l.pos += trimMarkerLen l.pos += trimMarkerLen
} }
l.pos += Pos(len(l.rightDelim)) l.pos += Pos(len(l.rightDelim))
if trimSpace { if trimSpace {
l.pos += leftTrimLength(l.input[l.pos:]) l.pos += leftTrimLength(l.input[l.pos:])
} }
l.ignore() l.ignore()
return lexText return lexText
} }
// lexRightDelim scans the right delimiter, which is known to be present, possib ly with a trim marker. // lexRightDelim scans the right delimiter, which is known to be present, possib ly with a trim marker.
func lexRightDelim(l *lexer) stateFn { func lexRightDelim(l *lexer) stateFn {
trimSpace := strings.HasPrefix(l.input[l.pos:], rightTrimMarker) trimSpace := hasRightTrimMarker(l.input[l.pos:])
if trimSpace { if trimSpace {
l.pos += trimMarkerLen l.pos += trimMarkerLen
l.ignore() l.ignore()
} }
l.pos += Pos(len(l.rightDelim)) l.pos += Pos(len(l.rightDelim))
l.emit(itemRightDelim) l.emit(itemRightDelim)
if trimSpace { if trimSpace {
l.pos += leftTrimLength(l.input[l.pos:]) l.pos += leftTrimLength(l.input[l.pos:])
l.ignore() l.ignore()
} }
skipping to change at line 366 skipping to change at line 369
// Spaces separate arguments; runs of spaces turn into itemSpace. // Spaces separate arguments; runs of spaces turn into itemSpace.
// Pipe symbols separate and are emitted. // Pipe symbols separate and are emitted.
delim, _ := l.atRightDelim() delim, _ := l.atRightDelim()
if delim { if delim {
if l.parenDepth == 0 { if l.parenDepth == 0 {
return lexRightDelim return lexRightDelim
} }
return l.errorf("unclosed left paren") return l.errorf("unclosed left paren")
} }
switch r := l.next(); { switch r := l.next(); {
case r == eof || isEndOfLine(r): case r == eof:
return l.errorf("unclosed action") return l.errorf("unclosed action")
case isSpace(r): case isSpace(r):
l.backup() // Put space back in case we have " -}}". l.backup() // Put space back in case we have " -}}".
return lexSpace return lexSpace
case r == '=': case r == '=':
l.emit(itemAssign) l.emit(itemAssign)
case r == ':': case r == ':':
if l.next() != '=' { if l.next() != '=' {
return l.errorf("expected :=") return l.errorf("expected :=")
} }
skipping to change at line 436 skipping to change at line 439
for { for {
r = l.peek() r = l.peek()
if !isSpace(r) { if !isSpace(r) {
break break
} }
l.next() l.next()
numSpaces++ numSpaces++
} }
// Be careful about a trim-marked closing delimiter, which has a minus // Be careful about a trim-marked closing delimiter, which has a minus
// after a space. We know there is a space, so check for the '-' that mig ht follow. // after a space. We know there is a space, so check for the '-' that mig ht follow.
if strings.HasPrefix(l.input[l.pos-1:], l.trimRightDelim) { if hasRightTrimMarker(l.input[l.pos-1:]) && strings.HasPrefix(l.input[l.p os-1+trimMarkerLen:], l.rightDelim) {
l.backup() // Before the space. l.backup() // Before the space.
if numSpaces == 1 { if numSpaces == 1 {
return lexRightDelim // On the delim, so go right to that . return lexRightDelim // On the delim, so go right to that .
} }
} }
l.emit(itemSpace) l.emit(itemSpace)
return lexInsideAction return lexInsideAction
} }
// lexIdentifier scans an alphanumeric. // lexIdentifier scans an alphanumeric.
skipping to change at line 523 skipping to change at line 526
l.emit(typ) l.emit(typ)
return lexInsideAction return lexInsideAction
} }
// atTerminator reports whether the input is at valid termination character to // atTerminator reports whether the input is at valid termination character to
// appear after an identifier. Breaks .X.Y into two pieces. Also catches cases // appear after an identifier. Breaks .X.Y into two pieces. Also catches cases
// like "$x+2" not being acceptable without a space, in case we decide one // like "$x+2" not being acceptable without a space, in case we decide one
// day to implement arithmetic. // day to implement arithmetic.
func (l *lexer) atTerminator() bool { func (l *lexer) atTerminator() bool {
r := l.peek() r := l.peek()
if isSpace(r) || isEndOfLine(r) { if isSpace(r) {
return true return true
} }
switch r { switch r {
case eof, '.', ',', '|', ':', ')', '(': case eof, '.', ',', '|', ':', ')', '(':
return true return true
} }
// Does r start the delimiter? This can be ambiguous (with delim=="//", $ x/2 will // Does r start the delimiter? This can be ambiguous (with delim=="//", $ x/2 will
// succeed but should fail) but only in extremely rare cases caused by wi llfully // succeed but should fail) but only in extremely rare cases caused by wi llfully
// bad choice of delimiter. // bad choice of delimiter.
if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r { if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r {
skipping to change at line 654 skipping to change at line 657
case '`': case '`':
break Loop break Loop
} }
} }
l.emit(itemRawString) l.emit(itemRawString)
return lexInsideAction return lexInsideAction
} }
// isSpace reports whether r is a space character. // isSpace reports whether r is a space character.
func isSpace(r rune) bool { func isSpace(r rune) bool {
return r == ' ' || r == '\t' return r == ' ' || r == '\t' || r == '\r' || r == '\n'
}
// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
return r == '\r' || r == '\n'
} }
// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore. // isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
func isAlphaNumeric(r rune) bool { func isAlphaNumeric(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
} }
func hasLeftTrimMarker(s string) bool {
return len(s) >= 2 && s[0] == trimMarker && isSpace(rune(s[1]))
}
func hasRightTrimMarker(s string) bool {
return len(s) >= 2 && isSpace(rune(s[0])) && s[1] == trimMarker
}
 End of changes. 17 change blocks. 
45 lines changed or deleted 42 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)