"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "tpl/internal/go_templates/texttemplate/parse/lex.go" between
hugo-0.80.0.tar.gz and hugo-0.81.0.tar.gz

About: Hugo is a static site generator that takes a source directory of Markdown files and templates and uses these as input to create a complete website (written in Go).

lex.go  (hugo-0.80.0)	:	lex.go  (hugo-0.81.0)
skipping to change at line 44		skipping to change at line 44
}		}
// itemType identifies the type of lex items.		// itemType identifies the type of lex items.
type itemType int		type itemType int
const (		const (
itemError itemType = iota // error occurred; value is text of erro r		itemError itemType = iota // error occurred; value is text of erro r
itemBool // boolean constant		itemBool // boolean constant
itemChar // printable ASCII character; grab bag f or comma etc.		itemChar // printable ASCII character; grab bag f or comma etc.
itemCharConstant // character constant		itemCharConstant // character constant
		itemComment // comment text
itemComplex // complex constant (1+2i); imaginary is just a number		itemComplex // complex constant (1+2i); imaginary is just a number
itemAssign // equals ('=') introducing an assignmen t		itemAssign // equals ('=') introducing an assignmen t
itemDeclare // colon-equals (':=') introducing a dec laration		itemDeclare // colon-equals (':=') introducing a dec laration
itemEOF		itemEOF
itemField // alphanumeric identifier starting with '.'		itemField // alphanumeric identifier starting with '.'
itemIdentifier // alphanumeric identifier not starting with '.'		itemIdentifier // alphanumeric identifier not starting with '.'
itemLeftDelim // left action delimiter		itemLeftDelim // left action delimiter
itemLeftParen // '(' inside action		itemLeftParen // '(' inside action
itemNumber // simple number, including imaginary		itemNumber // simple number, including imaginary
itemPipe // pipe symbol		itemPipe // pipe symbol
skipping to change at line 94		skipping to change at line 95
"template": itemTemplate,		"template": itemTemplate,
"with": itemWith,		"with": itemWith,
}		}
const eof = -1		const eof = -1
// Trimming spaces.		// Trimming spaces.
// If the action begins "{{- " rather than "{{", then all space/tab/newlines		// If the action begins "{{- " rather than "{{", then all space/tab/newlines
// preceding the action are trimmed; conversely if it ends " -}}" the		// preceding the action are trimmed; conversely if it ends " -}}" the
// leading spaces are trimmed. This is done entirely in the lexer; the		// leading spaces are trimmed. This is done entirely in the lexer; the
// parser never sees it happen. We require an ASCII space to be		// parser never sees it happen. We require an ASCII space (' ', \t, \r, \n)
// present to avoid ambiguity with things like "{{-3}}". It reads		// to be present to avoid ambiguity with things like "{{-3}}". It reads
// better with the space present anyway. For simplicity, only ASCII		// better with the space present anyway. For simplicity, only ASCII
// space does the job.		// does the job.
const (		const (
spaceChars = " \t\r\n" // These are the space characters defined by		spaceChars = " \t\r\n" // These are the space characters defined by G
Go itself.		o itself.
leftTrimMarker = "- " // Attached to left delimiter, trims trailing		trimMarker = '-' // Attached to left/right delimiter, trims tra
spaces from preceding text.		iling spaces from preceding/following text.
rightTrimMarker = " -" // Attached to right delimiter, trims leading		trimMarkerLen = Pos(1 + 1) // marker plus space before or after
spaces from following text.
trimMarkerLen = Pos(len(leftTrimMarker))
)		)
// stateFn represents the state of the scanner as a function that returns the ne xt state.		// stateFn represents the state of the scanner as a function that returns the ne xt state.
type stateFn func(*lexer) stateFn		type stateFn func(*lexer) stateFn
// lexer holds the state of the scanner.		// lexer holds the state of the scanner.
type lexer struct {		type lexer struct {
name string // the name of the input; used only for error re		name string // the name of the input; used only for error repor
ports		ts
input string // the string being scanned		input string // the string being scanned
leftDelim string // start of action		leftDelim string // start of action
rightDelim string // end of action		rightDelim string // end of action
trimRightDelim string // end of action with trim marker		emitComment bool // emit itemComment tokens.
pos Pos // current position in the input		pos Pos // current position in the input
start Pos // start position of this item		start Pos // start position of this item
width Pos // width of last rune read from input		width Pos // width of last rune read from input
items chan item // channel of scanned items		items chan item // channel of scanned items
parenDepth int // nesting depth of ( ) exprs		parenDepth int // nesting depth of ( ) exprs
line int // 1+number of newlines seen		line int // 1+number of newlines seen
startLine int // start line of this item		startLine int // start line of this item
}		}
// next returns the next rune in the input.		// next returns the next rune in the input.
func (l *lexer) next() rune {		func (l *lexer) next() rune {
if int(l.pos) >= len(l.input) {		if int(l.pos) >= len(l.input) {
l.width = 0		l.width = 0
return eof		return eof
}		}
r, w := utf8.DecodeRuneInString(l.input[l.pos:])		r, w := utf8.DecodeRuneInString(l.input[l.pos:])
l.width = Pos(w)		l.width = Pos(w)
skipping to change at line 206		skipping to change at line 206
}		}
// drain drains the output so the lexing goroutine will exit.		// drain drains the output so the lexing goroutine will exit.
// Called by the parser, not in the lexing goroutine.		// Called by the parser, not in the lexing goroutine.
func (l *lexer) drain() {		func (l *lexer) drain() {
for range l.items {		for range l.items {
}		}
}		}
// lex creates a new scanner for the input string.		// lex creates a new scanner for the input string.
func lex(name, input, left, right string) *lexer {		func lex(name, input, left, right string, emitComment bool) *lexer {
if left == "" {		if left == "" {
left = leftDelim		left = leftDelim
}		}
if right == "" {		if right == "" {
right = rightDelim		right = rightDelim
}		}
l := &lexer{		l := &lexer{
name: name,		name: name,
input: input,		input: input,
leftDelim: left,		leftDelim: left,
rightDelim: right,		rightDelim: right,
trimRightDelim: rightTrimMarker + right,		emitComment: emitComment,
items: make(chan item),		items: make(chan item),
line: 1,		line: 1,
startLine: 1,		startLine: 1,
}		}
go l.run()		go l.run()
return l		return l
}		}
// run runs the state machine for the lexer.		// run runs the state machine for the lexer.
func (l *lexer) run() {		func (l *lexer) run() {
for state := lexText; state != nil; {		for state := lexText; state != nil; {
state = state(l)		state = state(l)
}		}
skipping to change at line 251		skipping to change at line 251
rightComment = "*/"		rightComment = "*/"
)		)
// lexText scans until an opening action delimiter, "{{".		// lexText scans until an opening action delimiter, "{{".
func lexText(l *lexer) stateFn {		func lexText(l *lexer) stateFn {
l.width = 0		l.width = 0
if x := strings.Index(l.input[l.pos:], l.leftDelim); x >= 0 {		if x := strings.Index(l.input[l.pos:], l.leftDelim); x >= 0 {
ldn := Pos(len(l.leftDelim))		ldn := Pos(len(l.leftDelim))
l.pos += Pos(x)		l.pos += Pos(x)
trimLength := Pos(0)		trimLength := Pos(0)
if strings.HasPrefix(l.input[l.pos+ldn:], leftTrimMarker) {		if hasLeftTrimMarker(l.input[l.pos+ldn:]) {
trimLength = rightTrimLength(l.input[l.start:l.pos])		trimLength = rightTrimLength(l.input[l.start:l.pos])
}		}
l.pos -= trimLength		l.pos -= trimLength
if l.pos > l.start {		if l.pos > l.start {
l.line += strings.Count(l.input[l.start:l.pos], "\n")		l.line += strings.Count(l.input[l.start:l.pos], "\n")
l.emit(itemText)		l.emit(itemText)
}		}
l.pos += trimLength		l.pos += trimLength
l.ignore()		l.ignore()
return lexLeftDelim		return lexLeftDelim
skipping to change at line 280		skipping to change at line 280
return nil		return nil
}		}
// rightTrimLength returns the length of the spaces at the end of the string.		// rightTrimLength returns the length of the spaces at the end of the string.
func rightTrimLength(s string) Pos {		func rightTrimLength(s string) Pos {
return Pos(len(s) - len(strings.TrimRight(s, spaceChars)))		return Pos(len(s) - len(strings.TrimRight(s, spaceChars)))
}		}
// atRightDelim reports whether the lexer is at a right delimiter, possibly prec eded by a trim marker.		// atRightDelim reports whether the lexer is at a right delimiter, possibly prec eded by a trim marker.
func (l *lexer) atRightDelim() (delim, trimSpaces bool) {		func (l *lexer) atRightDelim() (delim, trimSpaces bool) {
if strings.HasPrefix(l.input[l.pos:], l.trimRightDelim) { // With trim ma rker.		if hasRightTrimMarker(l.input[l.pos:]) && strings.HasPrefix(l.input[l.pos +trimMarkerLen:], l.rightDelim) { // With trim marker.
return true, true		return true, true
}		}
if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { // Without trim mar ker.		if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { // Without trim mar ker.
return true, false		return true, false
}		}
return false, false		return false, false
}		}
// leftTrimLength returns the length of the spaces at the beginning of the strin g.		// leftTrimLength returns the length of the spaces at the beginning of the strin g.
func leftTrimLength(s string) Pos {		func leftTrimLength(s string) Pos {
return Pos(len(s) - len(strings.TrimLeft(s, spaceChars)))		return Pos(len(s) - len(strings.TrimLeft(s, spaceChars)))
}		}
// lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker.		// lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker.
func lexLeftDelim(l *lexer) stateFn {		func lexLeftDelim(l *lexer) stateFn {
l.pos += Pos(len(l.leftDelim))		l.pos += Pos(len(l.leftDelim))
trimSpace := strings.HasPrefix(l.input[l.pos:], leftTrimMarker)		trimSpace := hasLeftTrimMarker(l.input[l.pos:])
afterMarker := Pos(0)		afterMarker := Pos(0)
if trimSpace {		if trimSpace {
afterMarker = trimMarkerLen		afterMarker = trimMarkerLen
}		}
if strings.HasPrefix(l.input[l.pos+afterMarker:], leftComment) {		if strings.HasPrefix(l.input[l.pos+afterMarker:], leftComment) {
l.pos += afterMarker		l.pos += afterMarker
l.ignore()		l.ignore()
return lexComment		return lexComment
}		}
l.emit(itemLeftDelim)		l.emit(itemLeftDelim)
skipping to change at line 326		skipping to change at line 326
l.pos += Pos(len(leftComment))		l.pos += Pos(len(leftComment))
i := strings.Index(l.input[l.pos:], rightComment)		i := strings.Index(l.input[l.pos:], rightComment)
if i < 0 {		if i < 0 {
return l.errorf("unclosed comment")		return l.errorf("unclosed comment")
}		}
l.pos += Pos(i + len(rightComment))		l.pos += Pos(i + len(rightComment))
delim, trimSpace := l.atRightDelim()		delim, trimSpace := l.atRightDelim()
if !delim {		if !delim {
return l.errorf("comment ends before closing delimiter")		return l.errorf("comment ends before closing delimiter")
}		}
		if l.emitComment {
		l.emit(itemComment)
		}
if trimSpace {		if trimSpace {
l.pos += trimMarkerLen		l.pos += trimMarkerLen
}		}
l.pos += Pos(len(l.rightDelim))		l.pos += Pos(len(l.rightDelim))
if trimSpace {		if trimSpace {
l.pos += leftTrimLength(l.input[l.pos:])		l.pos += leftTrimLength(l.input[l.pos:])
}		}
l.ignore()		l.ignore()
return lexText		return lexText
}		}
// lexRightDelim scans the right delimiter, which is known to be present, possib ly with a trim marker.		// lexRightDelim scans the right delimiter, which is known to be present, possib ly with a trim marker.
func lexRightDelim(l *lexer) stateFn {		func lexRightDelim(l *lexer) stateFn {
trimSpace := strings.HasPrefix(l.input[l.pos:], rightTrimMarker)		trimSpace := hasRightTrimMarker(l.input[l.pos:])
if trimSpace {		if trimSpace {
l.pos += trimMarkerLen		l.pos += trimMarkerLen
l.ignore()		l.ignore()
}		}
l.pos += Pos(len(l.rightDelim))		l.pos += Pos(len(l.rightDelim))
l.emit(itemRightDelim)		l.emit(itemRightDelim)
if trimSpace {		if trimSpace {
l.pos += leftTrimLength(l.input[l.pos:])		l.pos += leftTrimLength(l.input[l.pos:])
l.ignore()		l.ignore()
}		}
skipping to change at line 366		skipping to change at line 369
// Spaces separate arguments; runs of spaces turn into itemSpace.		// Spaces separate arguments; runs of spaces turn into itemSpace.
// Pipe symbols separate and are emitted.		// Pipe symbols separate and are emitted.
delim, _ := l.atRightDelim()		delim, _ := l.atRightDelim()
if delim {		if delim {
if l.parenDepth == 0 {		if l.parenDepth == 0 {
return lexRightDelim		return lexRightDelim
}		}
return l.errorf("unclosed left paren")		return l.errorf("unclosed left paren")
}		}
switch r := l.next(); {		switch r := l.next(); {
case r == eof || isEndOfLine(r):		case r == eof:
return l.errorf("unclosed action")		return l.errorf("unclosed action")
case isSpace(r):		case isSpace(r):
l.backup() // Put space back in case we have " -}}".		l.backup() // Put space back in case we have " -}}".
return lexSpace		return lexSpace
case r == '=':		case r == '=':
l.emit(itemAssign)		l.emit(itemAssign)
case r == ':':		case r == ':':
if l.next() != '=' {		if l.next() != '=' {
return l.errorf("expected :=")		return l.errorf("expected :=")
}		}
skipping to change at line 436		skipping to change at line 439
for {		for {
r = l.peek()		r = l.peek()
if !isSpace(r) {		if !isSpace(r) {
break		break
}		}
l.next()		l.next()
numSpaces++		numSpaces++
}		}
// Be careful about a trim-marked closing delimiter, which has a minus		// Be careful about a trim-marked closing delimiter, which has a minus
// after a space. We know there is a space, so check for the '-' that mig ht follow.		// after a space. We know there is a space, so check for the '-' that mig ht follow.
if strings.HasPrefix(l.input[l.pos-1:], l.trimRightDelim) {		if hasRightTrimMarker(l.input[l.pos-1:]) && strings.HasPrefix(l.input[l.p os-1+trimMarkerLen:], l.rightDelim) {
l.backup() // Before the space.		l.backup() // Before the space.
if numSpaces == 1 {		if numSpaces == 1 {
return lexRightDelim // On the delim, so go right to that .		return lexRightDelim // On the delim, so go right to that .
}		}
}		}
l.emit(itemSpace)		l.emit(itemSpace)
return lexInsideAction		return lexInsideAction
}		}
// lexIdentifier scans an alphanumeric.		// lexIdentifier scans an alphanumeric.
skipping to change at line 523		skipping to change at line 526
l.emit(typ)		l.emit(typ)
return lexInsideAction		return lexInsideAction
}		}
// atTerminator reports whether the input is at valid termination character to		// atTerminator reports whether the input is at valid termination character to
// appear after an identifier. Breaks .X.Y into two pieces. Also catches cases		// appear after an identifier. Breaks .X.Y into two pieces. Also catches cases
// like "$x+2" not being acceptable without a space, in case we decide one		// like "$x+2" not being acceptable without a space, in case we decide one
// day to implement arithmetic.		// day to implement arithmetic.
func (l *lexer) atTerminator() bool {		func (l *lexer) atTerminator() bool {
r := l.peek()		r := l.peek()
if isSpace(r) || isEndOfLine(r) {		if isSpace(r) {
return true		return true
}		}
switch r {		switch r {
case eof, '.', ',', '|', ':', ')', '(':		case eof, '.', ',', '|', ':', ')', '(':
return true		return true
}		}
// Does r start the delimiter? This can be ambiguous (with delim=="//", $ x/2 will		// Does r start the delimiter? This can be ambiguous (with delim=="//", $ x/2 will
// succeed but should fail) but only in extremely rare cases caused by wi llfully		// succeed but should fail) but only in extremely rare cases caused by wi llfully
// bad choice of delimiter.		// bad choice of delimiter.
if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r {		if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r {
skipping to change at line 654		skipping to change at line 657
case '`':		case '`':
break Loop		break Loop
}		}
}		}
l.emit(itemRawString)		l.emit(itemRawString)
return lexInsideAction		return lexInsideAction
}		}
// isSpace reports whether r is a space character.		// isSpace reports whether r is a space character.
func isSpace(r rune) bool {		func isSpace(r rune) bool {
return r == ' ' || r == '\t'		return r == ' ' || r == '\t' || r == '\r' || r == '\n'
}
// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
return r == '\r' || r == '\n'
}		}
// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.		// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
func isAlphaNumeric(r rune) bool {		func isAlphaNumeric(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)		return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}		}
		func hasLeftTrimMarker(s string) bool {
		return len(s) >= 2 && s[0] == trimMarker && isSpace(rune(s[1]))
		}
		func hasRightTrimMarker(s string) bool {
		return len(s) >= 2 && isSpace(rune(s[0])) && s[1] == trimMarker
		}
End of changes. 17 change blocks.
45 lines changed or deleted		42 lines changed or added

---