ref: 8388cd90e89358f73ddcb7f496a1a8dc5c30c36c
dir: /hugolib/shortcodeparser.go/
// Copyright 2015 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"fmt"
"strings"
"unicode"
"unicode/utf8"
)
// The lexical scanning below is highly inspired by the great talk given by
// Rob Pike called "Lexical Scanning in Go" (it's on YouTube, Google it!).
// See slides here: http://cuddle.googlecode.com/hg/talk/lex.html
// parsing
type pageTokens struct {
lexer *pagelexer
token [3]item // 3-item look-ahead is what we currently need
peekCount int
}
func (t *pageTokens) next() item {
if t.peekCount > 0 {
t.peekCount--
} else {
t.token[0] = t.lexer.nextItem()
}
return t.token[t.peekCount]
}
// backs up one token.
func (t *pageTokens) backup() {
t.peekCount++
}
// backs up two tokens.
func (t *pageTokens) backup2(t1 item) {
t.token[1] = t1
t.peekCount = 2
}
// backs up three tokens.
func (t *pageTokens) backup3(t2, t1 item) {
t.token[1] = t1
t.token[2] = t2
t.peekCount = 3
}
// check for non-error and non-EOF types coming next
func (t *pageTokens) isValueNext() bool {
i := t.peek()
return i.typ != tError && i.typ != tEOF
}
// look at, but do not consume, the next item
// repeated, sequential calls will return the same item
func (t *pageTokens) peek() item {
if t.peekCount > 0 {
return t.token[t.peekCount-1]
}
t.peekCount = 1
t.token[0] = t.lexer.nextItem()
return t.token[0]
}
// convencience method to consume the next n tokens, but back off Errors and EOF
func (t *pageTokens) consume(cnt int) {
for i := 0; i < cnt; i++ {
token := t.next()
if token.typ == tError || token.typ == tEOF {
t.backup()
break
}
}
}
// lexical scanning
// position (in bytes)
type pos int
type item struct {
typ itemType
pos pos
val string
}
func (i item) String() string {
switch {
case i.typ == tEOF:
return "EOF"
case i.typ == tError:
return i.val
case i.typ > tKeywordMarker:
return fmt.Sprintf("<%s>", i.val)
case len(i.val) > 20:
return fmt.Sprintf("%.20q...", i.val)
}
return fmt.Sprintf("[%s]", i.val)
}
type itemType int
const (
tError itemType = iota
tEOF
// shortcode items
tLeftDelimScNoMarkup
tRightDelimScNoMarkup
tLeftDelimScWithMarkup
tRightDelimScWithMarkup
tScClose
tScName
tScParam
tScParamVal
//itemIdentifier
tText // plain text, used for everything outside the shortcodes
// preserved for later - keywords come after this
tKeywordMarker
)
const eof = -1
// returns the next state in scanner.
type stateFunc func(*pagelexer) stateFunc
type pagelexer struct {
name string
input string
state stateFunc
pos pos // input position
start pos // item start position
width pos // width of last element
lastPos pos // position of the last item returned by nextItem
// shortcode state
currLeftDelimItem itemType
currRightDelimItem itemType
currShortcodeName string // is only set when a shortcode is in opened state
closingState int // > 0 = on its way to be closed
elementStepNum int // step number in element
paramElements int // number of elements (name + value = 2) found first
openShortcodes map[string]bool // set of shortcodes in open state
// items delivered to client
items []item
}
// note: the input position here is normally 0 (start), but
// can be set if position of first shortcode is known
func newShortcodeLexer(name, input string, inputPosition pos) *pagelexer {
lexer := &pagelexer{
name: name,
input: input,
currLeftDelimItem: tLeftDelimScNoMarkup,
currRightDelimItem: tRightDelimScNoMarkup,
pos: inputPosition,
openShortcodes: make(map[string]bool),
items: make([]item, 0, 5),
}
lexer.runShortcodeLexer()
return lexer
}
// main loop
// this looks kind of funky, but it works
func (l *pagelexer) runShortcodeLexer() {
for l.state = lexTextOutsideShortcodes; l.state != nil; {
l.state = l.state(l)
}
}
// state functions
const (
leftDelimScNoMarkup = "{{<"
rightDelimScNoMarkup = ">}}"
leftDelimScWithMarkup = "{{%"
rightDelimScWithMarkup = "%}}"
leftComment = "/*" // comments in this context us used to to mark shortcodes as "not really a shortcode"
rightComment = "*/"
)
func (l *pagelexer) next() rune {
if int(l.pos) >= len(l.input) {
l.width = 0
return eof
}
// looks expensive, but should produce the same iteration sequence as the string range loop
// see: http://blog.golang.org/strings
runeValue, runeWidth := utf8.DecodeRuneInString(l.input[l.pos:])
l.width = pos(runeWidth)
l.pos += l.width
return runeValue
}
// peek, but no consume
func (l *pagelexer) peek() rune {
r := l.next()
l.backup()
return r
}
// steps back one
func (l *pagelexer) backup() {
l.pos -= l.width
}
// sends an item back to the client.
func (l *pagelexer) emit(t itemType) {
l.items = append(l.items, item{t, l.start, l.input[l.start:l.pos]})
l.start = l.pos
}
// special case, do not send '\\' back to client
func (l *pagelexer) ignoreEscapesAndEmit(t itemType) {
val := strings.Map(func(r rune) rune {
if r == '\\' {
return -1
}
return r
}, l.input[l.start:l.pos])
l.items = append(l.items, item{t, l.start, val})
l.start = l.pos
}
// gets the current value (for debugging and error handling)
func (l *pagelexer) current() string {
return l.input[l.start:l.pos]
}
// ignore current element
func (l *pagelexer) ignore() {
l.start = l.pos
}
// nice to have in error logs
func (l *pagelexer) lineNum() int {
return strings.Count(l.input[:l.lastPos], "\n") + 1
}
// nil terminates the parser
func (l *pagelexer) errorf(format string, args ...interface{}) stateFunc {
l.items = append(l.items, item{tError, l.start, fmt.Sprintf(format, args...)})
return nil
}
// consumes and returns the next item
func (l *pagelexer) nextItem() item {
item := l.items[0]
l.items = l.items[1:]
l.lastPos = item.pos
return item
}
// scans until an opening shortcode opening bracket.
// if no shortcodes, it will keep on scanning until EOF
func lexTextOutsideShortcodes(l *pagelexer) stateFunc {
for {
if strings.HasPrefix(l.input[l.pos:], leftDelimScWithMarkup) || strings.HasPrefix(l.input[l.pos:], leftDelimScNoMarkup) {
if l.pos > l.start {
l.emit(tText)
}
if strings.HasPrefix(l.input[l.pos:], leftDelimScWithMarkup) {
l.currLeftDelimItem = tLeftDelimScWithMarkup
l.currRightDelimItem = tRightDelimScWithMarkup
} else {
l.currLeftDelimItem = tLeftDelimScNoMarkup
l.currRightDelimItem = tRightDelimScNoMarkup
}
return lexShortcodeLeftDelim
}
if l.next() == eof {
break
}
}
// Done!
if l.pos > l.start {
l.emit(tText)
}
l.emit(tEOF)
return nil
}
func lexShortcodeLeftDelim(l *pagelexer) stateFunc {
l.pos += pos(len(l.currentLeftShortcodeDelim()))
if strings.HasPrefix(l.input[l.pos:], leftComment) {
return lexShortcodeComment
}
l.emit(l.currentLeftShortcodeDelimItem())
l.elementStepNum = 0
l.paramElements = 0
return lexInsideShortcode
}
func lexShortcodeComment(l *pagelexer) stateFunc {
posRightComment := strings.Index(l.input[l.pos:], rightComment+l.currentRightShortcodeDelim())
if posRightComment <= 1 {
return l.errorf("comment must be closed")
}
// we emit all as text, except the comment markers
l.emit(tText)
l.pos += pos(len(leftComment))
l.ignore()
l.pos += pos(posRightComment - len(leftComment))
l.emit(tText)
l.pos += pos(len(rightComment))
l.ignore()
l.pos += pos(len(l.currentRightShortcodeDelim()))
l.emit(tText)
return lexTextOutsideShortcodes
}
func lexShortcodeRightDelim(l *pagelexer) stateFunc {
l.closingState = 0
l.pos += pos(len(l.currentRightShortcodeDelim()))
l.emit(l.currentRightShortcodeDelimItem())
return lexTextOutsideShortcodes
}
// either:
// 1. param
// 2. "param" or "param\"
// 3. param="123" or param="123\"
// 4. param="Some \"escaped\" text"
func lexShortcodeParam(l *pagelexer, escapedQuoteStart bool) stateFunc {
first := true
nextEq := false
var r rune
for {
r = l.next()
if first {
if r == '"' {
// a positional param with quotes
if l.paramElements == 2 {
return l.errorf("got quoted positional parameter. Cannot mix named and positional parameters")
}
l.paramElements = 1
l.backup()
return lexShortcodeQuotedParamVal(l, !escapedQuoteStart, tScParam)
}
first = false
} else if r == '=' {
// a named param
l.backup()
nextEq = true
break
}
if !isAlphaNumericOrHyphen(r) {
l.backup()
break
}
}
if l.paramElements == 0 {
l.paramElements++
if nextEq {
l.paramElements++
}
} else {
if nextEq && l.paramElements == 1 {
return l.errorf("got named parameter '%s'. Cannot mix named and positional parameters", l.current())
} else if !nextEq && l.paramElements == 2 {
return l.errorf("got positional parameter '%s'. Cannot mix named and positional parameters", l.current())
}
}
l.emit(tScParam)
return lexInsideShortcode
}
func lexShortcodeQuotedParamVal(l *pagelexer, escapedQuotedValuesAllowed bool, typ itemType) stateFunc {
openQuoteFound := false
escapedInnerQuoteFound := false
escapedQuoteState := 0
Loop:
for {
switch r := l.next(); {
case r == '\\':
if l.peek() == '"' {
if openQuoteFound && !escapedQuotedValuesAllowed {
l.backup()
break Loop
} else if openQuoteFound {
// the coming quoute is inside
escapedInnerQuoteFound = true
escapedQuoteState = 1
}
}
case r == eof, r == '\n':
return l.errorf("unterminated quoted string in shortcode parameter-argument: '%s'", l.current())
case r == '"':
if escapedQuoteState == 0 {
if openQuoteFound {
l.backup()
break Loop
} else {
openQuoteFound = true
l.ignore()
}
} else {
escapedQuoteState = 0
}
}
}
if escapedInnerQuoteFound {
l.ignoreEscapesAndEmit(typ)
} else {
l.emit(typ)
}
r := l.next()
if r == '\\' {
if l.peek() == '"' {
// ignore the escaped closing quote
l.ignore()
l.next()
l.ignore()
}
} else if r == '"' {
// ignore closing quote
l.ignore()
} else {
// handled by next state
l.backup()
}
return lexInsideShortcode
}
// scans an alphanumeric inside shortcode
func lexIdentifierInShortcode(l *pagelexer) stateFunc {
lookForEnd := false
Loop:
for {
switch r := l.next(); {
case isAlphaNumericOrHyphen(r):
// Allow forward slash inside names to make it possible to create namespaces.
case r == '/':
default:
l.backup()
word := l.input[l.start:l.pos]
if l.closingState > 0 && !l.openShortcodes[word] {
return l.errorf("closing tag for shortcode '%s' does not match start tag", word)
} else if l.closingState > 0 {
l.openShortcodes[word] = false
lookForEnd = true
}
l.closingState = 0
l.currShortcodeName = word
l.openShortcodes[word] = true
l.elementStepNum++
l.emit(tScName)
break Loop
}
}
if lookForEnd {
return lexEndOfShortcode
}
return lexInsideShortcode
}
func lexEndOfShortcode(l *pagelexer) stateFunc {
if strings.HasPrefix(l.input[l.pos:], l.currentRightShortcodeDelim()) {
return lexShortcodeRightDelim
}
switch r := l.next(); {
case isSpace(r):
l.ignore()
default:
return l.errorf("unclosed shortcode")
}
return lexEndOfShortcode
}
// scans the elements inside shortcode tags
func lexInsideShortcode(l *pagelexer) stateFunc {
if strings.HasPrefix(l.input[l.pos:], l.currentRightShortcodeDelim()) {
return lexShortcodeRightDelim
}
switch r := l.next(); {
case r == eof:
// eol is allowed inside shortcodes; this may go to end of document before it fails
return l.errorf("unclosed shortcode action")
case isSpace(r), isEndOfLine(r):
l.ignore()
case r == '=':
l.ignore()
return lexShortcodeQuotedParamVal(l, l.peek() != '\\', tScParamVal)
case r == '/':
if l.currShortcodeName == "" {
return l.errorf("got closing shortcode, but none is open")
}
l.closingState++
l.emit(tScClose)
case r == '\\':
l.ignore()
if l.peek() == '"' {
return lexShortcodeParam(l, true)
}
case l.elementStepNum > 0 && (isAlphaNumericOrHyphen(r) || r == '"'): // positional params can have quotes
l.backup()
return lexShortcodeParam(l, false)
case isAlphaNumeric(r):
l.backup()
return lexIdentifierInShortcode
default:
return l.errorf("unrecognized character in shortcode action: %#U. Note: Parameters with non-alphanumeric args must be quoted", r)
}
return lexInsideShortcode
}
// state helpers
func (l *pagelexer) currentLeftShortcodeDelimItem() itemType {
return l.currLeftDelimItem
}
func (l *pagelexer) currentRightShortcodeDelimItem() itemType {
return l.currRightDelimItem
}
func (l *pagelexer) currentLeftShortcodeDelim() string {
if l.currLeftDelimItem == tLeftDelimScWithMarkup {
return leftDelimScWithMarkup
}
return leftDelimScNoMarkup
}
func (l *pagelexer) currentRightShortcodeDelim() string {
if l.currRightDelimItem == tRightDelimScWithMarkup {
return rightDelimScWithMarkup
}
return rightDelimScNoMarkup
}
// helper functions
func isSpace(r rune) bool {
return r == ' ' || r == '\t'
}
func isAlphaNumericOrHyphen(r rune) bool {
// let unquoted YouTube ids as positional params slip through (they contain hyphens)
return isAlphaNumeric(r) || r == '-'
}
func isEndOfLine(r rune) bool {
return r == '\r' || r == '\n'
}
func isAlphaNumeric(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}