ref: b08d9f26ae00b6c55f615ad6b8fc36608e684204
parent: 4ecf56b891339e7a463280ab65e0eeebb8e5e206
author: Bjørn Erik Pedersen <[email protected]>
date: Tue Oct 20 16:35:12 EDT 2015
Make replaceShortcodeTokens rewrite the input slice Currently a `[]byte` copy is returned. In most cases this is the safe thing to do, but we should just modify/grow the slice as needed. This is faster and consumes less memory: ``` benchmark old ns/op new ns/op delta BenchmarkReplaceShortcodeTokens-4 7350 4419 -39.88% benchmark old allocs new allocs delta BenchmarkReplaceShortcodeTokens-4 5 1 -80.00% benchmark old bytes new bytes delta BenchmarkReplaceShortcodeTokens-4 4816 1152 -76.08% ``` This commit is aso a small spring cleaning of duplicated code in the different `PageConvert` methods. Fixes #1516
--- a/hugolib/handler_page.go
+++ b/hugolib/handler_page.go
@@ -56,25 +56,7 @@
func (h markdownHandler) Extensions() []string { return []string{"mdown", "markdown", "md"} } func (h markdownHandler) PageConvert(p *Page, t tpl.Template) HandledResult {- p.ProcessShortcodes(t)
-
- tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
- if len(p.contentShortCodes) > 0 {- replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
- tmpContent, tmpTableOfContents)
- if err != nil {- jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())- return HandledResult{err: err}- }
- tmpContent = replaced[0]
- tmpTableOfContents = replaced[1]
- }
-
- p.Content = helpers.BytesToHTML(tmpContent)
- p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
- return HandledResult{err: nil}+ return commonConvert(p, t)
}
type htmlHandler struct {@@ -84,21 +66,18 @@
func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} } func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {p.ProcessShortcodes(t)
- var content []byte
var err error
if len(p.contentShortCodes) > 0 {- content, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
+ p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
if err != nil {- jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())+ jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error()) return HandledResult{err: err}}
- } else {- content = p.rawContent
}
- p.Content = helpers.BytesToHTML(content)
+ p.Content = helpers.BytesToHTML(p.rawContent)
return HandledResult{err: nil}}
@@ -108,27 +87,7 @@
func (h asciidocHandler) Extensions() []string { return []string{"asciidoc", "adoc", "ad"} } func (h asciidocHandler) PageConvert(p *Page, t tpl.Template) HandledResult {- p.ProcessShortcodes(t)
-
- // TODO(spf13) Add/Refactor AsciiDoc Logic here
- tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
- if len(p.contentShortCodes) > 0 {- replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
- tmpContent, tmpTableOfContents)
- if err != nil {- jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())- return HandledResult{err: err}- }
- tmpContent = replaced[0]
- tmpTableOfContents = replaced[1]
- }
-
- p.Content = helpers.BytesToHTML(tmpContent)
- p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
- //err := p.Convert()
- return HandledResult{page: p, err: nil}+ return commonConvert(p, t)
}
type rstHandler struct {@@ -137,25 +96,7 @@
func (h rstHandler) Extensions() []string { return []string{"rest", "rst"} } func (h rstHandler) PageConvert(p *Page, t tpl.Template) HandledResult {- p.ProcessShortcodes(t)
-
- tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
- if len(p.contentShortCodes) > 0 {- replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
- tmpContent, tmpTableOfContents)
- if err != nil {- jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())- return HandledResult{err: err}- }
- tmpContent = replaced[0]
- tmpTableOfContents = replaced[1]
- }
-
- p.Content = helpers.BytesToHTML(tmpContent)
- p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
- return HandledResult{err: nil}+ return commonConvert(p, t)
}
type mmarkHandler struct {@@ -164,20 +105,26 @@
func (h mmarkHandler) Extensions() []string { return []string{"mmark"} } func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult {+ return commonConvert(p, t)
+}
+
+func commonConvert(p *Page, t tpl.Template) HandledResult {p.ProcessShortcodes(t)
- tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
+ var err error
+ renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
+
if len(p.contentShortCodes) > 0 {- tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, p.contentShortCodes)
+ renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes)
if err != nil {- jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())+ jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error()) return HandledResult{err: err}- } else {- tmpContent = tmpContentWithTokensReplaced
}
}
+
+ tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent)
p.Content = helpers.BytesToHTML(tmpContent)
p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -204,6 +204,7 @@
p.Truncated = len(bytes.Trim(sections[1], " \n\r")) > 0
}
+ // TODO(bep) consider doing this once only
renderedHeader := p.renderBytes(header)
if len(p.contentShortCodes) > 0 {tmpContentWithTokensReplaced, err :=
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -453,21 +453,8 @@
}
-// replaceShortcodeTokensInsources calls replaceShortcodeTokens for every source given.
-func replaceShortcodeTokensInsources(prefix string, replacements map[string]string, sources ...[]byte) (b [][]byte, err error) {- result := make([][]byte, len(sources))
- for i, s := range sources {- b, err := replaceShortcodeTokens(s, prefix, replacements)
-
- if err != nil {- return nil, err
- }
- result[i] = b
- }
- return result, nil
-}
-
// Replace prefixed shortcode tokens (HUGOSHORTCODE-1, HUGOSHORTCODE-2) with the real content.
+// Note: This function will rewrite the input slice.
func replaceShortcodeTokens(source []byte, prefix string, replacements map[string]string) ([]byte, error) { if len(replacements) == 0 {@@ -474,9 +461,6 @@
return source, nil
}
- buff := bp.GetBuffer()
- defer bp.PutBuffer(buff)
-
sourceLen := len(source)
start := 0
@@ -507,28 +491,14 @@
}
}
- oldVal := source[j:end]
- _, err := buff.Write(source[start:j])
- if err != nil {- return nil, errors.New("buff write failed")- }
- _, err = buff.Write(newVal)
- if err != nil {- return nil, errors.New("buff write failed")- }
- start = j + len(oldVal)
+ // This and other cool slice tricks: https://github.com/golang/go/wiki/SliceTricks
+ source = append(source[:j], append(newVal, source[end:]...)...)
k = bytes.Index(source[start:], pre)
+
}
- _, err := buff.Write(source[start:])
- if err != nil {- return nil, errors.New("buff write failed")- }
- bc := make([]byte, buff.Len(), buff.Len())
- copy(bc, buff.Bytes())
-
- return bc, nil
+ return source, nil
}
func getShortcodeTemplate(name string, t tpl.Template) *template.Template {--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -405,6 +405,14 @@
expect interface{} }{ {"Hello {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello World."},+ {"Hello {@{@PREFIX-1@}@.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, false},+ {"{@{@PREFIX2-1@}@}", "PREFIX2", map[string]string{"{@{@PREFIX2-1@}@}": "World"}, "World"},+ {"Hello World!", "PREFIX2", map[string]string{}, "Hello World!"},+ {"!{@{@PREFIX-1@}@}", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "!World"},+ {"{@{@PREFIX-1@}@}!", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "World!"},+ {"!{@{@PREFIX-1@}@}!", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "!World!"},+ {"@{@PREFIX-1@}@}", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "@{@PREFIX-1@}@}"},+ {"Hello {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "To You My Old Friend Who Told Me This Fantastic Story"}, "Hello To You My Old Friend Who Told Me This Fantastic Story."}, {"A {@{@A-1@}@} asdf {@{@A-2@}@}.", "A", map[string]string{"{@{@A-1@}@}": "v1", "{@{@A-2@}@}": "v2"}, "A v1 asdf v2."}, {"Hello {@{@PREFIX2-1@}@}. Go {@{@PREFIX2-2@}@}, Go, Go {@{@PREFIX2-3@}@} Go Go!.", "PREFIX2", map[string]string{"{@{@PREFIX2-1@}@}": "Europe", "{@{@PREFIX2-2@}@}": "Jonny", "{@{@PREFIX2-3@}@}": "Johnny"}, "Hello Europe. Go Jonny, Go, Go Johnny Go Go!."}, {"A {@{@PREFIX-2@}@} {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "A", "{@{@PREFIX-2@}@}": "B"}, "A B A."},@@ -420,11 +428,11 @@
{"Hello <p>{@{@PREFIX-1@}@}. END</p>.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello <p>World. END</p>."}, {"<p>Hello {@{@PREFIX-1@}@}</p>. END.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "<p>Hello World</p>. END."}, {"Hello <p>{@{@PREFIX-1@}@}12", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello <p>World12"},- // Make sure the buffering expands as needed
{"Hello {@{@P-1@}@}. {@{@P-1@}@}-{@{@P-1@}@} {@{@P-1@}@} {@{@P-1@}@} {@{@P-1@}@} END", "P", map[string]string{"{@{@P-1@}@}": strings.Repeat("BC", 100)}, fmt.Sprintf("Hello %s. %s-%s %s %s %s END", strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100))}, } {+
results, err := replaceShortcodeTokens([]byte(this.input), this.prefix, this.replacements)
if b, ok := this.expect.(bool); ok && !b {