ref: 85535084dea4d3e3adf1ebd08ae57b39d76e1904
parent: 19084eaf74246feac61d618c55031369520dfa8e
author: Bjørn Erik Pedersen <[email protected]>
date: Sun Apr 22 10:07:29 EDT 2018
hugolib: Process and render shortcodes in their order of appearance Fixes #3359
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -602,8 +602,8 @@
}
func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) {- if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 {- p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName())+ if p.shortcodeState != nil && p.shortcodeState.contentShortcodes.Len() > 0 {+ p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", p.shortcodeState.contentShortcodes.Len(), p.BaseFileName())err := p.shortcodeState.executeShortcodesForDelta(p)
if err != nil {--- /dev/null
+++ b/hugolib/orderedMap.go
@@ -1,0 +1,100 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "fmt"
+ "sync"
+)
+
+type orderedMap struct {+ sync.RWMutex
+ keys []interface{}+ m map[interface{}]interface{}+}
+
+func newOrderedMap() *orderedMap {+ return &orderedMap{m: make(map[interface{}]interface{})}+}
+
+func newOrderedMapFromStringMapString(m map[string]string) *orderedMap {+ om := newOrderedMap()
+ for k, v := range m {+ om.Add(k, v)
+ }
+ return om
+}
+
+func (m *orderedMap) Add(k, v interface{}) {+ m.Lock()
+ _, found := m.m[k]
+ if found {+ panic(fmt.Sprintf("%v already added", v))+ }
+ m.m[k] = v
+ m.keys = append(m.keys, k)
+ m.Unlock()
+
+}
+
+func (m *orderedMap) Get(k interface{}) (interface{}, bool) {+ m.RLock()
+ defer m.RUnlock()
+ v, found := m.m[k]
+ return v, found
+}
+
+func (m *orderedMap) Contains(k interface{}) bool {+ m.RLock()
+ defer m.RUnlock()
+ _, found := m.m[k]
+ return found
+}
+
+func (m *orderedMap) Keys() []interface{} {+ m.RLock()
+ defer m.RUnlock()
+ return m.keys
+}
+
+func (m *orderedMap) Len() int {+ m.RLock()
+ defer m.RUnlock()
+ return len(m.keys)
+}
+
+// Some shortcuts for known types.
+func (m *orderedMap) getShortcode(k interface{}) *shortcode {+ v, found := m.Get(k)
+ if !found {+ return nil
+ }
+ return v.(*shortcode)
+}
+
+func (m *orderedMap) getShortcodeRenderer(k interface{}) func() (string, error) {+ v, found := m.Get(k)
+ if !found {+ return nil
+ }
+ return v.(func() (string, error))
+}
+
+func (m *orderedMap) getString(k interface{}) string {+ v, found := m.Get(k)
+ if !found {+ return ""
+ }
+ return v.(string)
+}
--- /dev/null
+++ b/hugolib/orderedMap_test.go
@@ -1,0 +1,69 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "fmt"
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestOrderedMap(t *testing.T) {+ t.Parallel()
+ assert := require.New(t)
+
+ m := newOrderedMap()
+ m.Add("b", "vb")+ m.Add("c", "vc")+ m.Add("a", "va")+ b, f1 := m.Get("b")+
+ assert.True(f1)
+ assert.Equal(b, "vb")
+ assert.True(m.Contains("b"))+ assert.False(m.Contains("e"))+
+ assert.Equal([]interface{}{"b", "c", "a"}, m.Keys())+
+}
+
+func TestOrderedMapConcurrent(t *testing.T) {+ t.Parallel()
+ assert := require.New(t)
+
+ var wg sync.WaitGroup
+
+ m := newOrderedMap()
+
+ for i := 1; i < 20; i++ {+ wg.Add(1)
+ go func(id int) {+ defer wg.Done()
+ key := fmt.Sprintf("key%d", id)+ val := key + "val"
+ m.Add(key, val)
+ v, found := m.Get(key)
+ assert.True(found)
+ assert.Equal(v, val)
+ assert.True(m.Contains(key))
+ assert.True(m.Len() > 0)
+ assert.True(len(m.Keys()) > 0)
+ }(i)
+
+ }
+
+ wg.Wait()
+}
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -180,11 +180,11 @@
p *PageWithoutContent
// This is all shortcode rendering funcs for all potential output formats.
- contentShortcodes map[scKey]func() (string, error)
+ contentShortcodes *orderedMap
// This map contains the new or changed set of shortcodes that need
// to be rendered for the current output format.
- contentShortcodesDelta map[scKey]func() (string, error)
+ contentShortcodesDelta *orderedMap
// This maps the shorcode placeholders with the rendered content.
// We will do (potential) partial re-rendering per output format,
@@ -192,7 +192,7 @@
renderedShortcodes map[string]string
// Maps the shortcodeplaceholder with the actual shortcode.
- shortcodes map[string]shortcode
+ shortcodes *orderedMap
// All the shortcode names in this set.
nameSet map[string]bool
@@ -216,8 +216,8 @@
func newShortcodeHandler(p *Page) *shortcodeHandler { return &shortcodeHandler{p: p.withoutContent(),
- contentShortcodes: make(map[scKey]func() (string, error)),
- shortcodes: make(map[string]shortcode),
+ contentShortcodes: newOrderedMap(),
+ shortcodes: newOrderedMap(),
nameSet: make(map[string]bool),
renderedShortcodes: make(map[string]string),
}
@@ -259,7 +259,7 @@
const innerCleanupRegexp = `\A<p>(.*)</p>\n\z`
const innerCleanupExpand = "$1"
-func prepareShortcodeForPage(placeholder string, sc shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) {+func prepareShortcodeForPage(placeholder string, sc *shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) {m := make(map[scKey]func() (string, error))
lang := p.Lang()
@@ -277,7 +277,7 @@
func renderShortcode(
tmplKey scKey,
- sc shortcode,
+ sc *shortcode,
parent *ShortcodeWithPage,
p *PageWithoutContent) string {@@ -298,8 +298,8 @@
switch innerData.(type) {case string:
inner += innerData.(string)
- case shortcode:
- inner += renderShortcode(tmplKey, innerData.(shortcode), data, p)
+ case *shortcode:
+ inner += renderShortcode(tmplKey, innerData.(*shortcode), data, p)
default:
p.s.Log.ERROR.Printf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ",sc.name, p.Path(), reflect.TypeOf(innerData))
@@ -363,48 +363,51 @@
contentShortcodes := s.contentShortcodesForOutputFormat(s.p.s.rc.Format)
- if s.contentShortcodesDelta == nil || len(s.contentShortcodesDelta) == 0 {+ if s.contentShortcodesDelta == nil || s.contentShortcodesDelta.Len() == 0 {s.contentShortcodesDelta = contentShortcodes
return true
}
- delta := make(map[scKey]func() (string, error))
+ delta := newOrderedMap()
- for k, v := range contentShortcodes {- if _, found := s.contentShortcodesDelta[k]; !found {- delta[k] = v
+ for _, k := range contentShortcodes.Keys() {+ if !s.contentShortcodesDelta.Contains(k) {+ v, _ := contentShortcodes.Get(k)
+ delta.Add(k, v)
}
}
s.contentShortcodesDelta = delta
- return len(delta) > 0
+ return delta.Len() > 0
}
-func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) map[scKey]func() (string, error) {- contentShortcodesForOuputFormat := make(map[scKey]func() (string, error))
+func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) *orderedMap {+ contentShortcodesForOuputFormat := newOrderedMap()
lang := s.p.Lang()
- for shortcodePlaceholder := range s.shortcodes {+ for _, key := range s.shortcodes.Keys() {+ shortcodePlaceholder := key.(string)
+ // shortcodePlaceholder := s.shortcodes.getShortcode(key)
key := newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder)
- renderFn, found := s.contentShortcodes[key]
+ renderFn, found := s.contentShortcodes.Get(key)
if !found {key.OutputFormat = ""
- renderFn, found = s.contentShortcodes[key]
+ renderFn, found = s.contentShortcodes.Get(key)
}
// Fall back to HTML
if !found && key.Suffix != "html" {key.Suffix = "html"
- renderFn, found = s.contentShortcodes[key]
+ renderFn, found = s.contentShortcodes.Get(key)
}
if !found { panic(fmt.Sprintf("Shortcode %q could not be found", shortcodePlaceholder))}
- contentShortcodesForOuputFormat[newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder)] = renderFn
+ contentShortcodesForOuputFormat.Add(newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder), renderFn)
}
return contentShortcodesForOuputFormat
@@ -412,13 +415,14 @@
func (s *shortcodeHandler) executeShortcodesForDelta(p *PageWithoutContent) error {- for k, render := range s.contentShortcodesDelta {+ for _, k := range s.contentShortcodesDelta.Keys() {+ render := s.contentShortcodesDelta.getShortcodeRenderer(k)
renderedShortcode, err := render()
if err != nil { return fmt.Errorf("Failed to execute shortcode in page %q: %s", p.Path(), err)}
- s.renderedShortcodes[k.ShortcodePlaceholder] = renderedShortcode
+ s.renderedShortcodes[k.(scKey).ShortcodePlaceholder] = renderedShortcode
}
return nil
@@ -425,14 +429,15 @@
}
-func createShortcodeRenderers(shortcodes map[string]shortcode, p *PageWithoutContent) map[scKey]func() (string, error) {+func createShortcodeRenderers(shortcodes *orderedMap, p *PageWithoutContent) *orderedMap {- shortcodeRenderers := make(map[scKey]func() (string, error))
+ shortcodeRenderers := newOrderedMap()
- for k, v := range shortcodes {- prepared := prepareShortcodeForPage(k, v, nil, p)
+ for _, k := range shortcodes.Keys() {+ v := shortcodes.getShortcode(k)
+ prepared := prepareShortcodeForPage(k.(string), v, nil, p)
for kk, vv := range prepared {- shortcodeRenderers[kk] = vv
+ shortcodeRenderers.Add(kk, vv)
}
}
@@ -444,8 +449,8 @@
// pageTokens state:
// - before: positioned just before the shortcode start
// - after: shortcode(s) consumed (plural when they are nested)
-func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *PageWithoutContent) (shortcode, error) {- sc := shortcode{}+func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *PageWithoutContent) (*shortcode, error) {+ sc := &shortcode{}var isInner = false
var currItem item
@@ -616,7 +621,7 @@
placeHolder := s.createShortcodePlaceholder()
result.WriteString(placeHolder)
- s.shortcodes[placeHolder] = currShortcode
+ s.shortcodes.Add(placeHolder, currShortcode)
case tEOF:
break Loop
case tError:
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -433,8 +433,8 @@
t.Fatalf("[%d] %s: Failed to compile regexp %q: %q", i, this.name, expected, err)}
- if strings.Count(content, shortcodePlaceholderPrefix) != len(shortCodes) {- t.Fatalf("[%d] %s: Not enough placeholders, found %d", i, this.name, len(shortCodes))+ if strings.Count(content, shortcodePlaceholderPrefix) != shortCodes.Len() {+ t.Fatalf("[%d] %s: Not enough placeholders, found %d", i, this.name, shortCodes.Len())}
if !r.MatchString(content) {@@ -441,8 +441,9 @@
t.Fatalf("[%d] %s: Shortcode extract didn't match. got %q but expected %q", i, this.name, content, expected)}
- for placeHolder, sc := range shortCodes {- if !strings.Contains(content, placeHolder) {+ for _, placeHolder := range shortCodes.Keys() {+ sc := shortCodes.getShortcode(placeHolder)
+ if !strings.Contains(content, placeHolder.(string)) { t.Fatalf("[%d] %s: Output does not contain placeholder %q", i, this.name, placeHolder)}
@@ -753,10 +754,11 @@
}
-func collectAndSortShortcodes(shortcodes map[string]shortcode) []string {+func collectAndSortShortcodes(shortcodes *orderedMap) []string {var asArray []string
- for key, sc := range shortcodes {+ for _, key := range shortcodes.Keys() {+ sc := shortcodes.getShortcode(key)
asArray = append(asArray, fmt.Sprintf("%s:%s", key, sc))}
@@ -879,5 +881,50 @@
newScKeyFromLangAndOutputFormat("en", output.AMPFormat, "EFGH")) require.Equal(t, scKey{Suffix: "html", ShortcodePlaceholder: "IJKL"}, newDefaultScKey("IJKL"))+
+}
+
+func TestPreserveShortcodeOrder(t *testing.T) {+ t.Parallel()
+ assert := require.New(t)
+
+ contentTemplate := `---
+title: doc%d
+weight: %d
+---
+# doc
+
+{{< increment >}}{{< s1 >}}{{< increment >}}{{< s2 >}}{{< increment >}}{{< s3 >}}{{< increment >}}{{< s4 >}}{{< increment >}}{{< s5 >}}+
+
+`
+
+ shortCodeTemplate := `v%d: {{ .Page.Scratch.Get "v" }}|`+
+ var shortcodes []string
+ var content []string
+
+ shortcodes = append(shortcodes, []string{"shortcodes/increment.html", `{{ .Page.Scratch.Add "v" 1}}`}...)+
+ for i := 1; i <= 5; i++ {+ shortcodes = append(shortcodes, []string{fmt.Sprintf("shortcodes/s%d.html", i), fmt.Sprintf(shortCodeTemplate, i)}...)+ }
+
+ for i := 1; i <= 3; i++ {+ content = append(content, []string{fmt.Sprintf("p%d.md", i), fmt.Sprintf(contentTemplate, i, i)}...)+ }
+
+ builder := newTestSitesBuilder(t).WithDefaultMultiSiteConfig()
+
+ builder.WithContent(content...).WithTemplatesAdded(shortcodes...).CreateSites().Build(BuildCfg{})+
+ s := builder.H.Sites[0]
+ assert.Equal(3, len(s.RegularPages))
+
+ p1 := s.RegularPages[0]
+
+ if !strings.Contains(string(p1.content()), `v1: 1|v2: 2|v3: 3|v4: 4|v5: 5`) {+ t.Fatal(p1.content())
+ }
}