ref: 822dc627a1cfdf1f97882f27761675ac6ace7669
parent: 43f9df0194d229805d80b13c9e38a7a0fec12cf4
author: Bjørn Erik Pedersen <[email protected]>
date: Fri Dec 21 11:21:13 EST 2018
tpl/transform: Add transform.Unmarshal func Fixes #5428
--- /dev/null
+++ b/cache/namedmemcache/named_cache.go
@@ -1,0 +1,84 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package namedmemcache provides a memory cache with a named lock. This is suitable
+// for situations where creating the cached resource can be time consuming or otherwise
+// resource hungry, or in situations where a "once only per key" is a requirement.
+package namedmemcache
+
+import (
+ "sync"
+
+ "github.com/BurntSushi/locker"
+)
+
+// Cache holds the cached values.
+type Cache struct {+ nlocker *locker.Locker
+ cache map[string]cacheEntry
+ mu sync.RWMutex
+}
+
+type cacheEntry struct {+ value interface{}+ err error
+}
+
+// New creates a new cache.
+func New() *Cache {+ return &Cache{+ nlocker: locker.NewLocker(),
+ cache: make(map[string]cacheEntry),
+ }
+}
+
+// Clear clears the cache state.
+func (c *Cache) Clear() {+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ c.cache = make(map[string]cacheEntry)
+ c.nlocker = locker.NewLocker()
+
+}
+
+// GetOrCreate tries to get the value with the given cache key, if not found
+// create will be called and cached.
+// This method is thread safe. It also guarantees that the create func for a given
+// key is invoced only once for this cache.
+func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) {+ c.mu.RLock()
+ entry, found := c.cache[key]
+ c.mu.RUnlock()
+
+ if found {+ return entry.value, entry.err
+ }
+
+ c.nlocker.Lock(key)
+ defer c.nlocker.Unlock(key)
+
+ // Double check
+ if entry, found := c.cache[key]; found {+ return entry.value, entry.err
+ }
+
+ // Create it.
+ value, err := create()
+
+ c.mu.Lock()
+ c.cache[key] = cacheEntry{value: value, err: err}+ c.mu.Unlock()
+
+ return value, err
+}
--- /dev/null
+++ b/cache/namedmemcache/named_cache_test.go
@@ -1,0 +1,80 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package namedmemcache
+
+import (
+ "fmt"
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNamedCache(t *testing.T) {+ t.Parallel()
+ assert := require.New(t)
+
+ cache := New()
+
+ counter := 0
+ create := func() (interface{}, error) {+ counter++
+ return counter, nil
+ }
+
+ for i := 0; i < 5; i++ {+ v1, err := cache.GetOrCreate("a1", create)+ assert.NoError(err)
+ assert.Equal(1, v1)
+ v2, err := cache.GetOrCreate("a2", create)+ assert.NoError(err)
+ assert.Equal(2, v2)
+ }
+
+ cache.Clear()
+
+ v3, err := cache.GetOrCreate("a2", create)+ assert.NoError(err)
+ assert.Equal(3, v3)
+}
+
+func TestNamedCacheConcurrent(t *testing.T) {+ t.Parallel()
+
+ assert := require.New(t)
+
+ var wg sync.WaitGroup
+
+ cache := New()
+
+ create := func(i int) func() (interface{}, error) {+ return func() (interface{}, error) {+ return i, nil
+ }
+ }
+
+ for i := 0; i < 10; i++ {+ wg.Add(1)
+ go func() {+ defer wg.Done()
+ for j := 0; j < 100; j++ {+ id := fmt.Sprintf("id%d", j)+ v, err := cache.GetOrCreate(id, create(j))
+ assert.NoError(err)
+ assert.Equal(j, v)
+ }
+ }()
+ }
+ wg.Wait()
+}
--- a/deps/deps.go
+++ b/deps/deps.go
@@ -123,6 +123,9 @@
// Add adds a function to a Listeners instance.
func (b *Listeners) Add(f func()) {+ if b == nil {+ return
+ }
b.Lock()
defer b.Unlock()
b.listeners = append(b.listeners, f)
@@ -190,6 +193,14 @@
if fs == nil {// Default to the production file system.
fs = hugofs.NewDefault(cfg.Language)
+ }
+
+ if cfg.MediaTypes == nil {+ cfg.MediaTypes = media.DefaultTypes
+ }
+
+ if cfg.OutputFormats == nil {+ cfg.OutputFormats = output.DefaultFormats
}
ps, err := helpers.NewPathSpec(fs, cfg.Language)
--- a/helpers/general.go
+++ b/helpers/general.go
@@ -394,11 +394,10 @@
return hex.EncodeToString(h.Sum(nil)), nil
}
-// MD5FromFile creates a MD5 hash from the given file.
-// It will not close the file.
-func MD5FromFile(f afero.File) (string, error) {+// MD5FromReader creates a MD5 hash from the given reader.
+func MD5FromReader(r io.Reader) (string, error) {h := md5.New()
- if _, err := io.Copy(h, f); err != nil {+ if _, err := io.Copy(h, r); err != nil {return "", nil
}
return hex.EncodeToString(h.Sum(nil)), nil
--- a/helpers/general_test.go
+++ b/helpers/general_test.go
@@ -272,7 +272,7 @@
req.NoError(err)
req.NotEqual(m3, m4)
- m5, err := MD5FromFile(bf2)
+ m5, err := MD5FromReader(bf2)
req.NoError(err)
req.NotEqual(m4, m5)
}
@@ -293,7 +293,7 @@
}
b.StartTimer()
if full {- if _, err := MD5FromFile(f); err != nil {+ if _, err := MD5FromReader(f); err != nil {b.Fatal(err)
}
} else {--- a/hugolib/resource_chain_test.go
+++ b/hugolib/resource_chain_test.go
@@ -339,6 +339,16 @@
assert.False(b.CheckExists("public/inline.min.css"), "Inline content should not be copied to /public")}},
+ {"unmarshal", func() bool { return true }, func(b *sitesBuilder) {+ b.WithTemplates("home.html", `+{{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }}+Slogan: {{ $toml.slogan }}+
+`)
+ }, func(b *sitesBuilder) {+ b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`)+ }},
+
{"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) {}},
}
--- a/media/mediaType.go
+++ b/media/mediaType.go
@@ -135,6 +135,8 @@
XMLType = Type{MainType: "application", SubType: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter} SVGType = Type{MainType: "image", SubType: "svg", mimeSuffix: "xml", Suffixes: []string{"svg"}, Delimiter: defaultDelimiter} TextType = Type{MainType: "text", SubType: "plain", Suffixes: []string{"txt"}, Delimiter: defaultDelimiter}+ TOMLType = Type{MainType: "application", SubType: "toml", Suffixes: []string{"toml"}, Delimiter: defaultDelimiter}+ YAMLType = Type{MainType: "application", SubType: "yaml", Suffixes: []string{"yaml", "yml"}, Delimiter: defaultDelimiter} OctetType = Type{MainType: "application", SubType: "octet-stream"})
@@ -154,6 +156,8 @@
SVGType,
TextType,
OctetType,
+ YAMLType,
+ TOMLType,
}
func init() {--- a/media/mediaType_test.go
+++ b/media/mediaType_test.go
@@ -39,6 +39,8 @@
{SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"}, {TextType, "text", "plain", "txt", "text/plain", "text/plain"}, {XMLType, "application", "xml", "xml", "application/xml", "application/xml"},+ {TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},+ {YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"}, } {require.Equal(t, test.expectedMainType, test.tp.MainType)
require.Equal(t, test.expectedSubType, test.tp.SubType)
@@ -49,6 +51,8 @@
require.Equal(t, test.expectedString, test.tp.String())
}
+
+ require.Equal(t, 15, len(DefaultTypes))
}
--- a/parser/metadecoders/format.go
+++ b/parser/metadecoders/format.go
@@ -17,6 +17,8 @@
"path/filepath"
"strings"
+ "github.com/gohugoio/hugo/media"
+
"github.com/gohugoio/hugo/parser/pageparser"
)
@@ -55,6 +57,18 @@
}
+// FormatFromMediaType gets the Format given a MIME type, empty string
+// if unknown.
+func FormatFromMediaType(m media.Type) Format {+ for _, suffix := range m.Suffixes {+ if f := FormatFromString(suffix); f != "" {+ return f
+ }
+ }
+
+ return ""
+}
+
// FormatFromFrontMatterType will return empty if not supported.
func FormatFromFrontMatterType(typ pageparser.ItemType) Format { switch typ {@@ -69,4 +83,40 @@
default:
return ""
}
+}
+
+// FormatFromContentString tries to detect the format (JSON, YAML or TOML)
+// in the given string.
+// It return an empty string if no format could be detected.
+func FormatFromContentString(data string) Format {+ jsonIdx := strings.Index(data, "{")+ yamlIdx := strings.Index(data, ":")
+ tomlIdx := strings.Index(data, "=")
+
+ if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) {+ return JSON
+ }
+
+ if isLowerIndexThan(yamlIdx, tomlIdx) {+ return YAML
+ }
+
+ if tomlIdx != -1 {+ return TOML
+ }
+
+ return ""
+}
+
+func isLowerIndexThan(first int, others ...int) bool {+ if first == -1 {+ return false
+ }
+ for _, other := range others {+ if other != -1 && other < first {+ return false
+ }
+ }
+
+ return true
}
--- a/parser/metadecoders/format_test.go
+++ b/parser/metadecoders/format_test.go
@@ -17,6 +17,8 @@
"fmt"
"testing"
+ "github.com/gohugoio/hugo/media"
+
"github.com/gohugoio/hugo/parser/pageparser"
"github.com/stretchr/testify/require"
@@ -41,6 +43,21 @@
}
}
+func TestFormatFromMediaType(t *testing.T) {+ assert := require.New(t)
+ for i, test := range []struct {+ m media.Type
+ expect Format
+ }{+ {media.JSONType, JSON},+ {media.YAMLType, YAML},+ {media.TOMLType, TOML},+ {media.CalendarType, ""},+ } {+ assert.Equal(test.expect, FormatFromMediaType(test.m), fmt.Sprintf("t%d", i))+ }
+}
+
func TestFormatFromFrontMatterType(t *testing.T) {assert := require.New(t)
for i, test := range []struct {@@ -54,5 +71,30 @@
{pageparser.TypeIgnore, ""}, } { assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))+ }
+}
+
+func TestFormatFromContentString(t *testing.T) {+ t.Parallel()
+ assert := require.New(t)
+
+ for i, test := range []struct {+ data string
+ expect interface{}+ }{+ {`foo = "bar"`, TOML},+ {` foo = "bar"`, TOML},+ {`foo="bar"`, TOML},+ {`foo: "bar"`, YAML},+ {`foo:"bar"`, YAML},+ {`{ "foo": "bar"`, JSON},+ {`asdfasdf`, Format("")},+ {``, Format("")},+ } {+ errMsg := fmt.Sprintf("[%d] %s", i, test.data)+
+ result := FormatFromContentString(test.data)
+
+ assert.Equal(test.expect, result, errMsg)
}
}
--- a/resource/resource.go
+++ b/resource/resource.go
@@ -50,6 +50,7 @@
_ ResourcesLanguageMerger = (*Resources)(nil)
_ permalinker = (*genericResource)(nil)
_ collections.Slicer = (*genericResource)(nil)
+ _ Identifier = (*genericResource)(nil)
)
var noData = make(map[string]interface{})@@ -76,6 +77,8 @@
// Resource represents a linkable resource, i.e. a content page, image etc.
type Resource interface {+ resourceBase
+
// Permalink represents the absolute link to this resource.
Permalink() string
@@ -87,9 +90,6 @@
// For content pages, this value is "page".
ResourceType() string
- // MediaType is this resource's MIME type.
- MediaType() media.Type
-
// Name is the logical name of this resource. This can be set in the front matter
// metadata for this resource. If not set, Hugo will assign a value.
// This will in most cases be the base filename.
@@ -109,6 +109,13 @@
Params() map[string]interface{}}
+// resourceBase pulls out the minimal set of operations to define a Resource,
+// to simplify testing etc.
+type resourceBase interface {+ // MediaType is this resource's MIME type.
+ MediaType() media.Type
+}
+
// ResourcesLanguageMerger describes an interface for merging resources from a
// different language.
type ResourcesLanguageMerger interface {@@ -121,12 +128,17 @@
TranslationKey() string
}
+// Identifier identifies a resource.
+type Identifier interface {+ Key() string
+}
+
// ContentResource represents a Resource that provides a way to get to its content.
// Most Resource types in Hugo implements this interface, including Page.
// This should be used with care, as it will read the file content into memory, but it
// should be cached as effectively as possible by the implementation.
type ContentResource interface {- Resource
+ resourceBase
// Content returns this resource's content. It will be equivalent to reading the content
// that RelPermalink points to in the published folder.
@@ -143,7 +155,7 @@
// ReadSeekCloserResource is a Resource that supports loading its content.
type ReadSeekCloserResource interface {- Resource
+ resourceBase
ReadSeekCloser() (hugio.ReadSeekCloser, error)
}
@@ -714,6 +726,10 @@
func (l *genericResource) RelPermalink() string {l.publishIfNeeded()
return l.relPermalinkFor(l.relTargetDirFile.path())
+}
+
+func (l *genericResource) Key() string {+ return l.relTargetDirFile.path()
}
func (l *genericResource) relPermalinkFor(target string) string {--- a/resource/resource_test.go
+++ b/resource/resource_test.go
@@ -50,6 +50,7 @@
assert.Equal("https://example.com/foo/foo.css", r.Permalink()) assert.Equal("/foo/foo.css", r.RelPermalink())+ assert.Equal("foo.css", r.Key()) assert.Equal("css", r.ResourceType())}
--- a/resource/transform.go
+++ b/resource/transform.go
@@ -38,6 +38,7 @@
_ ContentResource = (*transformedResource)(nil)
_ ReadSeekCloserResource = (*transformedResource)(nil)
_ collections.Slicer = (*transformedResource)(nil)
+ _ Identifier = (*transformedResource)(nil)
)
func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) {@@ -249,6 +250,13 @@
return m
}
+func (r *transformedResource) Key() string {+ if err := r.initTransform(false, false); err != nil {+ return ""
+ }
+ return r.linker.relPermalinkFor(r.Target)
+}
+
func (r *transformedResource) Permalink() string { if err := r.initTransform(false, true); err != nil {return ""
@@ -481,8 +489,8 @@
}
return nil
-
}
+
func (r *transformedResource) initTransform(setContent, publish bool) error { r.transformInit.Do(func() {r.published = publish
--- a/tpl/transform/init.go
+++ b/tpl/transform/init.go
@@ -95,6 +95,14 @@
},
)
+ ns.AddMethodMapping(ctx.Unmarshal,
+ []string{"unmarshal"},+ [][2]string{+ {`{{ "hello = \"Hello World\"" | transform.Unmarshal }}`, "map[hello:Hello World]"},+ {`{{ "hello = \"Hello World\"" | resources.FromString "data/greetings.toml" | transform.Unmarshal }}`, "map[hello:Hello World]"},+ },
+ )
+
return ns
}
--- a/tpl/transform/remarshal.go
+++ b/tpl/transform/remarshal.go
@@ -2,9 +2,10 @@
import (
"bytes"
- "errors"
"strings"
+ "github.com/pkg/errors"
+
"github.com/gohugoio/hugo/parser"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/spf13/cast"
@@ -34,9 +35,9 @@
return "", err
}
- fromFormat, err := detectFormat(from)
- if err != nil {- return "", err
+ fromFormat := metadecoders.FormatFromContentString(from)
+ if fromFormat == "" {+ return "", errors.New("failed to detect format from content")}
meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
@@ -55,25 +56,4 @@
}
return "", errors.New("failed to detect target data serialization format")-}
-
-func detectFormat(data string) (metadecoders.Format, error) {- jsonIdx := strings.Index(data, "{")- yamlIdx := strings.Index(data, ":")
- tomlIdx := strings.Index(data, "=")
-
- if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) {- return metadecoders.JSON, nil
- }
-
- if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) {- return metadecoders.YAML, nil
- }
-
- if tomlIdx != -1 {- return metadecoders.TOML, nil
- }
-
- return "", errors.New("failed to detect data serialization format")-
}
--- a/tpl/transform/remarshal_test.go
+++ b/tpl/transform/remarshal_test.go
@@ -18,7 +18,6 @@
"testing"
"github.com/gohugoio/hugo/helpers"
- "github.com/gohugoio/hugo/parser/metadecoders"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
@@ -170,35 +169,4 @@
_, err = ns.Remarshal("json", "asdf")assert.Error(err)
-}
-
-func TestRemarshalDetectFormat(t *testing.T) {- t.Parallel()
- assert := require.New(t)
-
- for i, test := range []struct {- data string
- expect interface{}- }{- {`foo = "bar"`, metadecoders.TOML},- {` foo = "bar"`, metadecoders.TOML},- {`foo="bar"`, metadecoders.TOML},- {`foo: "bar"`, metadecoders.YAML},- {`foo:"bar"`, metadecoders.YAML},- {`{ "foo": "bar"`, metadecoders.JSON},- {`asdfasdf`, false},- {``, false},- } {- errMsg := fmt.Sprintf("[%d] %s", i, test.data)-
- result, err := detectFormat(test.data)
-
- if b, ok := test.expect.(bool); ok && !b {- assert.Error(err, errMsg)
- continue
- }
-
- assert.NoError(err, errMsg)
- assert.Equal(test.expect, result)
- }
}
--- a/tpl/transform/transform.go
+++ b/tpl/transform/transform.go
@@ -19,6 +19,8 @@
"html"
"html/template"
+ "github.com/gohugoio/hugo/cache/namedmemcache"
+
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/cast"
@@ -26,14 +28,22 @@
// New returns a new instance of the transform-namespaced template functions.
func New(deps *deps.Deps) *Namespace {+ cache := namedmemcache.New()
+ deps.BuildStartListeners.Add(
+ func() {+ cache.Clear()
+ })
+
return &Namespace{- deps: deps,
+ cache: cache,
+ deps: deps,
}
}
// Namespace provides template functions for the "transform" namespace.
type Namespace struct {- deps *deps.Deps
+ cache *namedmemcache.Cache
+ deps *deps.Deps
}
// Emojify returns a copy of s with all emoji codes replaced with actual emojis.
--- a/tpl/transform/transform_test.go
+++ b/tpl/transform/transform_test.go
@@ -34,7 +34,6 @@
t.Parallel()
v := viper.New()
- v.Set("contentDir", "content")ns := New(newDeps(v))
for i, test := range []struct {@@ -215,7 +214,6 @@
t.Parallel()
v := viper.New()
- v.Set("contentDir", "content")ns := New(newDeps(v))
for i, test := range []struct {@@ -241,8 +239,11 @@
}
func newDeps(cfg config.Provider) *deps.Deps {+ cfg.Set("contentDir", "content")+ cfg.Set("i18nDir", "i18n")+
l := langs.NewLanguage("en", cfg)- l.Set("i18nDir", "i18n")+
cs, err := helpers.NewContentSpec(l)
if err != nil {panic(err)
--- /dev/null
+++ b/tpl/transform/unmarshal.go
@@ -1,0 +1,98 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transform
+
+import (
+ "io/ioutil"
+
+ "github.com/gohugoio/hugo/common/hugio"
+
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/parser/metadecoders"
+ "github.com/gohugoio/hugo/resource"
+ "github.com/pkg/errors"
+
+ "github.com/spf13/cast"
+)
+
+// Unmarshal unmarshals the data given, which can be either a string
+// or a Resource. Supported formats are JSON, TOML and YAML.
+func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {+
+ // All the relevant Resource types implements ReadSeekCloserResource,
+ // which should be the most effective way to get the content.
+ if r, ok := data.(resource.ReadSeekCloserResource); ok {+ var key string
+ var reader hugio.ReadSeekCloser
+
+ if k, ok := r.(resource.Identifier); ok {+ key = k.Key()
+ }
+
+ if key == "" {+ reader, err := r.ReadSeekCloser()
+ if err != nil {+ return nil, err
+ }
+ defer reader.Close()
+
+ key, err = helpers.MD5FromReader(reader)
+ if err != nil {+ return nil, err
+ }
+
+ reader.Seek(0, 0)
+ }
+
+ return ns.cache.GetOrCreate(key, func() (interface{}, error) {+ f := metadecoders.FormatFromMediaType(r.MediaType())
+ if f == "" {+ return nil, errors.Errorf("MIME %q not supported", r.MediaType())+ }
+
+ if reader == nil {+ var err error
+ reader, err = r.ReadSeekCloser()
+ if err != nil {+ return nil, err
+ }
+ defer reader.Close()
+ }
+
+ b, err := ioutil.ReadAll(reader)
+ if err != nil {+ return nil, err
+ }
+
+ return metadecoders.Unmarshal(b, f)
+ })
+
+ }
+
+ dataStr, err := cast.ToStringE(data)
+ if err != nil {+ return nil, errors.Errorf("type %T not supported", data)+ }
+
+ key := helpers.MD5String(dataStr)
+
+ return ns.cache.GetOrCreate(key, func() (interface{}, error) {+ f := metadecoders.FormatFromContentString(dataStr)
+ if f == "" {+ return nil, errors.New("unknown format")+ }
+
+ return metadecoders.Unmarshal([]byte(dataStr), f)
+ })
+}
--- /dev/null
+++ b/tpl/transform/unmarshal_test.go
@@ -1,0 +1,185 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transform
+
+import (
+ "fmt"
+ "math/rand"
+ "strings"
+ "testing"
+
+ "github.com/gohugoio/hugo/common/hugio"
+
+ "github.com/gohugoio/hugo/media"
+
+ "github.com/gohugoio/hugo/resource"
+ "github.com/spf13/viper"
+ "github.com/stretchr/testify/require"
+)
+
+const (
+ testJSON = `
+
+{+ "ROOT_KEY": {+ "title": "example glossary",
+ "GlossDiv": {+ "title": "S",
+ "GlossList": {+ "GlossEntry": {+ "ID": "SGML",
+ "SortAs": "SGML",
+ "GlossTerm": "Standard Generalized Markup Language",
+ "Acronym": "SGML",
+ "Abbrev": "ISO 8879:1986",
+ "GlossDef": {+ "para": "A meta-markup language, used to create markup languages such as DocBook.",
+ "GlossSeeAlso": ["GML", "XML"]
+ },
+ "GlossSee": "markup"
+ }
+ }
+ }
+ }
+}
+
+ `
+)
+
+var _ resource.ReadSeekCloserResource = (*testContentResource)(nil)
+
+type testContentResource struct {+ content string
+ mime media.Type
+
+ key string
+}
+
+func (t testContentResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {+ return hugio.NewReadSeekerNoOpCloserFromString(t.content), nil
+}
+
+func (t testContentResource) MediaType() media.Type {+ return t.mime
+}
+
+func (t testContentResource) Key() string {+ return t.key
+}
+
+func TestUnmarshal(t *testing.T) {+
+ v := viper.New()
+ ns := New(newDeps(v))
+ assert := require.New(t)
+
+ assertSlogan := func(m map[string]interface{}) {+ assert.Equal("Hugo Rocks!", m["slogan"])+ }
+
+ for i, test := range []struct {+ data interface{}+ expect interface{}+ }{+ {`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) {+ assertSlogan(m)
+ }},
+ {`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) {+ assertSlogan(m)
+ }},
+ {`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) {+ assertSlogan(m)
+ }},
+ {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) {+ assertSlogan(m)
+ }},
+ {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) {+ assertSlogan(m)
+ }},
+ {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) {+ assertSlogan(m)
+ }},
+ // errors
+ {"thisisnotavaliddataformat", false},+ {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false},+ {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false},+ {"thisisnotavaliddataformat", false},+ {`{ notjson }`, false},+ {tstNoStringer{}, false},+ } {+ errMsg := fmt.Sprintf("[%d]", i)+
+ result, err := ns.Unmarshal(test.data)
+
+ if b, ok := test.expect.(bool); ok && !b {+ assert.Error(err, errMsg)
+ } else if fn, ok := test.expect.(func(m map[string]interface{})); ok {+ assert.NoError(err, errMsg)
+ m, ok := result.(map[string]interface{})+ assert.True(ok, errMsg)
+ fn(m)
+ } else {+ assert.NoError(err, errMsg)
+ assert.Equal(test.expect, result, errMsg)
+ }
+
+ }
+}
+
+func BenchmarkUnmarshalString(b *testing.B) {+ v := viper.New()
+ ns := New(newDeps(v))
+
+ const numJsons = 100
+
+ var jsons [numJsons]string
+ for i := 0; i < numJsons; i++ {+ jsons[i] = strings.Replace(testJSON, "ROOT_KEY", fmt.Sprintf("root%d", i), 1)+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {+ result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
+ if err != nil {+ b.Fatal(err)
+ }
+ if result == nil {+ b.Fatal("no result")+ }
+ }
+}
+
+func BenchmarkUnmarshalResource(b *testing.B) {+ v := viper.New()
+ ns := New(newDeps(v))
+
+ const numJsons = 100
+
+ var jsons [numJsons]testContentResource
+ for i := 0; i < numJsons; i++ {+ key := fmt.Sprintf("root%d", i)+ jsons[i] = testContentResource{key: key, content: strings.Replace(testJSON, "ROOT_KEY", key, 1), mime: media.JSONType}+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {+ result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
+ if err != nil {+ b.Fatal(err)
+ }
+ if result == nil {+ b.Fatal("no result")+ }
+ }
+}