ref: 129c27ee6e9fed98dbfebeaa272fd52757b475b2
parent: 44da60d869578423dea529db62ed613588a2a560
author: Bjørn Erik Pedersen <[email protected]>
date: Sat Oct 20 07:16:18 EDT 2018
parser/metadecoders: Consolidate the metadata decoders See #5324
--- a/commands/convert.go
+++ b/commands/convert.go
@@ -14,8 +14,19 @@
package commands
import (
+ "bytes"
+ "fmt"
+ "strings"
"time"
+ "github.com/gohugoio/hugo/hugofs"
+
+ "github.com/gohugoio/hugo/helpers"
+
+ "github.com/gohugoio/hugo/parser"
+ "github.com/gohugoio/hugo/parser/metadecoders"
+ "github.com/gohugoio/hugo/parser/pageparser"
+
src "github.com/gohugoio/hugo/source"
"github.com/pkg/errors"
@@ -23,7 +34,6 @@
"path/filepath"
- "github.com/gohugoio/hugo/parser"
"github.com/spf13/cast"
"github.com/spf13/cobra"
)
@@ -60,7 +70,7 @@
Long: `toJSON converts all front matter in the content directory
to use JSON for the front matter.`,
RunE: func(cmd *cobra.Command, args []string) error {- return cc.convertContents(rune([]byte(parser.JSONLead)[0]))
+ return cc.convertContents(metadecoders.JSON)
},
},
&cobra.Command{@@ -69,7 +79,7 @@
Long: `toTOML converts all front matter in the content directory
to use TOML for the front matter.`,
RunE: func(cmd *cobra.Command, args []string) error {- return cc.convertContents(rune([]byte(parser.TOMLLead)[0]))
+ return cc.convertContents(metadecoders.TOML)
},
},
&cobra.Command{@@ -78,7 +88,7 @@
Long: `toYAML converts all front matter in the content directory
to use YAML for the front matter.`,
RunE: func(cmd *cobra.Command, args []string) error {- return cc.convertContents(rune([]byte(parser.YAMLLead)[0]))
+ return cc.convertContents(metadecoders.YAML)
},
},
)
@@ -91,7 +101,7 @@
return cc
}
-func (cc *convertCmd) convertContents(mark rune) error {+func (cc *convertCmd) convertContents(format metadecoders.Format) error { if cc.outputDir == "" && !cc.unsafe { return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")}
@@ -114,7 +124,7 @@
site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files") for _, p := range site.AllPages {- if err := cc.convertAndSavePage(p, site, mark); err != nil {+ if err := cc.convertAndSavePage(p, site, format); err != nil {return err
}
}
@@ -121,10 +131,10 @@
return nil
}
-func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {+func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, targetFormat metadecoders.Format) error {// The resources are not in .Site.AllPages.
for _, r := range p.Resources.ByType("page") {- if err := cc.convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil {+ if err := cc.convertAndSavePage(r.(*hugolib.Page), site, targetFormat); err != nil {return err
}
}
@@ -134,23 +144,21 @@
return nil
}
+ errMsg := fmt.Errorf("Error processing file %q", p.Path())+
site.Log.INFO.Println("Attempting to convert", p.LogicalName())- newPage, err := site.NewPage(p.LogicalName())
- if err != nil {- return err
- }
f, _ := p.File.(src.ReadableFile)
file, err := f.Open()
if err != nil {- site.Log.ERROR.Println("Error reading file:", p.Path())+ site.Log.ERROR.Println(errMsg)
file.Close()
return nil
}
- psr, err := parser.ReadFrom(file)
+ psr, err := pageparser.Parse(file)
if err != nil {- site.Log.ERROR.Println("Error processing file:", p.Path())+ site.Log.ERROR.Println(errMsg)
file.Close()
return err
}
@@ -157,14 +165,35 @@
file.Close()
- metadata, err := psr.Metadata()
+ var sourceFormat, sourceContent []byte
+ var fromFormat metadecoders.Format
+
+ iter := psr.Iterator()
+
+ walkFn := func(item pageparser.Item) bool {+ if sourceFormat != nil {+ // The rest is content.
+ sourceContent = psr.Input()[item.Pos:]
+ // Done
+ return false
+ } else if item.IsFrontMatter() {+ fromFormat = metadecoders.FormatFromFrontMatterType(item.Type)
+ sourceFormat = item.Val
+ }
+ return true
+
+ }
+
+ iter.PeekWalk(walkFn)
+
+ metadata, err := metadecoders.UnmarshalToMap(sourceFormat, fromFormat)
if err != nil {- site.Log.ERROR.Println("Error processing file:", p.Path())+ site.Log.ERROR.Println(errMsg)
return err
}
// better handling of dates in formats that don't have support for them
- if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {+ if fromFormat == metadecoders.JSON || fromFormat == metadecoders.YAML || fromFormat == metadecoders.TOML {newMetadata := cast.ToStringMap(metadata)
for k, v := range newMetadata { switch vv := v.(type) {@@ -175,18 +204,26 @@
metadata = newMetadata
}
- newPage.SetSourceContent(psr.Content())
- if err = newPage.SetSourceMetaData(metadata, mark); err != nil {- site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err)- return nil
+ var newContent bytes.Buffer
+ err = parser.InterfaceToFrontMatter2(metadata, targetFormat, &newContent)
+ if err != nil {+ site.Log.ERROR.Println(errMsg)
+ return err
}
+ newContent.Write(sourceContent)
+
newFilename := p.Filename()
+
if cc.outputDir != "" {- newFilename = filepath.Join(cc.outputDir, p.Dir(), newPage.LogicalName())
+ contentDir := strings.TrimSuffix(newFilename, p.Path())
+ contentDir = filepath.Base(contentDir)
+
+ newFilename = filepath.Join(cc.outputDir, contentDir, p.Path())
}
- if err = newPage.SaveSourceAs(newFilename); err != nil {+ fs := hugofs.Os
+ if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil {return errors.Wrapf(err, "Failed to save file %q:", newFilename)
}
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -42,7 +42,7 @@
"github.com/gohugoio/hugo/config"
- "github.com/gohugoio/hugo/parser"
+ "github.com/gohugoio/hugo/parser/metadecoders"
flag "github.com/spf13/pflag"
"github.com/fsnotify/fsnotify"
@@ -1017,7 +1017,7 @@
b, err := afero.ReadFile(fs, path)
- tomlMeta, err := parser.HandleTOMLMetaData(b)
+ tomlMeta, err := metadecoders.UnmarshalToMap(b, metadecoders.TOML)
if err != nil {continue
--- a/commands/import_jekyll.go
+++ b/commands/import_jekyll.go
@@ -25,6 +25,8 @@
"strings"
"time"
+ "github.com/gohugoio/hugo/parser/metadecoders"
+
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib"
@@ -253,7 +255,7 @@
return nil
}
- c, err := parser.HandleYAMLMetaData(b)
+ c, err := metadecoders.UnmarshalToMap(b, metadecoders.YAML)
if err != nil {return nil
--- a/go.mod
+++ b/go.mod
@@ -51,7 +51,7 @@
github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d
github.com/spf13/pflag v1.0.2
github.com/spf13/viper v1.2.0
- github.com/stretchr/testify v1.2.2
+ github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9
github.com/tdewolff/minify v2.3.5+incompatible
github.com/tdewolff/parse v2.3.3+incompatible // indirect
github.com/tdewolff/test v0.0.0-20171106182207-265427085153 // indirect
@@ -63,6 +63,7 @@
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect
golang.org/x/text v0.3.0
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+ gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
gopkg.in/yaml.v2 v2.2.1
)
--- a/go.sum
+++ b/go.sum
@@ -65,6 +65,7 @@
github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c=
github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -118,6 +119,8 @@
github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9 h1:kcVw9CGDqYBy0TTpIq2+BNR4W9poqiwEPBh/OYX5CaU=
+github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/tdewolff/minify v2.3.5+incompatible h1:oFxBKxTIY1F/1DEJhLeh/T507W56JqZtWVrawFcdadI=
github.com/tdewolff/minify v2.3.5+incompatible/go.mod h1:9Ov578KJUmAWpS6NeZwRZyT56Uf6o3Mcz9CEsg8USYs=
github.com/tdewolff/parse v2.3.3+incompatible h1:q6OSjvHtvBucLb34z24OH1xl5wGdw1mI9Vd38Qj9evs=
@@ -143,5 +146,7 @@
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -44,6 +44,7 @@
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/parser/metadecoders"
"github.com/markbates/inflect"
@@ -53,7 +54,6 @@
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugolib/pagemeta"
"github.com/gohugoio/hugo/output"
- "github.com/gohugoio/hugo/parser"
"github.com/gohugoio/hugo/related"
"github.com/gohugoio/hugo/source"
"github.com/gohugoio/hugo/tpl"
@@ -949,16 +949,8 @@
defer file.Close()
content := helpers.ReaderToBytes(file)
- switch f.Extension() {- case "yaml", "yml":
- return parser.HandleYAMLData(content)
- case "json":
- return parser.HandleJSONData(content)
- case "toml":
- return parser.HandleTOMLMetaData(content)
- default:
- return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension())- }
+ format := metadecoders.FormatFromString(f.Extension())
+ return metadecoders.Unmarshal(content, format)
}
func (s *Site) readDataFromSourceFS() error {--- a/parser/frontmatter.go
+++ b/parser/frontmatter.go
@@ -19,16 +19,12 @@
"bytes"
"encoding/json"
"errors"
- "fmt"
"io"
"strings"
- "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/parser/metadecoders"
- "github.com/spf13/cast"
-
"github.com/BurntSushi/toml"
- "github.com/chaseadamsio/goorgeous"
"gopkg.in/yaml.v2"
)
@@ -79,6 +75,82 @@
}
}
+func InterfaceToConfig2(in interface{}, format metadecoders.Format, w io.Writer) error {+ if in == nil {+ return errors.New("input was nil")+ }
+
+ switch format {+ case metadecoders.YAML:
+ b, err := yaml.Marshal(in)
+ if err != nil {+ return err
+ }
+
+ _, err = w.Write(b)
+ return err
+
+ case metadecoders.TOML:
+ return toml.NewEncoder(w).Encode(in)
+ case metadecoders.JSON:
+ b, err := json.MarshalIndent(in, "", " ")
+ if err != nil {+ return err
+ }
+
+ _, err = w.Write(b)
+ if err != nil {+ return err
+ }
+
+ _, err = w.Write([]byte{'\n'})+ return err
+
+ default:
+ return errors.New("Unsupported Format provided")+ }
+}
+
+func InterfaceToFrontMatter2(in interface{}, format metadecoders.Format, w io.Writer) error {+ if in == nil {+ return errors.New("input was nil")+ }
+
+ switch format {+ case metadecoders.YAML:
+ _, err := w.Write([]byte(YAMLDelimUnix))
+ if err != nil {+ return err
+ }
+
+ err = InterfaceToConfig2(in, format, w)
+ if err != nil {+ return err
+ }
+
+ _, err = w.Write([]byte(YAMLDelimUnix))
+ return err
+
+ case metadecoders.TOML:
+ _, err := w.Write([]byte(TOMLDelimUnix))
+ if err != nil {+ return err
+ }
+
+ err = InterfaceToConfig2(in, format, w)
+
+ if err != nil {+ return err
+ }
+
+ _, err = w.Write([]byte("\n" + TOMLDelimUnix))+ return err
+
+ default:
+ return InterfaceToConfig2(in, format, w)
+ }
+}
+
// InterfaceToFrontMatter encodes a given input into a frontmatter
// representation based upon the mark with the appropriate front matter delimiters
// surrounding the output, which is written to w.
@@ -155,34 +227,6 @@
}
}
-// DetectFrontMatter detects the type of frontmatter analysing its first character.
-func DetectFrontMatter(mark rune) (f *FrontmatterType) {- switch mark {- case '-':
- return &FrontmatterType{HandleYAMLMetaData, []byte(YAMLDelim), []byte(YAMLDelim), false}- case '+':
- return &FrontmatterType{HandleTOMLMetaData, []byte(TOMLDelim), []byte(TOMLDelim), false}- case '{':- return &FrontmatterType{HandleJSONMetaData, []byte{'{'}, []byte{'}'}, true}- case '#':
- return &FrontmatterType{HandleOrgMetaData, []byte("#+"), []byte("\n"), false}- default:
- return nil
- }
-}
-
-// HandleTOMLMetaData unmarshals TOML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleTOMLMetaData(datum []byte) (map[string]interface{}, error) {- m := map[string]interface{}{}- datum = removeTOMLIdentifier(datum)
-
- _, err := toml.Decode(string(datum), &m)
-
- return m, err
-
-}
-
// removeTOMLIdentifier removes, if necessary, beginning and ending TOML
// frontmatter delimiters from a byte slice.
func removeTOMLIdentifier(datum []byte) []byte {@@ -199,126 +243,4 @@
b = bytes.Trim(b, "\r\n")
return bytes.TrimSuffix(b, []byte(TOMLDelim))
-}
-
-// HandleYAMLMetaData unmarshals YAML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-// TODO(bep) 2errors remove these handlers (and hopefully package)
-func HandleYAMLMetaData(datum []byte) (map[string]interface{}, error) {- m := map[string]interface{}{}- err := yaml.Unmarshal(datum, &m)
-
- // To support boolean keys, the `yaml` package unmarshals maps to
- // map[interface{}]interface{}. Here we recurse through the result- // and change all maps to map[string]interface{} like we would've- // gotten from `json`.
- if err == nil {- for k, v := range m {- if vv, changed := stringifyMapKeys(v); changed {- m[k] = vv
- }
- }
- }
-
- return m, err
-}
-
-// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleYAMLData(datum []byte) (interface{}, error) {- var m interface{}- err := yaml.Unmarshal(datum, &m)
- if err != nil {- return nil, err
- }
-
- // To support boolean keys, the `yaml` package unmarshals maps to
- // map[interface{}]interface{}. Here we recurse through the result- // and change all maps to map[string]interface{} like we would've- // gotten from `json`.
- if mm, changed := stringifyMapKeys(m); changed {- return mm, nil
- }
-
- return m, nil
-}
-
-// stringifyMapKeys recurses into in and changes all instances of
-// map[interface{}]interface{} to map[string]interface{}. This is useful to-// work around the impedence mismatch between JSON and YAML unmarshaling that's
-// described here: https://github.com/go-yaml/yaml/issues/139
-//
-// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
-func stringifyMapKeys(in interface{}) (interface{}, bool) {- switch in := in.(type) {- case []interface{}:- for i, v := range in {- if vv, replaced := stringifyMapKeys(v); replaced {- in[i] = vv
- }
- }
- case map[interface{}]interface{}:- res := make(map[string]interface{})- var (
- ok bool
- err error
- )
- for k, v := range in {- var ks string
-
- if ks, ok = k.(string); !ok {- ks, err = cast.ToStringE(k)
- if err != nil {- ks = fmt.Sprintf("%v", k)- }
- // TODO(bep) added in Hugo 0.37, remove some time in the future.
- helpers.DistinctFeedbackLog.Printf("WARNING: YAML data/frontmatter with keys of type %T is since Hugo 0.37 converted to strings", k)- }
- if vv, replaced := stringifyMapKeys(v); replaced {- res[ks] = vv
- } else {- res[ks] = v
- }
- }
- return res, true
- }
-
- return nil, false
-}
-
-// HandleJSONMetaData unmarshals JSON-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleJSONMetaData(datum []byte) (map[string]interface{}, error) {- m := make(map[string]interface{})-
- if datum == nil {- // Package json returns on error on nil input.
- // Return an empty map to be consistent with our other supported
- // formats.
- return m, nil
- }
-
- err := json.Unmarshal(datum, &m)
- return m, err
-}
-
-// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleJSONData(datum []byte) (interface{}, error) {- if datum == nil {- // Package json returns on error on nil input.
- // Return an empty map to be consistent with our other supported
- // formats.
- return make(map[string]interface{}), nil- }
-
- var f interface{}- err := json.Unmarshal(datum, &f)
- return f, err
-}
-
-// HandleOrgMetaData unmarshals org-mode encoded datum and returns a Go
-// interface representing the encoded data structure.
-func HandleOrgMetaData(datum []byte) (map[string]interface{}, error) {- return goorgeous.OrgHeaders(datum)
}
--- a/parser/frontmatter_test.go
+++ b/parser/frontmatter_test.go
@@ -132,116 +132,6 @@
}
}
-func TestHandleTOMLMetaData(t *testing.T) {- cases := []struct {- input []byte
- want interface{}- isErr bool
- }{- {nil, map[string]interface{}{}, false},- {[]byte("title = \"test 1\""), map[string]interface{}{"title": "test 1"}, false},- {[]byte("a = [1, 2, 3]"), map[string]interface{}{"a": []interface{}{int64(1), int64(2), int64(3)}}, false},- {[]byte("b = [\n[1, 2],\n[3, 4]\n]"), map[string]interface{}{"b": []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, false},- // errors
- {[]byte("z = [\n[1, 2]\n[3, 4]\n]"), nil, true},- }
-
- for i, c := range cases {- res, err := HandleTOMLMetaData(c.input)
- if err != nil {- if c.isErr {- continue
- }
- t.Fatalf("[%d] unexpected error value: %v", i, err)- }
-
- if !reflect.DeepEqual(res, c.want) {- t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)- }
- }
-}
-
-func TestHandleYAMLMetaData(t *testing.T) {- cases := []struct {- input []byte
- want interface{}- isErr bool
- }{- {nil, map[string]interface{}{}, false},- {[]byte("title: test 1"), map[string]interface{}{"title": "test 1"}, false},- {[]byte("a: Easy!\nb:\n c: 2\n d: [3, 4]"), map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}, false},- {[]byte("a:\n true: 1\n false: 2"), map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}, false},- // errors
- {[]byte("z = not toml"), nil, true},- }
-
- for i, c := range cases {- res, err := HandleYAMLMetaData(c.input)
- if err != nil {- if c.isErr {- continue
- }
- t.Fatalf("[%d] unexpected error value: %v", i, err)- }
-
- if !reflect.DeepEqual(res, c.want) {- t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)- }
- }
-}
-
-func TestHandleJSONMetaData(t *testing.T) {- cases := []struct {- input []byte
- want interface{}- isErr bool
- }{- {nil, map[string]interface{}{}, false},- {[]byte("{\"title\": \"test 1\"}"), map[string]interface{}{"title": "test 1"}, false},- // errors
- {[]byte("{noquotes}"), nil, true},- }
-
- for i, c := range cases {- res, err := HandleJSONMetaData(c.input)
- if err != nil {- if c.isErr {- continue
- }
- t.Fatalf("[%d] unexpected error value: %v", i, err)- }
-
- if !reflect.DeepEqual(res, c.want) {- t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)- }
- }
-}
-
-func TestHandleOrgMetaData(t *testing.T) {- cases := []struct {- input []byte
- want interface{}- isErr bool
- }{- {nil, map[string]interface{}{}, false},- {[]byte("#+title: test 1\n"), map[string]interface{}{"title": "test 1"}, false},- }
-
- for i, c := range cases {- res, err := HandleOrgMetaData(c.input)
- if err != nil {- if c.isErr {- continue
- }
- t.Fatalf("[%d] unexpected error value: %v", i, err)- }
-
- if !reflect.DeepEqual(res, c.want) {- t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)- }
- }
-}
-
func TestFormatToLeadRune(t *testing.T) { for i, this := range []struct {kind string
@@ -264,41 +154,6 @@
}
}
-func TestDetectFrontMatter(t *testing.T) {- cases := []struct {- mark rune
- want *FrontmatterType
- }{- // funcs are uncomparable, so we ignore FrontmatterType.Parse in these tests
- {'-', &FrontmatterType{nil, []byte(YAMLDelim), []byte(YAMLDelim), false}},- {'+', &FrontmatterType{nil, []byte(TOMLDelim), []byte(TOMLDelim), false}},- {'{', &FrontmatterType{nil, []byte("{"), []byte("}"), true}},- {'#', &FrontmatterType{nil, []byte("#+"), []byte("\n"), false}},- {'$', nil},- }
-
- for _, c := range cases {- res := DetectFrontMatter(c.mark)
- if res == nil {- if c.want == nil {- continue
- }
-
- t.Fatalf("want %v, got %v", *c.want, res)- }
-
- if !reflect.DeepEqual(res.markstart, c.want.markstart) {- t.Errorf("markstart mismatch: want %v, got %v", c.want.markstart, res.markstart)- }
- if !reflect.DeepEqual(res.markend, c.want.markend) {- t.Errorf("markend mismatch: want %v, got %v", c.want.markend, res.markend)- }
- if !reflect.DeepEqual(res.includeMark, c.want.includeMark) {- t.Errorf("includeMark mismatch: want %v, got %v", c.want.includeMark, res.includeMark)- }
- }
-}
-
func TestRemoveTOMLIdentifier(t *testing.T) { cases := []struct {input string
@@ -321,64 +176,6 @@
}
}
-func TestStringifyYAMLMapKeys(t *testing.T) {- cases := []struct {- input interface{}- want interface{}- replaced bool
- }{- {- map[interface{}]interface{}{"a": 1, "b": 2},- map[string]interface{}{"a": 1, "b": 2},- true,
- },
- {- map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},- map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},- true,
- },
- {- map[interface{}]interface{}{true: 1, "b": false},- map[string]interface{}{"true": 1, "b": false},- true,
- },
- {- map[interface{}]interface{}{1: "a", 2: "b"},- map[string]interface{}{"1": "a", "2": "b"},- true,
- },
- {- map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},- map[string]interface{}{"a": map[string]interface{}{"b": 1}},- true,
- },
- {- map[string]interface{}{"a": map[string]interface{}{"b": 1}},- map[string]interface{}{"a": map[string]interface{}{"b": 1}},- false,
- },
- {- []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},- []interface{}{map[string]interface{}{"1": "a", "2": "b"}},- false,
- },
- }
-
- for i, c := range cases {- res, replaced := stringifyMapKeys(c.input)
-
- if c.replaced != replaced {- t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)- }
- if !c.replaced {- res = c.input
- }
- if !reflect.DeepEqual(res, c.want) {- t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)- }
- }
-}
-
func BenchmarkFrontmatterTags(b *testing.B) { for _, frontmatter := range []string{"JSON", "YAML", "YAML2", "TOML"} {@@ -388,69 +185,6 @@
}
}
-func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {- maps := make([]map[interface{}]interface{}, b.N)- for i := 0; i < b.N; i++ {- maps[i] = map[interface{}]interface{}{- "a": map[interface{}]interface{}{- "b": 32,
- "c": 43,
- "d": map[interface{}]interface{}{- "b": 32,
- "c": 43,
- },
- },
- "b": []interface{}{"a", "b"},- "c": "d",
- }
- }
- b.ResetTimer()
- for i := 0; i < b.N; i++ {- stringifyMapKeys(maps[i])
- }
-}
-
-func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {- m := map[string]interface{}{- "a": map[string]interface{}{- "b": 32,
- "c": 43,
- "d": map[string]interface{}{- "b": 32,
- "c": 43,
- },
- },
- "b": []interface{}{"a", "b"},- "c": "d",
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {- stringifyMapKeys(m)
- }
-}
-
-func BenchmarkStringifyMapKeysIntegers(b *testing.B) {- maps := make([]map[interface{}]interface{}, b.N)- for i := 0; i < b.N; i++ {- maps[i] = map[interface{}]interface{}{- 1: map[interface{}]interface{}{- 4: 32,
- 5: 43,
- 6: map[interface{}]interface{}{- 7: 32,
- 8: 43,
- },
- },
- 2: []interface{}{"a", "b"},- 3: "d",
- }
- }
- b.ResetTimer()
- for i := 0; i < b.N; i++ {- stringifyMapKeys(maps[i])
- }
-}
func doBenchmarkFrontmatter(b *testing.B, fileformat string, numTags int) {yamlTemplate := `---
name: "Tags"
--- a/parser/metadecoders/decoder.go
+++ b/parser/metadecoders/decoder.go
@@ -15,81 +15,139 @@
import (
"encoding/json"
+ "fmt"
"github.com/BurntSushi/toml"
"github.com/chaseadamsio/goorgeous"
- "github.com/gohugoio/hugo/parser/pageparser"
"github.com/pkg/errors"
+ "github.com/spf13/cast"
yaml "gopkg.in/yaml.v2"
)
-type Format string
-
-const (
- // These are the supported metdata formats in Hugo. Most of these are also
- // supported as /data formats.
- ORG Format = "org"
- JSON Format = "json"
- TOML Format = "toml"
- YAML Format = "yaml"
-)
-
-// FormatFromFrontMatterType will return empty if not supported.
-func FormatFromFrontMatterType(typ pageparser.ItemType) Format {- switch typ {- case pageparser.TypeFrontMatterJSON:
- return JSON
- case pageparser.TypeFrontMatterORG:
- return ORG
- case pageparser.TypeFrontMatterTOML:
- return TOML
- case pageparser.TypeFrontMatterYAML:
- return YAML
- default:
- return ""
- }
-}
-
// UnmarshalToMap will unmarshall data in format f into a new map. This is
// what's needed for Hugo's front matter decoding.
func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) { m := make(map[string]interface{})-
if data == nil {return m, nil
}
+ err := unmarshal(data, f, &m)
+
+ return m, err
+
+}
+
+// Unmarshal will unmarshall data in format f into an interface{}.+// This is what's needed for Hugo's /data handling.
+func Unmarshal(data []byte, f Format) (interface{}, error) {+ if data == nil {+ return make(map[string]interface{}), nil+ }
+ var v interface{}+ err := unmarshal(data, f, &v)
+
+ return v, err
+}
+
+// unmarshal unmarshals data in format f into v.
+func unmarshal(data []byte, f Format, v interface{}) error {+
var err error
switch f {case ORG:
- m, err = goorgeous.OrgHeaders(data)
+ vv, err := goorgeous.OrgHeaders(data)
+ if err != nil {+ return err
+ }
+ switch v.(type) {+ case *map[string]interface{}:+ *v.(*map[string]interface{}) = vv+ default:
+ *v.(*interface{}) = vv+ }
case JSON:
- err = json.Unmarshal(data, &m)
+ err = json.Unmarshal(data, v)
case TOML:
- _, err = toml.Decode(string(data), &m)
+ err = toml.Unmarshal(data, v)
case YAML:
- err = yaml.Unmarshal(data, &m)
+ err = yaml.Unmarshal(data, v)
- // To support boolean keys, the `yaml` package unmarshals maps to
+ // To support boolean keys, the YAML package unmarshals maps to
// map[interface{}]interface{}. Here we recurse through the result // and change all maps to map[string]interface{} like we would've// gotten from `json`.
- if err == nil {- for k, v := range m {- if vv, changed := stringifyMapKeys(v); changed {- m[k] = vv
- }
+ var ptr interface{}+ switch v.(type) {+ case *map[string]interface{}:+ ptr = *v.(*map[string]interface{})+ case *interface{}:+ ptr = *v.(*interface{})+ default:
+ return errors.Errorf("unknown type %T in YAML unmarshal", v)+ }
+
+ if mm, changed := stringifyMapKeys(ptr); changed {+ switch v.(type) {+ case *map[string]interface{}:+ *v.(*map[string]interface{}) = mm.(map[string]interface{})+ case *interface{}:+ *v.(*interface{}) = mm}
}
default:
- return nil, errors.Errorf("unmarshal of format %q is not supported", f)+ return errors.Errorf("unmarshal of format %q is not supported", f)}
- if err != nil {- return nil, errors.Wrapf(err, "unmarshal failed for format %q", f)
- }
+ return err
- return m, nil
+}
+// stringifyMapKeys recurses into in and changes all instances of
+// map[interface{}]interface{} to map[string]interface{}. This is useful to+// work around the impedence mismatch between JSON and YAML unmarshaling that's
+// described here: https://github.com/go-yaml/yaml/issues/139
+//
+// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
+func stringifyMapKeys(in interface{}) (interface{}, bool) {+
+ switch in := in.(type) {+ case []interface{}:+ for i, v := range in {+ if vv, replaced := stringifyMapKeys(v); replaced {+ in[i] = vv
+ }
+ }
+ case map[string]interface{}:+ for k, v := range in {+ if vv, changed := stringifyMapKeys(v); changed {+ in[k] = vv
+ }
+ }
+ case map[interface{}]interface{}:+ res := make(map[string]interface{})+ var (
+ ok bool
+ err error
+ )
+ for k, v := range in {+ var ks string
+
+ if ks, ok = k.(string); !ok {+ ks, err = cast.ToStringE(k)
+ if err != nil {+ ks = fmt.Sprintf("%v", k)+ }
+ }
+ if vv, replaced := stringifyMapKeys(v); replaced {+ res[ks] = vv
+ } else {+ res[ks] = v
+ }
+ }
+ return res, true
+ }
+
+ return nil, false
}
--- /dev/null
+++ b/parser/metadecoders/decoder_test.go
@@ -1,0 +1,207 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+ "fmt"
+ "reflect"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalToMap(t *testing.T) {+ assert := require.New(t)
+
+ expect := map[string]interface{}{"a": "b"}+
+ for i, test := range []struct {+ data string
+ format Format
+ expect interface{}+ }{+ {`a = "b"`, TOML, expect},+ {`a: "b"`, YAML, expect},+ // Make sure we get all string keys, even for YAML
+ {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},+ {"a:\n true: 1\n false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}},+ {`{ "a": "b" }`, JSON, expect},+ {`#+a: b`, ORG, expect},+ // errors
+ {`a = b`, TOML, false},+ } {+ msg := fmt.Sprintf("%d: %s", i, test.format)+ m, err := UnmarshalToMap([]byte(test.data), test.format)
+ if b, ok := test.expect.(bool); ok && !b {+ assert.Error(err, msg)
+ } else {+ assert.NoError(err, msg)
+ assert.Equal(test.expect, m, msg)
+ }
+ }
+}
+
+func TestUnmarshalToInterface(t *testing.T) {+ assert := require.New(t)
+
+ expect := map[string]interface{}{"a": "b"}+
+ for i, test := range []struct {+ data string
+ format Format
+ expect interface{}+ }{+ {`[ "Brecker", "Blake", "Redman" ]`, JSON, []interface{}{"Brecker", "Blake", "Redman"}},+ {`{ "a": "b" }`, JSON, expect},+ {`#+a: b`, ORG, expect},+ {`a = "b"`, TOML, expect},+ {`a: "b"`, YAML, expect},+ {"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},+ // errors
+ {`a = "`, TOML, false},+ } {+ msg := fmt.Sprintf("%d: %s", i, test.format)+ m, err := Unmarshal([]byte(test.data), test.format)
+ if b, ok := test.expect.(bool); ok && !b {+ assert.Error(err, msg)
+ } else {+ assert.NoError(err, msg)
+ assert.Equal(test.expect, m, msg)
+ }
+
+ }
+
+}
+
+func TestStringifyYAMLMapKeys(t *testing.T) {+ cases := []struct {+ input interface{}+ want interface{}+ replaced bool
+ }{+ {+ map[interface{}]interface{}{"a": 1, "b": 2},+ map[string]interface{}{"a": 1, "b": 2},+ true,
+ },
+ {+ map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},+ map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},+ true,
+ },
+ {+ map[interface{}]interface{}{true: 1, "b": false},+ map[string]interface{}{"true": 1, "b": false},+ true,
+ },
+ {+ map[interface{}]interface{}{1: "a", 2: "b"},+ map[string]interface{}{"1": "a", "2": "b"},+ true,
+ },
+ {+ map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},+ map[string]interface{}{"a": map[string]interface{}{"b": 1}},+ true,
+ },
+ {+ map[string]interface{}{"a": map[string]interface{}{"b": 1}},+ map[string]interface{}{"a": map[string]interface{}{"b": 1}},+ false,
+ },
+ {+ []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},+ []interface{}{map[string]interface{}{"1": "a", "2": "b"}},+ false,
+ },
+ }
+
+ for i, c := range cases {+ res, replaced := stringifyMapKeys(c.input)
+
+ if c.replaced != replaced {+ t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)+ }
+ if !c.replaced {+ res = c.input
+ }
+ if !reflect.DeepEqual(res, c.want) {+ t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)+ }
+ }
+}
+
+func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {+ maps := make([]map[interface{}]interface{}, b.N)+ for i := 0; i < b.N; i++ {+ maps[i] = map[interface{}]interface{}{+ "a": map[interface{}]interface{}{+ "b": 32,
+ "c": 43,
+ "d": map[interface{}]interface{}{+ "b": 32,
+ "c": 43,
+ },
+ },
+ "b": []interface{}{"a", "b"},+ "c": "d",
+ }
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {+ stringifyMapKeys(maps[i])
+ }
+}
+
+func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {+ m := map[string]interface{}{+ "a": map[string]interface{}{+ "b": 32,
+ "c": 43,
+ "d": map[string]interface{}{+ "b": 32,
+ "c": 43,
+ },
+ },
+ "b": []interface{}{"a", "b"},+ "c": "d",
+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {+ stringifyMapKeys(m)
+ }
+}
+
+func BenchmarkStringifyMapKeysIntegers(b *testing.B) {+ maps := make([]map[interface{}]interface{}, b.N)+ for i := 0; i < b.N; i++ {+ maps[i] = map[interface{}]interface{}{+ 1: map[interface{}]interface{}{+ 4: 32,
+ 5: 43,
+ 6: map[interface{}]interface{}{+ 7: 32,
+ 8: 43,
+ },
+ },
+ 2: []interface{}{"a", "b"},+ 3: "d",
+ }
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {+ stringifyMapKeys(maps[i])
+ }
+}
--- /dev/null
+++ b/parser/metadecoders/format.go
@@ -1,0 +1,66 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+ "strings"
+
+ "github.com/gohugoio/hugo/parser/pageparser"
+)
+
+type Format string
+
+const (
+ // These are the supported metdata formats in Hugo. Most of these are also
+ // supported as /data formats.
+ ORG Format = "org"
+ JSON Format = "json"
+ TOML Format = "toml"
+ YAML Format = "yaml"
+)
+
+// FormatFromString turns formatStr, typically a file extension without any ".",
+// into a Format. It returns an empty string for unknown formats.
+func FormatFromString(formatStr string) Format {+ formatStr = strings.ToLower(formatStr)
+ switch formatStr {+ case "yaml", "yml":
+ return YAML
+ case "json":
+ return JSON
+ case "toml":
+ return TOML
+ case "org":
+ return ORG
+ }
+
+ return ""
+
+}
+
+// FormatFromFrontMatterType will return empty if not supported.
+func FormatFromFrontMatterType(typ pageparser.ItemType) Format {+ switch typ {+ case pageparser.TypeFrontMatterJSON:
+ return JSON
+ case pageparser.TypeFrontMatterORG:
+ return ORG
+ case pageparser.TypeFrontMatterTOML:
+ return TOML
+ case pageparser.TypeFrontMatterYAML:
+ return YAML
+ default:
+ return ""
+ }
+}
--- /dev/null
+++ b/parser/metadecoders/format_test.go
@@ -1,0 +1,57 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/gohugoio/hugo/parser/pageparser"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestFormatFromString(t *testing.T) {+ assert := require.New(t)
+ for i, test := range []struct {+ s string
+ expect Format
+ }{+ {"json", JSON},+ {"yaml", YAML},+ {"yml", YAML},+ {"toml", TOML},+ {"tOMl", TOML},+ {"org", ORG},+ {"foo", ""},+ } {+ assert.Equal(test.expect, FormatFromString(test.s), fmt.Sprintf("t%d", i))+ }
+}
+
+func TestFormatFromFrontMatterType(t *testing.T) {+ assert := require.New(t)
+ for i, test := range []struct {+ typ pageparser.ItemType
+ expect Format
+ }{+ {pageparser.TypeFrontMatterJSON, JSON},+ {pageparser.TypeFrontMatterTOML, TOML},+ {pageparser.TypeFrontMatterYAML, YAML},+ {pageparser.TypeFrontMatterORG, ORG},+ {pageparser.TypeIgnore, ""},+ } {+ assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))+ }
+}
--- a/parser/metadecoders/json.go
+++ /dev/null
@@ -1,31 +1,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package metadecoders
-
-import "encoding/json"
-
-// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleJSONData(datum []byte) (interface{}, error) {- if datum == nil {- // Package json returns on error on nil input.
- // Return an empty map to be consistent with our other supported
- // formats.
- return make(map[string]interface{}), nil- }
-
- var f interface{}- err := json.Unmarshal(datum, &f)
- return f, err
-}
--- a/parser/metadecoders/yaml.go
+++ /dev/null
@@ -1,84 +1,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The metadecoders package contains functions to decode metadata (e.g. page front matter)
-// from different formats: TOML, YAML, JSON.
-package metadecoders
-
-import (
- "fmt"
-
- "github.com/spf13/cast"
- yaml "gopkg.in/yaml.v2"
-)
-
-// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleYAMLData(datum []byte) (interface{}, error) {- var m interface{}- err := yaml.Unmarshal(datum, &m)
- if err != nil {- return nil, err
- }
-
- // To support boolean keys, the `yaml` package unmarshals maps to
- // map[interface{}]interface{}. Here we recurse through the result- // and change all maps to map[string]interface{} like we would've- // gotten from `json`.
- if mm, changed := stringifyMapKeys(m); changed {- return mm, nil
- }
-
- return m, nil
-}
-
-// stringifyMapKeys recurses into in and changes all instances of
-// map[interface{}]interface{} to map[string]interface{}. This is useful to-// work around the impedence mismatch between JSON and YAML unmarshaling that's
-// described here: https://github.com/go-yaml/yaml/issues/139
-//
-// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
-func stringifyMapKeys(in interface{}) (interface{}, bool) {- switch in := in.(type) {- case []interface{}:- for i, v := range in {- if vv, replaced := stringifyMapKeys(v); replaced {- in[i] = vv
- }
- }
- case map[interface{}]interface{}:- res := make(map[string]interface{})- var (
- ok bool
- err error
- )
- for k, v := range in {- var ks string
-
- if ks, ok = k.(string); !ok {- ks, err = cast.ToStringE(k)
- if err != nil {- ks = fmt.Sprintf("%v", k)- }
- }
- if vv, replaced := stringifyMapKeys(v); replaced {- res[ks] = vv
- } else {- res[ks] = v
- }
- }
- return res, true
- }
-
- return nil, false
-}
--- a/parser/page.go
+++ b/parser/page.go
@@ -101,15 +101,8 @@
// Metadata returns the unmarshalled frontmatter data.
func (p *page) Metadata() (meta map[string]interface{}, err error) {- frontmatter := p.FrontMatter()
- if len(frontmatter) != 0 {- fm := DetectFrontMatter(rune(frontmatter[0]))
- if fm != nil {- meta, err = fm.Parse(frontmatter)
- }
- }
- return
+ return nil, nil
}
// ReadFrom reads the content from an io.Reader and constructs a page.
--- a/parser/page_test.go
+++ b/parser/page_test.go
@@ -1,130 +1,1 @@
package parser
-
-import (
- "fmt"
- "strings"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestPage(t *testing.T) {- cases := []struct {- raw string
-
- content string
- frontmatter string
- renderable bool
- metadata map[string]interface{}- }{- {- testPageLeader + jsonPageFrontMatter + "\n" + testPageTrailer + jsonPageContent,
- jsonPageContent,
- jsonPageFrontMatter,
- true,
- map[string]interface{}{- "title": "JSON Test 1",
- "social": []interface{}{- []interface{}{"a", "#"},- []interface{}{"b", "#"},- },
- },
- },
- {- testPageLeader + tomlPageFrontMatter + testPageTrailer + tomlPageContent,
- tomlPageContent,
- tomlPageFrontMatter,
- true,
- map[string]interface{}{- "title": "TOML Test 1",
- "social": []interface{}{- []interface{}{"a", "#"},- []interface{}{"b", "#"},- },
- },
- },
- {- testPageLeader + yamlPageFrontMatter + testPageTrailer + yamlPageContent,
- yamlPageContent,
- yamlPageFrontMatter,
- true,
- map[string]interface{}{- "title": "YAML Test 1",
- "social": []interface{}{- []interface{}{"a", "#"},- []interface{}{"b", "#"},- },
- },
- },
- {- testPageLeader + orgPageFrontMatter + orgPageContent,
- orgPageContent,
- orgPageFrontMatter,
- true,
- map[string]interface{}{- "TITLE": "Org Test 1",
- "categories": []string{"a", "b"},- },
- },
- }
-
- for i, c := range cases {- p := pageMust(ReadFrom(strings.NewReader(c.raw)))
- meta, err := p.Metadata()
-
- mesg := fmt.Sprintf("[%d]", i)-
- require.Nil(t, err, mesg)
- assert.Equal(t, c.content, string(p.Content()), mesg+" content")
- assert.Equal(t, c.frontmatter, string(p.FrontMatter()), mesg+" frontmatter")
- assert.Equal(t, c.renderable, p.IsRenderable(), mesg+" renderable")
- assert.Equal(t, c.metadata, meta, mesg+" metadata")
- }
-}
-
-var (
- testWhitespace = "\t\t\n\n"
- testPageLeader = "\ufeff" + testWhitespace + "<!--[metadata]>\n"
- testPageTrailer = "\n<![end-metadata]-->\n"
-
- jsonPageContent = "# JSON Test\n"
- jsonPageFrontMatter = `{- "title": "JSON Test 1",
- "social": [
- ["a", "#"],
- ["b", "#"]
- ]
-}`
-
- tomlPageContent = "# TOML Test\n"
- tomlPageFrontMatter = `+++
-title = "TOML Test 1"
-social = [
- ["a", "#"],
- ["b", "#"],
-]
-+++
-`
-
- yamlPageContent = "# YAML Test\n"
- yamlPageFrontMatter = `---
-title: YAML Test 1
-social:
- - - "a"
- - "#"
- - - "b"
- - "#"
----
-`
-
- orgPageContent = "* Org Test\n"
- orgPageFrontMatter = `#+TITLE: Org Test 1
-#+categories: a b
-`
-
- pageHTMLComment = `<!--
- This is a sample comment.
--->
-`
-)
--- a/parser/pageparser/item.go
+++ b/parser/pageparser/item.go
@@ -20,7 +20,7 @@
type Item struct {Type ItemType
- pos pos
+ Pos Pos
Val []byte
}
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -25,7 +25,7 @@
)
// position (in bytes)
-type pos int
+type Pos int
const eof = -1
@@ -47,9 +47,9 @@
input []byte
stateStart stateFunc
state stateFunc
- pos pos // input position
- start pos // item start position
- width pos // width of last element
+ pos Pos // input position
+ start Pos // item start position
+ width Pos // width of last element
// Set when we have parsed any summary divider
summaryDividerChecked bool
@@ -73,7 +73,7 @@
// note: the input position here is normally 0 (start), but
// can be set if position of first shortcode is known
// TODO(bep) 2errors byte
-func newPageLexer(input []byte, inputPosition pos, stateStart stateFunc) *pageLexer {+func newPageLexer(input []byte, inputPosition Pos, stateStart stateFunc) *pageLexer { lexer := &pageLexer{input: input,
pos: inputPosition,
@@ -131,7 +131,7 @@
}
runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:])
- l.width = pos(runeWidth)
+ l.width = Pos(runeWidth)
l.pos += l.width
return runeValue
}
@@ -210,7 +210,7 @@
l3 = l.index(leftDelimSc)
skip := minPositiveIndex(l1, l2, l3)
if skip > 0 {- l.pos += pos(skip)
+ l.pos += Pos(skip)
}
for {@@ -234,7 +234,7 @@
l.emit(tText)
}
l.summaryDividerChecked = true
- l.pos += pos(len(summaryDivider))
+ l.pos += Pos(len(summaryDivider))
//l.consumeCRLF()
l.emit(TypeLeadSummaryDivider)
} else if l.hasPrefix(summaryDividerOrg) {@@ -242,7 +242,7 @@
l.emit(tText)
}
l.summaryDividerChecked = true
- l.pos += pos(len(summaryDividerOrg))
+ l.pos += Pos(len(summaryDividerOrg))
//l.consumeCRLF()
l.emit(TypeSummaryDividerOrg)
}
@@ -291,12 +291,12 @@
if right == -1 { return l.errorf("starting HTML comment with no end")}
- l.pos += pos(right) + pos(len(htmlCOmmentEnd))
+ l.pos += Pos(right) + Pos(len(htmlCOmmentEnd))
l.emit(TypeHTMLComment)
} else {// Not need to look further. Hugo treats this as plain HTML,
// no front matter, no shortcodes, no nothing.
- l.pos = pos(len(l.input))
+ l.pos = Pos(len(l.input))
l.emit(TypeHTMLDocument)
}
}
@@ -434,7 +434,7 @@
}
func lexShortcodeLeftDelim(l *pageLexer) stateFunc {- l.pos += pos(len(l.currentLeftShortcodeDelim()))
+ l.pos += Pos(len(l.currentLeftShortcodeDelim()))
if l.hasPrefix(leftComment) {return lexShortcodeComment
}
@@ -451,13 +451,13 @@
}
// we emit all as text, except the comment markers
l.emit(tText)
- l.pos += pos(len(leftComment))
+ l.pos += Pos(len(leftComment))
l.ignore()
- l.pos += pos(posRightComment - len(leftComment))
+ l.pos += Pos(posRightComment - len(leftComment))
l.emit(tText)
- l.pos += pos(len(rightComment))
+ l.pos += Pos(len(rightComment))
l.ignore()
- l.pos += pos(len(l.currentRightShortcodeDelim()))
+ l.pos += Pos(len(l.currentRightShortcodeDelim()))
l.emit(tText)
return lexMainSection
}
@@ -464,7 +464,7 @@
func lexShortcodeRightDelim(l *pageLexer) stateFunc {l.closingState = 0
- l.pos += pos(len(l.currentRightShortcodeDelim()))
+ l.pos += Pos(len(l.currentRightShortcodeDelim()))
l.emit(l.currentRightShortcodeDelimItem())
return lexMainSection
}
--- a/parser/pageparser/pageparser.go
+++ b/parser/pageparser/pageparser.go
@@ -48,7 +48,7 @@
}
func parseMainSection(input []byte, from int) Result {- lexer := newPageLexer(input, pos(from), lexMainSection) // TODO(bep) 2errors
+ lexer := newPageLexer(input, Pos(from), lexMainSection) // TODO(bep) 2errors
lexer.run()
return lexer
}
@@ -57,7 +57,7 @@
// if needed.
type Iterator struct {l *pageLexer
- lastPos pos // position of the last item returned by nextItem
+ lastPos Pos // position of the last item returned by nextItem
}
// consumes and returns the next item
@@ -69,7 +69,7 @@
var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")} func (t *Iterator) current() Item {- if t.lastPos >= pos(len(t.l.items)) {+ if t.lastPos >= Pos(len(t.l.items)) {return errIndexOutOfBounds
}
return t.l.items[t.lastPos]
@@ -98,7 +98,7 @@
// PeekWalk will feed the next items in the iterator to walkFn
// until it returns false.
func (t *Iterator) PeekWalk(walkFn func(item Item) bool) {- for i := t.lastPos + 1; i < pos(len(t.l.items)); i++ {+ for i := t.lastPos + 1; i < Pos(len(t.l.items)); i++ {item := t.l.items[i]
if !walkFn(item) {break
@@ -120,5 +120,5 @@
// LineNumber returns the current line number. Used for logging.
func (t *Iterator) LineNumber() int {- return bytes.Count(t.l.input[:t.current().pos], lf) + 1
+ return bytes.Count(t.l.input[:t.current().Pos], lf) + 1
}
--- a/parser/pageparser/pageparser_intro_test.go
+++ b/parser/pageparser/pageparser_intro_test.go
@@ -59,9 +59,7 @@
{"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}}, {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}}, {"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, "\n"), tstSomeText, tstEOF}},-
{"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeHTMLComment, "<!--\n---\nfoo: \"bar\"\n---\n-->"), tstSomeText, tstEOF}},-
// Note that we keep all bytes as they are, but we need to handle CRLF
{"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}}, {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}},--- a/tpl/transform/remarshal.go
+++ b/tpl/transform/remarshal.go
@@ -6,6 +6,7 @@
"strings"
"github.com/gohugoio/hugo/parser"
+ "github.com/gohugoio/hugo/parser/metadecoders"
"github.com/spf13/cast"
)
@@ -38,22 +39,8 @@
return "", err
}
- var metaHandler func(d []byte) (map[string]interface{}, error)+ meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
- switch fromFormat {- case "yaml":
- metaHandler = parser.HandleYAMLMetaData
- case "toml":
- metaHandler = parser.HandleTOMLMetaData
- case "json":
- metaHandler = parser.HandleJSONMetaData
- }
-
- meta, err := metaHandler([]byte(from))
- if err != nil {- return "", err
- }
-
var result bytes.Buffer
if err := parser.InterfaceToConfig(meta, mark, &result); err != nil {return "", err
@@ -76,21 +63,21 @@
return 0, errors.New("failed to detect target data serialization format")}
-func detectFormat(data string) (string, error) {+func detectFormat(data string) (metadecoders.Format, error) { jsonIdx := strings.Index(data, "{")yamlIdx := strings.Index(data, ":")
tomlIdx := strings.Index(data, "=")
if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) {- return "json", nil
+ return metadecoders.JSON, nil
}
if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) {- return "yaml", nil
+ return metadecoders.YAML, nil
}
if tomlIdx != -1 {- return "toml", nil
+ return metadecoders.TOML, nil
}
return "", errors.New("failed to detect data serialization format")--- a/tpl/transform/remarshal_test.go
+++ b/tpl/transform/remarshal_test.go
@@ -18,6 +18,7 @@
"testing"
"github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/parser/metadecoders"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
@@ -179,12 +180,12 @@
data string
expect interface{} }{- {`foo = "bar"`, "toml"},- {` foo = "bar"`, "toml"},- {`foo="bar"`, "toml"},- {`foo: "bar"`, "yaml"},- {`foo:"bar"`, "yaml"},- {`{ "foo": "bar"`, "json"},+ {`foo = "bar"`, metadecoders.TOML},+ {` foo = "bar"`, metadecoders.TOML},+ {`foo="bar"`, metadecoders.TOML},+ {`foo: "bar"`, metadecoders.YAML},+ {`foo:"bar"`, metadecoders.YAML},+ {`{ "foo": "bar"`, metadecoders.JSON}, {`asdfasdf`, false}, {``, false}, } {@@ -198,6 +199,6 @@
}
assert.NoError(err, errMsg)
- assert.Equal(test.expect, result, errMsg)
+ assert.Equal(test.expect, result)
}
}