ref: 085ce15f7c041ba71772a0cd6eed4f06b6b0ca34
parent: 274d324c8bb818a76ffc5c35afcc33f4cf9eb5c3
author: Noah Campbell <[email protected]>
date: Mon Aug 5 03:53:58 EDT 2013
Adding ability to read from io.Reader This allows for testing without relying on the file system. Parsing algorithm to not read the entire file into memory.
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -14,12 +14,15 @@
package hugolib
import (
+ "bufio"
"bytes"
"encoding/json"
+ "errors"
"fmt"
"github.com/BurntSushi/toml"
"github.com/theplant/blackfriday"
"html/template"
+ "io"
"io/ioutil"
"launchpad.net/goyaml"
"os"
@@ -28,6 +31,7 @@
"sort"
"strings"
"time"
+ "unicode"
)
var _ = filepath.Base("")@@ -126,6 +130,22 @@
return strings.ToLower(page.Type()) + "/" + layout + ".html"
}
+func ReadFrom(buf io.Reader, name string) (page *Page, err error) {+ if len(name) == 0 {+ return nil, errors.New("Zero length page name")+ }
+
+ p := initializePage(name)
+
+ if err = p.parse(buf); err != nil {+ return
+ }
+
+ p.analyzePage()
+
+ return &p, nil
+}
+
// TODO should return errors as well
// TODO new page should return just a page
// TODO initalize separately... load from reader (file, or []byte)
@@ -133,7 +153,6 @@
p := initializePage(filename)
if err := p.buildPageFromFile(); err != nil {fmt.Println(err)
- os.Exit(1)
}
p.analyzePage()
@@ -146,49 +165,6 @@
p.FuzzyWordCount = int((p.WordCount+100)/100) * 100
}
-// TODO //rewrite to use byte methods instead
-func (page *Page) parseYamlMetaData(data []byte) ([]string, error) {- var err error
-
- datum, lines := splitPageContent(data, "---", "---")
- d, err := page.handleYamlMetaData([]byte(strings.Join(datum, "\n")))
-
- if err != nil {- return lines, err
- }
-
- err = page.handleMetaData(d)
- return lines, err
-}
-
-func (page *Page) parseTomlMetaData(data []byte) ([]string, error) {- var err error
-
- datum, lines := splitPageContent(data, "+++", "+++")
- d, err := page.handleTomlMetaData([]byte(strings.Join(datum, "\n")))
-
- if err != nil {- return lines, err
- }
-
- err = page.handleMetaData(d)
- return lines, err
-}
-
-func (page *Page) parseJsonMetaData(data []byte) ([]string, error) {- var err error
-
- datum, lines := splitPageContent(data, "{", "}")- d, err := page.handleJsonMetaData([]byte(strings.Join(datum, "\n")))
-
- if err != nil {- return lines, err
- }
-
- err = page.handleMetaData(d)
- return lines, err
-}
-
func splitPageContent(data []byte, start string, end string) ([]string, []string) {lines := strings.Split(string(data), "\n")
datum := lines[0:]
@@ -211,18 +187,6 @@
break
}
}
- } else { // Start token & end token are the same- for i, line := range lines {- if found == 1 && strings.HasPrefix(line, end) {- datum = lines[1:i]
- lines = lines[i+1:]
- break
- }
-
- if found == 0 && strings.HasPrefix(line, start) {- found = 1
- }
- }
}
return datum, lines
}
@@ -272,7 +236,7 @@
return f, nil
}
-func (page *Page) handleMetaData(f interface{}) error {+func (page *Page) update(f interface{}) error { m := f.(map[string]interface{}) for k, v := range m {@@ -304,7 +268,6 @@
page.Status = interfaceToString(v)
default:
// If not one of the explicit values, store in Params
- //fmt.Println(strings.ToLower(k))
switch vv := v.(type) {case string: // handle string values
page.Params[strings.ToLower(k)] = vv
@@ -340,25 +303,106 @@
return nil
}
-func (page *Page) Err(message string) {- fmt.Println(page.FileName + " : " + message)
+// TODO return error on last line instead of nil
+func (page *Page) parseFrontMatter(data *bufio.Reader) (err error) {+
+ if err = checkEmpty(data); err != nil {+ return err
+ }
+
+ var mark rune
+ if mark, err = chompWhitespace(data); err != nil {+ return err
+ }
+
+ f := page.detectFrontMatter(mark)
+ if f == nil {+ return errors.New("unable to match beginning front matter delimiter")+ }
+
+ if found, err := beginFrontMatter(data, f); err != nil || !found {+ return errors.New("unable to match beginning front matter delimiter")+ }
+
+ var frontmatter = new(bytes.Buffer)
+ for {+ line, _, err := data.ReadLine()
+ if err != nil {+ if err == io.EOF {+ return errors.New("unable to match ending front matter delimiter")+ }
+ return err
+ }
+ if bytes.Equal(line, f.markend) {+ break
+ }
+ frontmatter.Write(line)
+ frontmatter.Write([]byte{'\n'})+ }
+
+ metadata, err := f.parse(frontmatter.Bytes())
+ if err != nil {+ return err
+ }
+
+ if err = page.update(metadata); err != nil {+ return err
+ }
+
+ return
}
-// TODO return error on last line instead of nil
-func (page *Page) parseFileHeading(data []byte) ([]string, error) {- if len(data) == 0 {- page.Err("Empty File, skipping")- } else {- switch data[0] {- case '{':- return page.parseJsonMetaData(data)
- case '-':
- return page.parseYamlMetaData(data)
- case '+':
- return page.parseTomlMetaData(data)
+func checkEmpty(data *bufio.Reader) (err error) {+ if _, _, err = data.ReadRune(); err != nil {+ return errors.New("unable to locate front matter")+ }
+ if err = data.UnreadRune(); err != nil {+ return errors.New("unable to unread first charactor in page buffer.")+ }
+ return
+}
+
+type frontmatterType struct {+ markstart, markend []byte
+ parse func([]byte) (interface{}, error)+}
+
+func (page *Page) detectFrontMatter(mark rune) (f *frontmatterType) {+ switch mark {+ case '-':
+ return &frontmatterType{[]byte{'-', '-', '-'}, []byte{'-', '-', '-'}, page.handleYamlMetaData}+ case '+':
+ return &frontmatterType{[]byte{'+', '+', '+'}, []byte{'+', '+', '+'}, page.handleTomlMetaData}+ case '{':+ return &frontmatterType{[]byte{'{'}, []byte{'}'}, page.handleJsonMetaData}+ default:
+ return nil
+ }
+}
+
+func beginFrontMatter(data *bufio.Reader, f *frontmatterType) (bool, error) {+ peek := make([]byte, 3)
+ _, err := data.Read(peek)
+ if err != nil {+ return false, err
+ }
+ return bytes.Equal(peek, f.markstart), nil
+}
+
+func chompWhitespace(data *bufio.Reader) (r rune, err error) {+ for {+ r, _, err = data.ReadRune()
+ if err != nil {+ return
}
+ if unicode.IsSpace(r) {+ continue
+ }
+ if err := data.UnreadRune(); err != nil {+ return r, errors.New("unable to unread first charactor in front matter.")+ }
+ return r, nil
}
- return nil, nil
}
func (p *Page) Render(layout ...string) template.HTML {@@ -378,19 +422,26 @@
return buffer
}
-func (page *Page) readFile() []byte {- var data, err = ioutil.ReadFile(page.FileName)
+func (page *Page) readFile() (data []byte, err error) {+ data, err = ioutil.ReadFile(page.FileName)
if err != nil {- PrintErr("Error Reading: " + page.FileName)- return nil
+ return nil, err
}
- return data
+ return data, nil
}
func (page *Page) buildPageFromFile() error {- data := page.readFile()
+ f, err := os.Open(page.FileName)
+ if err != nil {+ return err
+ }
+ return page.parse(bufio.NewReader(f))
+}
- content, err := page.parseFileHeading(data)
+func (page *Page) parse(reader io.Reader) error {+ data := bufio.NewReader(reader)
+
+ err := page.parseFrontMatter(data)
if err != nil {return err
}
@@ -397,27 +448,24 @@
switch page.Markup {case "md":
- page.convertMarkdown(content)
+ page.convertMarkdown(data)
case "rst":
- page.convertRestructuredText(content)
+ page.convertRestructuredText(data)
}
return nil
}
-func (page *Page) convertMarkdown(lines []string) {-
- page.RawMarkdown = strings.Join(lines, "\n")
- content := string(blackfriday.MarkdownCommon([]byte(page.RawMarkdown)))
+func (page *Page) convertMarkdown(lines io.Reader) {+ b := new(bytes.Buffer)
+ b.ReadFrom(lines)
+ content := string(blackfriday.MarkdownCommon(b.Bytes()))
page.Content = template.HTML(content)
page.Summary = template.HTML(TruncateWordsToWholeSentence(StripHTML(StripShortcodes(content)), summaryLength))
}
-func (page *Page) convertRestructuredText(lines []string) {-
- page.RawMarkdown = strings.Join(lines, "\n")
-
+func (page *Page) convertRestructuredText(lines io.Reader) { cmd := exec.Command("rst2html.py")- cmd.Stdin = strings.NewReader(page.RawMarkdown)
+ cmd.Stdin = lines
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {--- /dev/null
+++ b/hugolib/page_test.go
@@ -1,0 +1,124 @@
+package hugolib
+
+import (
+ "html/template"
+ "io"
+ "strings"
+ "testing"
+)
+
+var EMPTY_PAGE = ""
+
+var SIMPLE_PAGE = `---
+title: Simple
+---
+Simple Page
+`
+
+var INVALID_FRONT_MATTER_MISSING = `This is a test`
+
+var INVALID_FRONT_MATTER_SHORT_DELIM = `
+--
+title: Short delim start
+---
+Short Delim
+`
+
+var INVALID_FRONT_MATTER_SHORT_DELIM_ENDING = `
+---
+title: Short delim ending
+--
+Short Delim
+`
+
+var INVALID_FRONT_MATTER_LEADING_WS = `
+
+ ---
+title: Leading WS
+---
+Leading
+`
+
+func checkError(t *testing.T, err error, expected string) {+ if err == nil {+ t.Fatalf("err is nil")+ }
+ if err.Error() != expected {+ t.Errorf("err.Error() returned: '%s'. Expected: '%s'", err.Error(), expected)+ }
+}
+
+func TestDegenerateEmptyPageZeroLengthName(t *testing.T) {+ _, err := ReadFrom(strings.NewReader(EMPTY_PAGE), "")
+ if err == nil {+ t.Fatalf("A zero length page name must return an error")+ }
+
+ checkError(t, err, "Zero length page name")
+}
+
+func TestDegenerateEmptyPage(t *testing.T) {+ _, err := ReadFrom(strings.NewReader(EMPTY_PAGE), "test")
+ if err == nil {+ t.Fatalf("Expected ReadFrom to return an error when an empty buffer is passed.")+ }
+
+ checkError(t, err, "unable to locate front matter")
+}
+
+func checkPageTitle(t *testing.T, page *Page, title string) {+ if page.Title != title {+ t.Fatalf("Page title is: %s. Expected %s", page.Title, title)+ }
+}
+
+func checkPageContent(t *testing.T, page *Page, content string) {+ if page.Content != template.HTML(content) {+ t.Fatalf("Page content is: %s. Expected %s", page.Content, content)+ }
+}
+
+func checkPageType(t *testing.T, page *Page, pageType string) {+ if page.Type() != pageType {+ t.Fatalf("Page type is: %s. Expected: %s", page.Type(), pageType)+ }
+}
+
+func checkPageLayout(t *testing.T, page *Page, layout string) {+ if page.Layout() != layout {+ t.Fatalf("Page layout is: %s. Expected: %s", page.Layout(), layout)+ }
+}
+
+func TestCreateNewPage(t *testing.T) {+ p, err := ReadFrom(strings.NewReader(SIMPLE_PAGE), "simple")
+ if err != nil {+ t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)+ }
+ checkPageTitle(t, p, "Simple")
+ checkPageContent(t, p, "<p>Simple Page</p>\n")
+ checkPageType(t, p, "page")
+ checkPageLayout(t, p, "page/single.html")
+}
+
+func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) {+ var tests = []struct {+ r io.Reader
+ err string
+ }{+ {strings.NewReader(INVALID_FRONT_MATTER_SHORT_DELIM), "unable to match beginning front matter delimiter"},+ {strings.NewReader(INVALID_FRONT_MATTER_SHORT_DELIM_ENDING), "unable to match ending front matter delimiter"},+ {strings.NewReader(INVALID_FRONT_MATTER_MISSING), "unable to match beginning front matter delimiter"},+ }
+ for _, test := range tests {+ _, err := ReadFrom(test.r, "invalid/front/matter/short/delim")
+ checkError(t, err, test.err)
+ }
+}
+
+func TestDegenerateInvalidFrontMatterLeadingWhitespace(t *testing.T) {+ _, err := ReadFrom(strings.NewReader(INVALID_FRONT_MATTER_LEADING_WS), "invalid/front/matter/leading/ws")
+ if err != nil {+ t.Fatalf("Unable to parse front matter given leading whitespace: %s", err)+ }
+}
--- a/hugolib/path_seperators_test.go
+++ b/hugolib/path_seperators_test.go
@@ -2,17 +2,27 @@
import (
"path/filepath"
+ "strings"
"testing"
)
+var SIMPLE_PAGE_YAML = `---
+contenttype: ""
+---
+Sample Text
+`
+
func TestDegenerateMissingFolderInPageFilename(t *testing.T) {- p := NewPage(filepath.Join("foobar"))+ p, err := ReadFrom(strings.NewReader(SIMPLE_PAGE_YAML), filepath.Join("foobar"))+ if err != nil {+ t.Fatalf("Error in ReadFrom")+ }
if p.Section != "" { t.Fatalf("No section should be set for a file path: foobar")}
}
-func TestCreateNewPage(t *testing.T) {+func TestNewPageWithFilePath(t *testing.T) { toCheck := []map[string]string{ {"input": filepath.Join("sub", "foobar.html"), "expect": "sub"}, {"input": filepath.Join("content", "sub", "foobar.html"), "expect": "sub"},@@ -20,13 +30,15 @@
}
for _, el := range toCheck {- p := NewPage(el["input"])
+ p, err := ReadFrom(strings.NewReader(SIMPLE_PAGE_YAML), el["input"])
+ if err != nil {+ t.Fatalf("Reading from SIMPLE_PAGE_YAML resulted in an error: %s", err)+ }
if p.Section != el["expect"] { t.Fatalf("Section not set to %s for page %s. Got: %s", el["expect"], el["input"], p.Section)}
}
}
-
func TestSettingOutFileOnPageContainsCorrectSlashes(t *testing.T) { s := NewSite(&Config{})