ref: 9a2f6c62a602fc701cc179ac9598092e85b342a8
parent: 08d41c3a48548597a33f460f209974b64591c6fa
author: coderzh <[email protected]>
date: Thu Oct 1 05:04:30 EDT 2015
Hugo import from jekyll usage: hugo import jekyll jekyll_root_path target_path Implemented: * Create new hugo site * Create config.yaml * Convert all markdown contents. * Copy all other files and folders to static Fixes #101
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -83,6 +83,7 @@
HugoCmd.AddCommand(undraftCmd)
HugoCmd.AddCommand(genautocompleteCmd)
HugoCmd.AddCommand(gendocCmd)
+ HugoCmd.AddCommand(importCmd)
}
//Initializes flags
--- /dev/null
+++ b/commands/import.go
@@ -1,0 +1,483 @@
+// Copyright © 2015 Steve Francia <[email protected]>.
+//
+// Licensed under the Simple Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://opensource.org/licenses/Simple-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/spf13/cast"
+ "github.com/spf13/cobra"
+ "github.com/spf13/hugo/helpers"
+ "github.com/spf13/hugo/hugofs"
+ "github.com/spf13/hugo/hugolib"
+ "github.com/spf13/hugo/parser"
+ jww "github.com/spf13/jwalterweatherman"
+)
+
+func init() {+ importCmd.AddCommand(importJekyllCmd)
+}
+
+var importCmd = &cobra.Command{+ Use: "import",
+ Short: "import from others",
+ Long: `import from others like jekyll.
+
+Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
+ Run: nil,
+}
+
+var importJekyllCmd = &cobra.Command{+ Use: "jekyll",
+ Short: "hugo import from jekyll",
+ Long: `hugo import from jekyll.
+
+Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
+ Run: importFromJekyll,
+}
+
+func importFromJekyll(cmd *cobra.Command, args []string) {+ jww.SetLogThreshold(jww.LevelTrace)
+ jww.SetStdoutThreshold(jww.LevelWarn)
+
+ if len(args) < 2 {+ jww.ERROR.Println(`Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
+ return
+ }
+
+ jekyllRoot, err := filepath.Abs(filepath.Clean(args[0]))
+ if err != nil {+ jww.ERROR.Println("Path error:", args[0])+ return
+ }
+
+ targetDir, err := filepath.Abs(filepath.Clean(args[1]))
+ if err != nil {+ jww.ERROR.Println("Path error:", args[1])+ return
+ }
+
+ createSiteFromJekyll(jekyllRoot, targetDir)
+
+ jww.INFO.Println("Import jekyll from:", jekyllRoot, "to:", targetDir)+ fmt.Println("Importing...")+
+ fileCount := 0
+ callback := func(path string, fi os.FileInfo, err error) error {+ if err != nil {+ return err
+ }
+
+ if fi.IsDir() {+ return nil
+ }
+
+ relPath, err := filepath.Rel(jekyllRoot, path)
+ if err != nil {+ jww.ERROR.Println("Get rel path error:", path)+ return err
+ }
+
+ relPath = filepath.ToSlash(relPath)
+ var draft bool = false
+
+ switch {+ case strings.HasPrefix(relPath, "_posts/"):
+ relPath = "content/post" + relPath[len("_posts"):]+ case strings.HasPrefix(relPath, "_drafts/"):
+ relPath = "content/draft" + relPath[len("_drafts"):]+ draft = true
+ default:
+ return nil
+ }
+
+ fileCount++
+ return convertJekyllPost(path, relPath, targetDir, draft)
+ }
+
+ err = filepath.Walk(jekyllRoot, callback)
+
+ if err != nil {+ fmt.Println(err)
+ } else {+ fmt.Println("Congratulations!", fileCount, "posts imported!")+ fmt.Println("Now, start hugo by yourself: \n" ++ "$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove")
+ fmt.Println("$ cd " + args[1] + "\n$ hugo server -w --theme=herring-cove")+ }
+}
+
+func createSiteFromJekyll(jekyllRoot, targetDir string) {+ mkdir(targetDir, "layouts")
+ mkdir(targetDir, "content")
+ mkdir(targetDir, "archetypes")
+ mkdir(targetDir, "static")
+ mkdir(targetDir, "data")
+ mkdir(targetDir, "themes")
+
+ jekyllConfig := loadJekyllConfig(jekyllRoot)
+ createConfigFromJekyll(targetDir, "yaml", jekyllConfig)
+
+ copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"))
+}
+
+func loadJekyllConfig(jekyllRoot string) map[string]interface{} {+ fs := hugofs.SourceFs
+ path := filepath.Join(jekyllRoot, "_config.yml")
+
+ exists, err := helpers.Exists(path, fs)
+
+ if err != nil || !exists {+ return nil
+ }
+
+ f, err := fs.Open(path)
+ if err != nil {+ return nil
+ }
+
+ defer f.Close()
+
+ b, err := ioutil.ReadAll(f)
+
+ if err != nil {+ return nil
+ }
+
+ c, err := parser.HandleYAMLMetaData(b)
+
+ if err != nil {+ return nil
+ }
+
+ return c.(map[string]interface{})+}
+
+func createConfigFromJekyll(inpath string, kind string, jekyllConfig map[string]interface{}) (err error) {+ title := "My New Hugo Site"
+ baseurl := "http://replace-this-with-your-hugo-site.com/"
+
+ for key, value := range jekyllConfig {+ lowerKey := strings.ToLower(key)
+
+ switch lowerKey {+ case "title":
+ if str, ok := value.(string); ok {+ title = str
+ }
+
+ case "url":
+ if str, ok := value.(string); ok {+ baseurl = str
+ }
+ }
+ }
+
+ in := map[string]interface{}{+ "baseurl": baseurl,
+ "title": title,
+ "languageCode": "en-us",
+ "disablePathToLower": true,
+ }
+ kind = parser.FormatSanitize(kind)
+
+ by, err := parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind))
+ if err != nil {+ return err
+ }
+
+ err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs)
+ if err != nil {+ return
+ }
+
+ return nil
+}
+
+func copyFile(source string, dest string) (err error) {+ sf, err := os.Open(source)
+ if err != nil {+ return err
+ }
+ defer sf.Close()
+ df, err := os.Create(dest)
+ if err != nil {+ return err
+ }
+ defer df.Close()
+ _, err = io.Copy(df, sf)
+ if err == nil {+ si, err := os.Stat(source)
+ if err != nil {+ err = os.Chmod(dest, si.Mode())
+ }
+
+ }
+ return
+}
+
+func copyDir(source string, dest string) (err error) {+ fi, err := os.Stat(source)
+ if err != nil {+ return err
+ }
+ if !fi.IsDir() {+ return errors.New(source + " is not a directory")
+ }
+ err = os.MkdirAll(dest, fi.Mode())
+ if err != nil {+ return err
+ }
+ entries, err := ioutil.ReadDir(source)
+ for _, entry := range entries {+ sfp := filepath.Join(source, entry.Name())
+ dfp := filepath.Join(dest, entry.Name())
+ if entry.IsDir() {+ err = copyDir(sfp, dfp)
+ if err != nil {+ jww.ERROR.Println(err)
+ }
+ } else {+ err = copyFile(sfp, dfp)
+ if err != nil {+ jww.ERROR.Println(err)
+ }
+ }
+
+ }
+ return nil
+}
+
+func copyJekyllFilesAndFolders(jekyllRoot string, dest string) (err error) {+ fi, err := os.Stat(jekyllRoot)
+ if err != nil {+ return err
+ }
+ if !fi.IsDir() {+ return errors.New(jekyllRoot + " is not a directory")
+ }
+ err = os.MkdirAll(dest, fi.Mode())
+ if err != nil {+ return err
+ }
+ entries, err := ioutil.ReadDir(jekyllRoot)
+ for _, entry := range entries {+ sfp := filepath.Join(jekyllRoot, entry.Name())
+ dfp := filepath.Join(dest, entry.Name())
+ if entry.IsDir() {+ if entry.Name()[0] != '_' && entry.Name()[0] != '.' {+ err = copyDir(sfp, dfp)
+ if err != nil {+ jww.ERROR.Println(err)
+ }
+ }
+ } else {+ lowerEntryName := strings.ToLower(entry.Name())
+ exceptSuffix := []string{".md", ".markdown", ".html", ".htm",+ ".xml", ".textile", "rakefile", "gemfile", ".lock"}
+ isExcept := false
+ for _, suffix := range exceptSuffix {+ if strings.HasSuffix(lowerEntryName, suffix) {+ isExcept = true
+ break
+ }
+ }
+
+ if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' {+ err = copyFile(sfp, dfp)
+ if err != nil {+ jww.ERROR.Println(err)
+ }
+ }
+ }
+
+ }
+ return nil
+}
+
+func parseJekyllFilename(filename string) (time.Time, string, error) {+ re := regexp.MustCompile(`(\d+-\d+-\d+)-(.+)\..*`)
+ r := re.FindAllStringSubmatch(filename, -1)
+ if len(r) == 0 {+ return time.Now(), "", errors.New("filename not match")+ }
+
+ postDate, err := time.Parse("2006-01-02", r[0][1])+ if err != nil {+ return time.Now(), "", err
+ }
+
+ postName := r[0][2]
+
+ return postDate, postName, nil
+}
+
+func convertJekyllPost(path, relPath, targetDir string, draft bool) error {+ jww.TRACE.Println("Converting", path)+
+ filename := filepath.Base(path)
+ postDate, postName, err := parseJekyllFilename(filename)
+ if err != nil {+ jww.ERROR.Println("Parse filename error:", filename)+ return err
+ }
+
+ jww.TRACE.Println(filename, postDate, postName)
+
+ targetFile := filepath.Join(targetDir, relPath)
+ targetParentDir := filepath.Dir(targetFile)
+ os.MkdirAll(targetParentDir, 0777)
+
+ contentBytes, err := ioutil.ReadFile(path)
+ if err != nil {+ jww.ERROR.Println("Read file error:", path)+ return err
+ }
+
+ psr, err := parser.ReadFrom(bytes.NewReader(contentBytes))
+ if err != nil {+ jww.ERROR.Println("Parse file error:", path)+ return err
+ }
+
+ metadata, err := psr.Metadata()
+ if err != nil {+ jww.ERROR.Println("Processing file error:", path)+ return err
+ }
+
+ newmetadata, err := convertJekyllMetaData(metadata, postName, postDate, draft)
+ if err != nil {+ jww.ERROR.Println("Convert metadata error:", path)+ return err
+ }
+
+ jww.TRACE.Println(newmetadata)
+ content := convertJekyllContent(newmetadata, string(psr.Content()))
+
+ page, err := hugolib.NewPage(filename)
+ if err != nil {+ jww.ERROR.Println("New page error", filename)+ return err
+ }
+
+ page.SetDir(targetParentDir)
+ page.SetSourceContent([]byte(content))
+ page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml"))+ page.SaveSourceAs(targetFile)
+
+ jww.TRACE.Println("Target file:", targetFile)+
+ return nil
+}
+
+func convertJekyllMetaData(m interface{}, postName string, postDate time.Time, draft bool) (interface{}, error) {+ url := postDate.Format("/2006/01/02/") + postName + "/"+
+ metadata, err := cast.ToStringMapE(m)
+ if err != nil {+ return nil, err
+ }
+
+ if draft {+ metadata["draft"] = true
+ }
+
+ for key, value := range metadata {+ lowerKey := strings.ToLower(key)
+
+ switch lowerKey {+ case "layout":
+ delete(metadata, key)
+ case "permalink":
+ if str, ok := value.(string); ok {+ url = str
+ }
+ delete(metadata, key)
+ case "category":
+ if str, ok := value.(string); ok {+ metadata["categories"] = []string{str}+ }
+ delete(metadata, key)
+ case "excerpt_separator":
+ if key != lowerKey {+ delete(metadata, key)
+ metadata[lowerKey] = value
+ }
+ case "date":
+ if str, ok := value.(string); ok {+ re := regexp.MustCompile(`(\d+):(\d+):(\d+)`)
+ r := re.FindAllStringSubmatch(str, -1)
+ if len(r) > 0 {+ hour, _ := strconv.Atoi(r[0][1])
+ minute, _ := strconv.Atoi(r[0][2])
+ second, _ := strconv.Atoi(r[0][3])
+ postDate = time.Date(postDate.Year(), postDate.Month(), postDate.Day(), hour, minute, second, 0, time.UTC)
+ }
+ }
+ delete(metadata, key)
+ }
+
+ }
+
+ metadata["url"] = url
+ metadata["date"] = postDate.Format(time.RFC3339)
+
+ return metadata, nil
+}
+
+func convertJekyllContent(m interface{}, content string) string {+ metadata, _ := cast.ToStringMapE(m)
+
+ lines := strings.Split(content, "\n")
+ var resultLines []string
+ for _, line := range lines {+ resultLines = append(resultLines, strings.Trim(line, "\r\n"))
+ }
+
+ content = strings.Join(resultLines, "\n")
+
+ excerptSep := "<!--more-->"
+ if value, ok := metadata["excerpt_separator"]; ok {+ if str, strOk := value.(string); strOk {+ content = strings.Replace(content, strings.TrimSpace(str), excerptSep, -1)
+ }
+ }
+
+ replaceList := []struct {+ re *regexp.Regexp
+ replace string
+ }{+ {regexp.MustCompile("<!-- more -->"), "<!--more-->"},+ {regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"},+ {regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), "{{< highlight $1 >}}"},+ {regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"},+ }
+
+ for _, replace := range replaceList {+ content = replace.re.ReplaceAllString(content, replace.replace)
+ }
+
+ return content
+}
--- /dev/null
+++ b/commands/import_test.go
@@ -1,0 +1,104 @@
+// Copyright © 2015 Steve Francia <[email protected]>.
+//
+// Licensed under the Simple Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://opensource.org/licenses/Simple-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+ "encoding/json"
+ "github.com/stretchr/testify/assert"
+ "testing"
+ "time"
+)
+
+func TestParseJekyllFilename(t *testing.T) {+ filenameArray := []string{+ "2015-01-02-test.md",
+ "2012-03-15-中文.markup",
+ }
+
+ expectResult := []struct {+ postDate time.Time
+ postName string
+ }{+ {time.Date(2015, time.January, 2, 0, 0, 0, 0, time.UTC), "test"},+ {time.Date(2012, time.March, 15, 0, 0, 0, 0, time.UTC), "中文"},+ }
+
+ for i, filename := range filenameArray {+ postDate, postName, err := parseJekyllFilename(filename)
+ assert.Equal(t, err, nil)
+ assert.Equal(t, expectResult[i].postDate.Format("2006-01-02"), postDate.Format("2006-01-02"))+ assert.Equal(t, expectResult[i].postName, postName)
+ }
+}
+
+func TestConvertJekyllMetadata(t *testing.T) {+ testDataList := []struct {+ metadata interface{}+ postName string
+ postDate time.Time
+ draft bool
+ expect string
+ }{+ {map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,+ `{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`},+ {map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), true,+ `{"date":"2015-10-01T00:00:00Z","draft":true,"url":"/2015/10/01/testPost/"}`},+ {map[interface{}]interface{}{"Permalink": "/permalink.html", "layout": "post"},+ "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
+ `{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`},+ {map[interface{}]interface{}{"permalink": "/permalink.html"},+ "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
+ `{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`},+ {map[interface{}]interface{}{"category": nil, "permalink": 123},+ "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
+ `{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`},+ {map[interface{}]interface{}{"Excerpt_Separator": "sep"},+ "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
+ `{"date":"2015-10-01T00:00:00Z","excerpt_separator":"sep","url":"/2015/10/01/testPost/"}`},+ {map[interface{}]interface{}{"category": "book", "layout": "post", "Others": "Goods", "Date": "2015-10-01 12:13:11"},+ "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
+ `{"Others":"Goods","categories":["book"],"date":"2015-10-01T12:13:11Z","url":"/2015/10/01/testPost/"}`},+ }
+
+ for _, data := range testDataList {+ result, err := convertJekyllMetaData(data.metadata, data.postName, data.postDate, data.draft)
+ assert.Equal(t, nil, err)
+ jsonResult, err := json.Marshal(result)
+ assert.Equal(t, nil, err)
+ assert.Equal(t, data.expect, string(jsonResult))
+ }
+}
+
+func TestConvertJekyllContent(t *testing.T) {+ testDataList := []struct {+ metadata interface{}+ content string
+ expect string
+ }{+ {map[interface{}]interface{}{},+ `Test content\n<!-- more -->\npart2 content`, `Test content\n<!--more-->\npart2 content`},
+ {map[interface{}]interface{}{"excerpt_separator": "<!--sep-->"},+ `Test content\n<!--sep-->\npart2 content`, `Test content\n<!--more-->\npart2 content`},
+ {map[interface{}]interface{}{}, "{% raw %}text{% endraw %}", "text"},+ {map[interface{}]interface{}{}, "{%raw%} text2 {%endraw %}", "text2"},+ {map[interface{}]interface{}{},+ "{% highlight go %}\nvar s int\n{% endhighlight %}",+ "{{< highlight go >}}\nvar s int\n{{< / highlight >}}"},+ }
+
+ for _, data := range testDataList {+ result := convertJekyllContent(data.metadata, data.content)
+ assert.Equal(t, data.expect, result)
+ }
+}