ref: 47c91a4ca2ac3db496b9d495757eda121b612bec
parent: 93addfcbeec301e4f37b0f83bc5735f1939f1895
author: Nate Finch <[email protected]>
date: Fri Aug 29 09:40:21 EDT 2014
Fix CreatePages This fixes #450. There are two problems: 1.) We're creating a new goroutine for every page. 2.) We're calling s.Pages = append(s.Pages, page) inside each goroutine. 1 is a problem if in that if you have a ton of pages, that's a ton of goroutines. It's not really useful to have more than a few goroutines at a time, and lots can actually make your code much slower, and, evidently, crash. 2 is a problem in that append is not thread safe. Sometimes it returns a new slice with a larger capacity, when the original slice isn't large enough. This can cause problems if two goroutines do this at the same time. The solution for 1 is to use a limited number of workers (I chose 2*GOMAXPROCS as a nice guess). The solution for 2 is to serialize access to s.Pages, which I did by doing it in a single goroutine.
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -19,6 +19,7 @@
"html/template"
"io"
"os"
+ "strconv"
"strings"
"sync"
"time"
@@ -311,60 +312,107 @@
return
}
-func (s *Site) CreatePages() (err error) {+type pageRenderResult struct {+ page *Page
+ err error
+}
+
+func (s *Site) CreatePages() error { if s.Source == nil { panic(fmt.Sprintf("s.Source not set %s", s.absContentDir()))}
if len(s.Source.Files()) < 1 {- return
+ return nil
}
- var wg sync.WaitGroup
- for _, fi := range s.Source.Files() {- wg.Add(1)
- go func(file *source.File) (err error) {- defer wg.Done()
+ files := s.Source.Files()
- page, err := NewPage(file.LogicalName)
- if err != nil {- return err
- }
- err = page.ReadFrom(file.Contents)
- if err != nil {- return err
- }
- page.Site = &s.Info
- page.Tmpl = s.Tmpl
- page.Section = file.Section
- page.Dir = file.Dir
+ results := make(chan pageRenderResult)
+ input := make(chan *source.File)
- //Handling short codes prior to Conversion to HTML
- page.ProcessShortcodes(s.Tmpl)
+ procs := getGoMaxProcs()
- err = page.Convert()
- if err != nil {- return err
- }
+ wg := &sync.WaitGroup{}- if page.ShouldBuild() {- s.Pages = append(s.Pages, page)
- }
+ for i := 0; i < procs*2; i++ {+ wg.Add(1)
+ go pageRenderer(s, input, results, wg)
+ }
- if page.IsDraft() {- s.draftCount += 1
- }
+ errs := make(chan error)
- if page.IsFuture() {- s.futureCount += 1
- }
+ // we can only have exactly one result collator, since it makes changes that
+ // must be synchronized.
+ go resultCollator(s, results, errs)
- return
- }(fi)
+ for _, fi := range files {+ input <- fi
}
+ close(input)
+
wg.Wait()
+
+ close(results)
+
+ return <-errs
+}
+
+func pageRenderer(s *Site, input <-chan *source.File, results chan<- pageRenderResult, wg *sync.WaitGroup) {+ for file := range input {+ page, err := NewPage(file.LogicalName)
+ if err != nil {+ results <- pageRenderResult{nil, err}+ continue
+ }
+ err = page.ReadFrom(file.Contents)
+ if err != nil {+ results <- pageRenderResult{nil, err}+ continue
+ }
+ page.Site = &s.Info
+ page.Tmpl = s.Tmpl
+ page.Section = file.Section
+ page.Dir = file.Dir
+
+ //Handling short codes prior to Conversion to HTML
+ page.ProcessShortcodes(s.Tmpl)
+
+ err = page.Convert()
+ if err != nil {+ results <- pageRenderResult{nil, err}+ continue
+ }
+ results <- pageRenderResult{page, nil}+ }
+ wg.Done()
+}
+
+func resultCollator(s *Site, results <-chan pageRenderResult, errs chan<- error) {+ errMsgs := []string{}+ for r := range results {+ if r.err != nil {+ errMsgs = append(errMsgs, r.err.Error())
+ continue
+ }
+
+ if r.page.ShouldBuild() {+ s.Pages = append(s.Pages, r.page)
+ }
+
+ if r.page.IsDraft() {+ s.draftCount += 1
+ }
+
+ if r.page.IsFuture() {+ s.futureCount += 1
+ }
+ }
s.Pages.Sort()
- return
+ if len(errMsgs) == 0 {+ errs <- nil
+ }
+ errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n"))}
func (s *Site) BuildSiteMeta() (err error) {@@ -1009,4 +1057,13 @@
}
return "0 of " + msg
+}
+
+func getGoMaxProcs() int {+ if gmp := os.Getenv("GOMAXPROCS"); gmp != "" {+ if p, err := strconv.Atoi(gmp); err != nil {+ return p
+ }
+ }
+ return 1
}