ref: c71e1b106e6011d148cac899f83c4685dee33a22
dir: /hugolib/hugo_sites.go/
// Copyright 2016-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"errors"
"fmt"
"strings"
"sync"
"github.com/spf13/hugo/deps"
"github.com/spf13/hugo/helpers"
"github.com/spf13/viper"
"github.com/spf13/hugo/source"
"github.com/spf13/hugo/tpl"
"github.com/spf13/hugo/tplapi"
)
// HugoSites represents the sites to build. Each site represents a language.
type HugoSites struct {
Sites []*Site
runMode runmode
multilingual *Multilingual
*deps.Deps
}
// NewHugoSites creates a new collection of sites given the input sites, building
// a language configuration based on those.
func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
if cfg.Language != nil {
return nil, errors.New("Cannot provide Language in Cfg when sites are provided")
}
langConfig, err := newMultiLingualFromSites(sites...)
if err != nil {
return nil, err
}
h := &HugoSites{
multilingual: langConfig,
Sites: sites}
for _, s := range sites {
s.owner = h
}
applyDepsIfNeeded(cfg, sites...)
h.Deps = sites[0].Deps
return h, nil
}
func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error {
if cfg.TemplateProvider == nil {
cfg.TemplateProvider = tpl.DefaultTemplateProvider
}
var (
d *deps.Deps
err error
)
for _, s := range sites {
if s.Deps != nil {
continue
}
if d == nil {
cfg.Language = s.Language
cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate)
d = deps.New(cfg)
if err := d.LoadTemplates(); err != nil {
return err
}
} else {
d, err = d.ForLanguage(s.Language)
if err != nil {
return err
}
}
s.Deps = d
}
return nil
}
// NewHugoSitesFromConfiguration creates HugoSites from the global Viper config.
// TODO(bep) globals rename this when all the globals are gone.
func NewHugoSitesFromConfiguration(cfg deps.DepsCfg) (*HugoSites, error) {
sites, err := createSitesFromConfig(cfg)
if err != nil {
return nil, err
}
return newHugoSites(cfg, sites...)
}
func (s *Site) withSiteTemplates(withTemplates ...func(templ tplapi.Template) error) func(templ tplapi.Template) error {
return func(templ tplapi.Template) error {
templ.LoadTemplates(s.absLayoutDir())
if s.hasTheme() {
templ.LoadTemplatesWithPrefix(s.absThemeDir()+"/layouts", "theme")
}
for _, wt := range withTemplates {
if wt == nil {
continue
}
if err := wt(templ); err != nil {
return err
}
}
return nil
}
}
func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
var (
sites []*Site
)
multilingual := viper.GetStringMap("languages")
if len(multilingual) == 0 {
l := helpers.NewDefaultLanguage()
cfg.Language = l
s, err := newSite(cfg)
if err != nil {
return nil, err
}
sites = append(sites, s)
}
if len(multilingual) > 0 {
var err error
languages, err := toSortedLanguages(multilingual)
if err != nil {
return nil, fmt.Errorf("Failed to parse multilingual config: %s", err)
}
for _, lang := range languages {
var s *Site
var err error
cfg.Language = lang
s, err = newSite(cfg)
if err != nil {
return nil, err
}
sites = append(sites, s)
}
}
return sites, nil
}
// Reset resets the sites and template caches, making it ready for a full rebuild.
func (h *HugoSites) reset() {
for i, s := range h.Sites {
h.Sites[i] = s.reset()
}
tpl.ResetCaches()
}
func (h *HugoSites) createSitesFromConfig() error {
depsCfg := deps.DepsCfg{Fs: h.Fs}
sites, err := createSitesFromConfig(depsCfg)
if err != nil {
return err
}
langConfig, err := newMultiLingualFromSites(sites...)
if err != nil {
return err
}
h.Sites = sites
for _, s := range sites {
s.owner = h
}
if err := applyDepsIfNeeded(depsCfg, sites...); err != nil {
return err
}
h.Deps = sites[0].Deps
h.multilingual = langConfig
return nil
}
func (h *HugoSites) toSiteInfos() []*SiteInfo {
infos := make([]*SiteInfo, len(h.Sites))
for i, s := range h.Sites {
infos[i] = &s.Info
}
return infos
}
// BuildCfg holds build options used to, as an example, skip the render step.
type BuildCfg struct {
// Whether we are in watch (server) mode
Watching bool
// Print build stats at the end of a build
PrintStats bool
// Reset site state before build. Use to force full rebuilds.
ResetState bool
// Re-creates the sites from configuration before a build.
// This is needed if new languages are added.
CreateSitesFromConfig bool
// Skip rendering. Useful for testing.
SkipRender bool
// Use this to indicate what changed (for rebuilds).
whatChanged *whatChanged
}
func (h *HugoSites) renderCrossSitesArtifacts() error {
if !h.multilingual.enabled() {
return nil
}
if viper.GetBool("disableSitemap") {
return nil
}
// TODO(bep) DRY
sitemapDefault := parseSitemap(viper.GetStringMap("sitemap"))
s := h.Sites[0]
smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"}
return s.renderAndWriteXML("sitemapindex",
sitemapDefault.Filename, h.toSiteInfos(), s.appendThemeTemplates(smLayouts)...)
}
func (h *HugoSites) assignMissingTranslations() error {
// This looks heavy, but it should be a small number of nodes by now.
allPages := h.findAllPagesByKindNotIn(KindPage)
for _, nodeType := range []string{KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} {
nodes := h.findPagesByKindIn(nodeType, allPages)
// Assign translations
for _, t1 := range nodes {
for _, t2 := range nodes {
if t1.isNewTranslation(t2) {
t1.translations = append(t1.translations, t2)
}
}
}
}
// Now we can sort the translations.
for _, p := range allPages {
if len(p.translations) > 0 {
pageBy(languagePageSort).Sort(p.translations)
}
}
return nil
}
// createMissingPages creates home page, taxonomies etc. that isnt't created as an
// effect of having a content file.
func (h *HugoSites) createMissingPages() error {
var newPages Pages
for _, s := range h.Sites {
// home pages
home := s.findPagesByKind(KindHome)
if len(home) > 1 {
panic("Too many homes")
}
if len(home) == 0 {
n := s.newHomePage()
s.Pages = append(s.Pages, n)
newPages = append(newPages, n)
}
// taxonomy list and terms pages
taxonomies := s.Language.GetStringMapString("taxonomies")
if len(taxonomies) > 0 {
taxonomyPages := s.findPagesByKind(KindTaxonomy)
taxonomyTermsPages := s.findPagesByKind(KindTaxonomyTerm)
for _, plural := range taxonomies {
tax := s.Taxonomies[plural]
foundTaxonomyPage := false
foundTaxonomyTermsPage := false
for key := range tax {
if s.Info.preserveTaxonomyNames {
key = s.PathSpec.MakePathSanitized(key)
}
for _, p := range taxonomyPages {
if p.sections[0] == plural && p.sections[1] == key {
foundTaxonomyPage = true
break
}
}
for _, p := range taxonomyTermsPages {
if p.sections[0] == plural {
foundTaxonomyTermsPage = true
break
}
}
if !foundTaxonomyPage {
n := s.newTaxonomyPage(plural, key)
s.Pages = append(s.Pages, n)
newPages = append(newPages, n)
}
if !foundTaxonomyTermsPage {
foundTaxonomyTermsPage = true
n := s.newTaxonomyTermsPage(plural)
s.Pages = append(s.Pages, n)
newPages = append(newPages, n)
}
}
}
}
sectionPages := s.findPagesByKind(KindSection)
if len(sectionPages) < len(s.Sections) {
for name, section := range s.Sections {
// A section may be created for the root content folder if a
// content file is placed there.
// We cannot create a section node for that, because
// that would overwrite the home page.
if name == "" {
continue
}
foundSection := false
for _, sectionPage := range sectionPages {
if sectionPage.sections[0] == name {
foundSection = true
break
}
}
if !foundSection {
n := s.newSectionPage(name, section)
s.Pages = append(s.Pages, n)
newPages = append(newPages, n)
}
}
}
}
if len(newPages) > 0 {
// This resorting is unfortunate, but it also needs to be sorted
// when sections are created.
first := h.Sites[0]
first.AllPages = append(first.AllPages, newPages...)
first.AllPages.Sort()
for _, s := range h.Sites {
s.Pages.Sort()
}
for i := 1; i < len(h.Sites); i++ {
h.Sites[i].AllPages = first.AllPages
}
}
return nil
}
func (h *HugoSites) setupTranslations() {
master := h.Sites[0]
for _, p := range master.rawAllPages {
if p.Lang() == "" {
panic("Page language missing: " + p.Title)
}
shouldBuild := p.shouldBuild()
for i, site := range h.Sites {
if strings.HasPrefix(site.Language.Lang, p.Lang()) {
site.updateBuildStats(p)
if shouldBuild {
site.Pages = append(site.Pages, p)
p.Site = &site.Info
}
}
if !shouldBuild {
continue
}
if i == 0 {
site.AllPages = append(site.AllPages, p)
}
}
}
// Pull over the collections from the master site
for i := 1; i < len(h.Sites); i++ {
h.Sites[i].AllPages = h.Sites[0].AllPages
h.Sites[i].Data = h.Sites[0].Data
}
if len(h.Sites) > 1 {
pages := h.Sites[0].AllPages
allTranslations := pagesToTranslationsMap(h.multilingual, pages)
assignTranslationsToPages(allTranslations, pages)
}
}
func (s *Site) preparePagesForRender(cfg *BuildCfg) {
pageChan := make(chan *Page)
wg := &sync.WaitGroup{}
for i := 0; i < getGoMaxProcs()*4; i++ {
wg.Add(1)
go func(pages <-chan *Page, wg *sync.WaitGroup) {
defer wg.Done()
for p := range pages {
if !cfg.whatChanged.other && p.rendered {
// No need to process it again.
continue
}
// If we got this far it means that this is either a new Page pointer
// or a template or similar has changed so wee need to do a rerendering
// of the shortcodes etc.
// Mark it as rendered
p.rendered = true
// If in watch mode, we need to keep the original so we can
// repeat this process on rebuild.
var workContentCopy []byte
if cfg.Watching {
workContentCopy = make([]byte, len(p.workContent))
copy(workContentCopy, p.workContent)
} else {
// Just reuse the same slice.
workContentCopy = p.workContent
}
if p.Markup == "markdown" {
tmpContent, tmpTableOfContents := helpers.ExtractTOC(workContentCopy)
p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
workContentCopy = tmpContent
}
var err error
if workContentCopy, err = handleShortcodes(p, s.Tmpl, workContentCopy); err != nil {
s.Log.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
}
if p.Markup != "html" {
// Now we know enough to create a summary of the page and count some words
summaryContent, err := p.setUserDefinedSummaryIfProvided(workContentCopy)
if err != nil {
s.Log.ERROR.Printf("Failed to set user defined summary for page %q: %s", p.Path(), err)
} else if summaryContent != nil {
workContentCopy = summaryContent.content
}
p.Content = helpers.BytesToHTML(workContentCopy)
if summaryContent == nil {
p.setAutoSummary()
}
} else {
p.Content = helpers.BytesToHTML(workContentCopy)
}
//analyze for raw stats
p.analyzePage()
}
}(pageChan, wg)
}
for _, p := range s.Pages {
pageChan <- p
}
close(pageChan)
wg.Wait()
}
// Pages returns all pages for all sites.
func (h *HugoSites) Pages() Pages {
return h.Sites[0].AllPages
}
func handleShortcodes(p *Page, t tplapi.Template, rawContentCopy []byte) ([]byte, error) {
if len(p.contentShortCodes) > 0 {
p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.contentShortCodes), p.BaseFileName())
shortcodes, err := executeShortcodeFuncMap(p.contentShortCodes)
if err != nil {
return rawContentCopy, err
}
rawContentCopy, err = replaceShortcodeTokens(rawContentCopy, shortcodePlaceholderPrefix, shortcodes)
if err != nil {
p.s.Log.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
}
}
return rawContentCopy, nil
}
func (s *Site) updateBuildStats(page *Page) {
if page.IsDraft() {
s.draftCount++
}
if page.IsFuture() {
s.futureCount++
}
if page.IsExpired() {
s.expiredCount++
}
}
func (h *HugoSites) findPagesByKindNotIn(kind string, inPages Pages) Pages {
return h.Sites[0].findPagesByKindNotIn(kind, inPages)
}
func (h *HugoSites) findPagesByKindIn(kind string, inPages Pages) Pages {
return h.Sites[0].findPagesByKindIn(kind, inPages)
}
func (h *HugoSites) findAllPagesByKind(kind string) Pages {
return h.findPagesByKindIn(kind, h.Sites[0].AllPages)
}
func (h *HugoSites) findAllPagesByKindNotIn(kind string) Pages {
return h.findPagesByKindNotIn(kind, h.Sites[0].AllPages)
}
// Convenience func used in tests.
func newHugoSitesFromSourceAndLanguages(input []source.ByteSource, languages helpers.Languages, cfg deps.DepsCfg) (*HugoSites, error) {
if len(languages) == 0 {
panic("Must provide at least one language")
}
first := &Site{
Language: languages[0],
Source: &source.InMemorySource{ByteSource: input},
}
if len(languages) == 1 {
return newHugoSites(cfg, first)
}
sites := make([]*Site, len(languages))
sites[0] = first
for i := 1; i < len(languages); i++ {
sites[i] = &Site{Language: languages[i]}
}
return newHugoSites(cfg, sites...)
}
// Convenience func used in tests.
func newHugoSitesDefaultLanguage(cfg deps.DepsCfg) (*HugoSites, error) {
return newHugoSitesFromSourceAndLanguages(nil, helpers.Languages{helpers.NewDefaultLanguage()}, cfg)
}