ref: ee4274244b574a4d8acca6fd80b83c4f1d3d0b73
dir: /hugolib/filesystems/basefs.go/
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package filesystems provides the fine grained file systems used by Hugo. These
// are typically virtual filesystems that are composites of project and theme content.
package filesystems
import (
"errors"
"os"
"path/filepath"
"strings"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"
"fmt"
"github.com/gohugoio/hugo/hugolib/paths"
"github.com/gohugoio/hugo/langs"
"github.com/spf13/afero"
)
// When we create a virtual filesystem with data and i18n bundles for the project and the themes,
// this is the name of the project's virtual root. It got it's funky name to make sure
// (or very unlikely) that it collides with a theme name.
const projectVirtualFolder = "__h__project"
var filePathSeparator = string(filepath.Separator)
// BaseFs contains the core base filesystems used by Hugo. The name "base" is used
// to underline that even if they can be composites, they all have a base path set to a specific
// resource folder, e.g "/my-project/content". So, no absolute filenames needed.
type BaseFs struct {
// SourceFilesystems contains the different source file systems.
*SourceFilesystems
// The filesystem used to publish the rendered site.
// This usually maps to /my-project/public.
PublishFs afero.Fs
themeFs afero.Fs
// TODO(bep) improve the "theme interaction"
AbsThemeDirs []string
}
// RelContentDir tries to create a path relative to the content root from
// the given filename. The return value is the path and language code.
func (b *BaseFs) RelContentDir(filename string) string {
for _, dirname := range b.SourceFilesystems.Content.Dirnames {
if strings.HasPrefix(filename, dirname) {
rel := strings.TrimPrefix(filename, dirname)
return strings.TrimPrefix(rel, filePathSeparator)
}
}
// Either not a content dir or already relative.
return filename
}
// SourceFilesystems contains the different source file systems. These can be
// composite file systems (theme and project etc.), and they have all root
// set to the source type the provides: data, i18n, static, layouts.
type SourceFilesystems struct {
Content *SourceFilesystem
Data *SourceFilesystem
I18n *SourceFilesystem
Layouts *SourceFilesystem
Archetypes *SourceFilesystem
Assets *SourceFilesystem
Resources *SourceFilesystem
// This is a unified read-only view of the project's and themes' workdir.
Work *SourceFilesystem
// When in multihost we have one static filesystem per language. The sync
// static files is currently done outside of the Hugo build (where there is
// a concept of a site per language).
// When in non-multihost mode there will be one entry in this map with a blank key.
Static map[string]*SourceFilesystem
}
// A SourceFilesystem holds the filesystem for a given source type in Hugo (data,
// i18n, layouts, static) and additional metadata to be able to use that filesystem
// in server mode.
type SourceFilesystem struct {
// This is a virtual composite filesystem. It expects path relative to a context.
Fs afero.Fs
// This is the base source filesystem. In real Hugo, this will be the OS filesystem.
// Use this if you need to resolve items in Dirnames below.
SourceFs afero.Fs
// Dirnames is absolute filenames to the directories in this filesystem.
Dirnames []string
// When syncing a source folder to the target (e.g. /public), this may
// be set to publish into a subfolder. This is used for static syncing
// in multihost mode.
PublishFolder string
}
// ContentStaticAssetFs will create a new composite filesystem from the content,
// static, and asset filesystems. The site language is needed to pick the correct static filesystem.
// The order is content, static and then assets.
// TODO(bep) check usage
func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs {
staticFs := s.StaticFs(lang)
base := afero.NewCopyOnWriteFs(s.Assets.Fs, staticFs)
return afero.NewCopyOnWriteFs(base, s.Content.Fs)
}
// StaticFs returns the static filesystem for the given language.
// This can be a composite filesystem.
func (s SourceFilesystems) StaticFs(lang string) afero.Fs {
var staticFs afero.Fs = hugofs.NoOpFs
if fs, ok := s.Static[lang]; ok {
staticFs = fs.Fs
} else if fs, ok := s.Static[""]; ok {
staticFs = fs.Fs
}
return staticFs
}
// StatResource looks for a resource in these filesystems in order: static, assets and finally content.
// If found in any of them, it returns FileInfo and the relevant filesystem.
// Any non os.IsNotExist error will be returned.
// An os.IsNotExist error wil be returned only if all filesystems return such an error.
// Note that if we only wanted to find the file, we could create a composite Afero fs,
// but we also need to know which filesystem root it lives in.
func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) {
for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} {
fs = fsToCheck
fi, err = fs.Stat(filename)
if err == nil || !os.IsNotExist(err) {
return
}
}
// Not found.
return
}
// IsStatic returns true if the given filename is a member of one of the static
// filesystems.
func (s SourceFilesystems) IsStatic(filename string) bool {
for _, staticFs := range s.Static {
if staticFs.Contains(filename) {
return true
}
}
return false
}
// IsContent returns true if the given filename is a member of the content filesystem.
func (s SourceFilesystems) IsContent(filename string) bool {
return s.Content.Contains(filename)
}
// IsLayout returns true if the given filename is a member of the layouts filesystem.
func (s SourceFilesystems) IsLayout(filename string) bool {
return s.Layouts.Contains(filename)
}
// IsData returns true if the given filename is a member of the data filesystem.
func (s SourceFilesystems) IsData(filename string) bool {
return s.Data.Contains(filename)
}
// IsAsset returns true if the given filename is a member of the asset filesystem.
func (s SourceFilesystems) IsAsset(filename string) bool {
return s.Assets.Contains(filename)
}
// IsI18n returns true if the given filename is a member of the i18n filesystem.
func (s SourceFilesystems) IsI18n(filename string) bool {
return s.I18n.Contains(filename)
}
// MakeStaticPathRelative makes an absolute static filename into a relative one.
// It will return an empty string if the filename is not a member of a static filesystem.
func (s SourceFilesystems) MakeStaticPathRelative(filename string) string {
for _, staticFs := range s.Static {
rel := staticFs.MakePathRelative(filename)
if rel != "" {
return rel
}
}
return ""
}
// MakePathRelative creates a relative path from the given filename.
// It will return an empty string if the filename is not a member of this filesystem.
func (d *SourceFilesystem) MakePathRelative(filename string) string {
for _, currentPath := range d.Dirnames {
if strings.HasPrefix(filename, currentPath) {
return strings.TrimPrefix(filename, currentPath)
}
}
return ""
}
func (d *SourceFilesystem) RealFilename(rel string) string {
fi, err := d.Fs.Stat(rel)
if err != nil {
return rel
}
if realfi, ok := fi.(hugofs.RealFilenameInfo); ok {
return realfi.RealFilename()
}
return rel
}
// Contains returns whether the given filename is a member of the current filesystem.
func (d *SourceFilesystem) Contains(filename string) bool {
for _, dir := range d.Dirnames {
if strings.HasPrefix(filename, dir) {
return true
}
}
return false
}
// RealDirs gets a list of absolute paths to directories starting from the given
// path.
func (d *SourceFilesystem) RealDirs(from string) []string {
var dirnames []string
for _, dir := range d.Dirnames {
dirname := filepath.Join(dir, from)
if _, err := d.SourceFs.Stat(dirname); err == nil {
dirnames = append(dirnames, dirname)
}
}
return dirnames
}
// WithBaseFs allows reuse of some potentially expensive to create parts that remain
// the same across sites/languages.
func WithBaseFs(b *BaseFs) func(*BaseFs) error {
return func(bb *BaseFs) error {
bb.themeFs = b.themeFs
bb.AbsThemeDirs = b.AbsThemeDirs
return nil
}
}
func newRealBase(base afero.Fs) afero.Fs {
return hugofs.NewBasePathRealFilenameFs(base.(*afero.BasePathFs))
}
// NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase
func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) {
fs := p.Fs
publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)
contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages)
if err != nil {
return nil, err
}
// Make sure we don't have any overlapping content dirs. That will never work.
for i, d1 := range absContentDirs {
for j, d2 := range absContentDirs {
if i == j {
continue
}
if strings.HasPrefix(d1, d2) || strings.HasPrefix(d2, d1) {
return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
}
}
}
b := &BaseFs{
PublishFs: publishFs,
}
for _, opt := range options {
if err := opt(b); err != nil {
return nil, err
}
}
builder := newSourceFilesystemsBuilder(p, b)
sourceFilesystems, err := builder.Build()
if err != nil {
return nil, err
}
sourceFilesystems.Content = &SourceFilesystem{
SourceFs: fs.Source,
Fs: contentFs,
Dirnames: absContentDirs,
}
b.SourceFilesystems = sourceFilesystems
b.themeFs = builder.themeFs
b.AbsThemeDirs = builder.absThemeDirs
return b, nil
}
type sourceFilesystemsBuilder struct {
p *paths.Paths
result *SourceFilesystems
themeFs afero.Fs
hasTheme bool
absThemeDirs []string
}
func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder {
return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}}
}
func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
if b.themeFs == nil && b.p.ThemeSet() {
themeFs, absThemeDirs, err := createThemesOverlayFs(b.p)
if err != nil {
return nil, err
}
if themeFs == nil {
panic("createThemesFs returned nil")
}
b.themeFs = themeFs
b.absThemeDirs = absThemeDirs
}
b.hasTheme = len(b.absThemeDirs) > 0
sfs, err := b.createRootMappingFs("dataDir", "data")
if err != nil {
return nil, err
}
b.result.Data = sfs
sfs, err = b.createRootMappingFs("i18nDir", "i18n")
if err != nil {
return nil, err
}
b.result.I18n = sfs
sfs, err = b.createFs(false, true, "layoutDir", "layouts")
if err != nil {
return nil, err
}
b.result.Layouts = sfs
sfs, err = b.createFs(false, true, "archetypeDir", "archetypes")
if err != nil {
return nil, err
}
b.result.Archetypes = sfs
sfs, err = b.createFs(false, true, "assetDir", "assets")
if err != nil {
return nil, err
}
b.result.Assets = sfs
sfs, err = b.createFs(true, false, "resourceDir", "resources")
if err != nil {
return nil, err
}
b.result.Resources = sfs
sfs, err = b.createFs(false, true, "", "")
if err != nil {
return nil, err
}
b.result.Work = sfs
err = b.createStaticFs()
if err != nil {
return nil, err
}
return b.result, nil
}
func (b *sourceFilesystemsBuilder) createFs(
mkdir bool,
readOnly bool,
dirKey, themeFolder string) (*SourceFilesystem, error) {
s := &SourceFilesystem{
SourceFs: b.p.Fs.Source,
}
if themeFolder == "" {
themeFolder = filePathSeparator
}
var dir string
if dirKey != "" {
dir = b.p.Cfg.GetString(dirKey)
if dir == "" {
return s, fmt.Errorf("config %q not set", dirKey)
}
}
var fs afero.Fs
absDir := b.p.AbsPathify(dir)
existsInSource := b.existsInSource(absDir)
if !existsInSource && mkdir {
// We really need this directory. Make it.
if err := b.p.Fs.Source.MkdirAll(absDir, 0777); err == nil {
existsInSource = true
}
}
if existsInSource {
fs = newRealBase(afero.NewBasePathFs(b.p.Fs.Source, absDir))
s.Dirnames = []string{absDir}
}
if b.hasTheme {
if !strings.HasPrefix(themeFolder, filePathSeparator) {
themeFolder = filePathSeparator + themeFolder
}
themeFolderFs := newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder))
if fs == nil {
fs = themeFolderFs
} else {
fs = afero.NewCopyOnWriteFs(themeFolderFs, fs)
}
for _, absThemeDir := range b.absThemeDirs {
absThemeFolderDir := filepath.Join(absThemeDir, themeFolder)
if b.existsInSource(absThemeFolderDir) {
s.Dirnames = append(s.Dirnames, absThemeFolderDir)
}
}
}
if fs == nil {
s.Fs = hugofs.NoOpFs
} else if readOnly {
s.Fs = afero.NewReadOnlyFs(fs)
} else {
s.Fs = fs
}
return s, nil
}
// Used for data, i18n -- we cannot use overlay filsesystems for those, but we need
// to keep a strict order.
func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) {
s := &SourceFilesystem{
SourceFs: b.p.Fs.Source,
}
projectDir := b.p.Cfg.GetString(dirKey)
if projectDir == "" {
return nil, fmt.Errorf("config %q not set", dirKey)
}
var fromTo []string
to := b.p.AbsPathify(projectDir)
if b.existsInSource(to) {
s.Dirnames = []string{to}
fromTo = []string{projectVirtualFolder, to}
}
for _, theme := range b.p.AllThemes {
to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder))
if b.existsInSource(to) {
s.Dirnames = append(s.Dirnames, to)
from := theme
fromTo = append(fromTo, from.Name, to)
}
}
if len(fromTo) == 0 {
s.Fs = hugofs.NoOpFs
return s, nil
}
fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...)
if err != nil {
return nil, err
}
s.Fs = afero.NewReadOnlyFs(fs)
return s, nil
}
func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool {
exists, _ := afero.Exists(b.p.Fs.Source, abspath)
return exists
}
func (b *sourceFilesystemsBuilder) createStaticFs() error {
isMultihost := b.p.Cfg.GetBool("multihost")
ms := make(map[string]*SourceFilesystem)
b.result.Static = ms
if isMultihost {
for _, l := range b.p.Languages {
s := &SourceFilesystem{
SourceFs: b.p.Fs.Source,
PublishFolder: l.Lang}
staticDirs := removeDuplicatesKeepRight(getStaticDirs(l))
if len(staticDirs) == 0 {
continue
}
for _, dir := range staticDirs {
absDir := b.p.AbsPathify(dir)
if !b.existsInSource(absDir) {
continue
}
s.Dirnames = append(s.Dirnames, absDir)
}
fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames)
if err != nil {
return err
}
if b.hasTheme {
themeFolder := "static"
fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs)
for _, absThemeDir := range b.absThemeDirs {
s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder))
}
}
s.Fs = fs
ms[l.Lang] = s
}
return nil
}
s := &SourceFilesystem{
SourceFs: b.p.Fs.Source,
}
var staticDirs []string
for _, l := range b.p.Languages {
staticDirs = append(staticDirs, getStaticDirs(l)...)
}
staticDirs = removeDuplicatesKeepRight(staticDirs)
if len(staticDirs) == 0 {
return nil
}
for _, dir := range staticDirs {
absDir := b.p.AbsPathify(dir)
if !b.existsInSource(absDir) {
continue
}
s.Dirnames = append(s.Dirnames, absDir)
}
fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames)
if err != nil {
return err
}
if b.hasTheme {
themeFolder := "static"
fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs)
for _, absThemeDir := range b.absThemeDirs {
s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder))
}
}
s.Fs = fs
ms[""] = s
return nil
}
func getStaticDirs(cfg config.Provider) []string {
var staticDirs []string
for i := -1; i <= 10; i++ {
staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
}
return staticDirs
}
func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
if id >= 0 {
key = fmt.Sprintf("%s%d", key, id)
}
return config.GetStringSlicePreserveString(cfg, key)
}
func createContentFs(fs afero.Fs,
workingDir,
defaultContentLanguage string,
languages langs.Languages) (afero.Fs, []string, error) {
var contentLanguages langs.Languages
var contentDirSeen = make(map[string]bool)
languageSet := make(map[string]bool)
// The default content language needs to be first.
for _, language := range languages {
if language.Lang == defaultContentLanguage {
contentLanguages = append(contentLanguages, language)
contentDirSeen[language.ContentDir] = true
}
languageSet[language.Lang] = true
}
for _, language := range languages {
if contentDirSeen[language.ContentDir] {
continue
}
if language.ContentDir == "" {
language.ContentDir = defaultContentLanguage
}
contentDirSeen[language.ContentDir] = true
contentLanguages = append(contentLanguages, language)
}
var absContentDirs []string
fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
return fs, absContentDirs, err
}
func createContentOverlayFs(source afero.Fs,
workingDir string,
languages langs.Languages,
languageSet map[string]bool,
absContentDirs *[]string) (afero.Fs, error) {
if len(languages) == 0 {
return source, nil
}
language := languages[0]
contentDir := language.ContentDir
if contentDir == "" {
panic("missing contentDir")
}
absContentDir := paths.AbsPathify(workingDir, language.ContentDir)
if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) {
absContentDir += paths.FilePathSeparator
}
// If root, remove the second '/'
if absContentDir == "//" {
absContentDir = paths.FilePathSeparator
}
if len(absContentDir) < 6 {
return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir)
}
*absContentDirs = append(*absContentDirs, absContentDir)
overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
if len(languages) == 1 {
return overlay, nil
}
base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)
if err != nil {
return nil, err
}
return hugofs.NewLanguageCompositeFs(base, overlay), nil
}
func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) {
themes := p.AllThemes
if len(themes) == 0 {
panic("AllThemes not set")
}
themesDir := p.AbsPathify(p.ThemesDir)
if themesDir == "" {
return nil, nil, errors.New("no themes dir set")
}
absPaths := make([]string, len(themes))
// The themes are ordered from left to right. We need to revert it to get the
// overlay logic below working as expected.
for i := 0; i < len(themes); i++ {
absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name)
}
fs, err := createOverlayFs(p.Fs.Source, absPaths)
fs = hugofs.NewNoLstatFs(fs)
return fs, absPaths, err
}
func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) {
if len(absPaths) == 0 {
return hugofs.NoOpFs, nil
}
if len(absPaths) == 1 {
return afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))), nil
}
base := afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0])))
overlay, err := createOverlayFs(source, absPaths[1:])
if err != nil {
return nil, err
}
return afero.NewCopyOnWriteFs(base, overlay), nil
}
func removeDuplicatesKeepRight(in []string) []string {
seen := make(map[string]bool)
var out []string
for i := len(in) - 1; i >= 0; i-- {
v := in[i]
if seen[v] {
continue
}
out = append([]string{v}, out...)
seen[v] = true
}
return out
}