shithub: hugo

Info  •  Files  •  Log  •  Branches

Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

--- a/benchbep.sh

+++ b/benchbep.sh

@@ -1,2 +1,1 @@

-gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench

-benchcmp -best 0.bench 1.bench

\ No newline at end of file

+gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"

\ No newline at end of file

--- a/cache/filecache/filecache.go

+++ b/cache/filecache/filecache.go

@@ -44,6 +44,9 @@

 	// 0 is effectively turning this cache off.

 	maxAge time.Duration

+	// When set, we just remove this entire root directory on expiration.

+	pruneAllRootDir string

+

 	nlocker *lockTracker

 }

@@ -77,11 +80,12 @@

 }

 // NewCache creates a new file cache with the given filesystem and max age.

-func NewCache(fs afero.Fs, maxAge time.Duration) *Cache {

+func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {

 	return &Cache{

-		Fs:      fs,

-		nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},

-		maxAge:  maxAge,

+		Fs:              fs,

+		nlocker:         &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},

+		maxAge:          maxAge,

+		pruneAllRootDir: pruneAllRootDir,

 	}

 }

@@ -307,9 +311,15 @@

 // NewCaches creates a new set of file caches from the given

 // configuration.

 func NewCaches(p *helpers.PathSpec) (Caches, error) {

-	dcfg, err := decodeConfig(p)

-	if err != nil {

-		return nil, err

+	var dcfg Configs

+	if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {

+		dcfg = c

+	} else {

+		var err error

+		dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)

+		if err != nil {

+			return nil, err

+		}

 	}

 	fs := p.Fs.Source

@@ -319,30 +329,25 @@

 		var cfs afero.Fs

 		if v.isResourceDir {

-			cfs = p.BaseFs.Resources.Fs

+			cfs = p.BaseFs.ResourcesCache

 		} else {

 			cfs = fs

 		}

-		var baseDir string

-		if !strings.HasPrefix(v.Dir, "_gen") {

-			// We do cache eviction (file removes) and since the user can set

-			// his/hers own cache directory, we really want to make sure

-			// we do not delete any files that do not belong to this cache.

-			// We do add the cache name as the root, but this is an extra safe

-			// guard. We skip the files inside /resources/_gen/ because

-			// that would be breaking.

-			baseDir = filepath.Join(v.Dir, filecacheRootDirname, k)

-		} else {

-			baseDir = filepath.Join(v.Dir, k)

-		}

-		if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {

+		baseDir := v.Dir

+

+		if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {

 			return nil, err

 		}

 		bfs := afero.NewBasePathFs(cfs, baseDir)

-		m[k] = NewCache(bfs, v.MaxAge)

+		var pruneAllRootDir string

+		if k == cacheKeyModules {

+			pruneAllRootDir = "pkg"

+		}

+

+		m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)

 	}

 	return m, nil

--- a/cache/filecache/filecache_config.go

+++ b/cache/filecache/filecache_config.go

@@ -19,6 +19,8 @@

 	"strings"

 	"time"

+	"github.com/gohugoio/hugo/config"

+

 	"github.com/gohugoio/hugo/helpers"

 	"github.com/mitchellh/mapstructure"

@@ -32,7 +34,7 @@

 	resourcesGenDir = ":resourceDir/_gen"

 )

-var defaultCacheConfig = cacheConfig{

+var defaultCacheConfig = Config{

 	MaxAge: -1, // Never expire

 	Dir:    ":cacheDir/:project",

 }

@@ -42,9 +44,20 @@

 	cacheKeyGetCSV  = "getcsv"

 	cacheKeyImages  = "images"

 	cacheKeyAssets  = "assets"

+	cacheKeyModules = "modules"

 )

-var defaultCacheConfigs = map[string]cacheConfig{

+type Configs map[string]Config

+

+func (c Configs) CacheDirModules() string {

+	return c[cacheKeyModules].Dir

+}

+

+var defaultCacheConfigs = Configs{

+	cacheKeyModules: {

+		MaxAge: -1,

+		Dir:    ":cacheDir/modules",

+	},

 	cacheKeyGetJSON: defaultCacheConfig,

 	cacheKeyGetCSV:  defaultCacheConfig,

 	cacheKeyImages: {

@@ -57,9 +70,7 @@

 	},

 }

-type cachesConfig map[string]cacheConfig

-

-type cacheConfig struct {

+type Config struct {

 	// Max age of cache entries in this cache. Any items older than this will

 	// be removed and not returned from the cache.

 	// a negative value means forever, 0 means cache is disabled.

@@ -88,13 +99,18 @@

 	return f[cacheKeyImages]

 }

+// ModulesCache gets the file cache for Hugo Modules.

+func (f Caches) ModulesCache() *Cache {

+	return f[cacheKeyModules]

+}

+

 // AssetsCache gets the file cache for assets (processed resources, SCSS etc.).

 func (f Caches) AssetsCache() *Cache {

 	return f[cacheKeyAssets]

 }

-func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {

-	c := make(cachesConfig)

+func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) {

+	c := make(Configs)

 	valid := make(map[string]bool)

 	// Add defaults

 	for k, v := range defaultCacheConfigs {

@@ -102,11 +118,9 @@

 		valid[k] = true

 	}

-	cfg := p.Cfg

-

 	m := cfg.GetStringMap(cachesConfigKey)

-	_, isOsFs := p.Fs.Source.(*afero.OsFs)

+	_, isOsFs := fs.(*afero.OsFs)

 	for k, v := range m {

 		cc := defaultCacheConfig

@@ -148,7 +162,7 @@

 		for i, part := range parts {

 			if strings.HasPrefix(part, ":") {

-				resolved, isResource, err := resolveDirPlaceholder(p, part)

+				resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part)

 				if err != nil {

 					return c, err

 				}

@@ -176,6 +190,18 @@

 			}

 		}

+		if !strings.HasPrefix(v.Dir, "_gen") {

+			// We do cache eviction (file removes) and since the user can set

+			// his/hers own cache directory, we really want to make sure

+			// we do not delete any files that do not belong to this cache.

+			// We do add the cache name as the root, but this is an extra safe

+			// guard. We skip the files inside /resources/_gen/ because

+			// that would be breaking.

+			v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k)

+		} else {

+			v.Dir = filepath.Join(v.Dir, k)

+		}

+

 		if disabled {

 			v.MaxAge = 0

 		}

@@ -187,15 +213,17 @@

 }

 // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...

-func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {

+func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) {

+	workingDir := cfg.GetString("workingDir")

+

 	switch strings.ToLower(placeholder) {

 	case ":resourcedir":

 		return "", true, nil

 	case ":cachedir":

-		d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)

+		d, err := helpers.GetCacheDir(fs, cfg)

 		return d, false, err

 	case ":project":

-		return filepath.Base(p.WorkingDir), false, nil

+		return filepath.Base(workingDir), false, nil

 	}

 	return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)

--- a/cache/filecache/filecache_config_test.go

+++ b/cache/filecache/filecache_config_test.go

@@ -20,10 +20,9 @@

 	"testing"

 	"time"

-	"github.com/gohugoio/hugo/helpers"

+	"github.com/spf13/afero"

 	"github.com/gohugoio/hugo/config"

-	"github.com/gohugoio/hugo/hugofs"

 	"github.com/spf13/viper"

 	"github.com/stretchr/testify/require"

@@ -57,22 +56,19 @@

 	cfg, err := config.FromConfigString(configStr, "toml")

 	assert.NoError(err)

-	fs := hugofs.NewMem(cfg)

-	p, err := helpers.NewPathSpec(fs, cfg)

+	fs := afero.NewMemMapFs()

+	decoded, err := DecodeConfig(fs, cfg)

 	assert.NoError(err)

-	decoded, err := decodeConfig(p)

-	assert.NoError(err)

+	assert.Equal(5, len(decoded))

-	assert.Equal(4, len(decoded))

-

 	c2 := decoded["getcsv"]

 	assert.Equal("11h0m0s", c2.MaxAge.String())

-	assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)

+	assert.Equal(filepath.FromSlash("/path/to/c2/filecache/getcsv"), c2.Dir)

 	c3 := decoded["images"]

 	assert.Equal(time.Duration(-1), c3.MaxAge)

-	assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)

+	assert.Equal(filepath.FromSlash("/path/to/c3/filecache/images"), c3.Dir)

 }

@@ -105,15 +101,12 @@

 	cfg, err := config.FromConfigString(configStr, "toml")

 	assert.NoError(err)

-	fs := hugofs.NewMem(cfg)

-	p, err := helpers.NewPathSpec(fs, cfg)

+	fs := afero.NewMemMapFs()

+	decoded, err := DecodeConfig(fs, cfg)

 	assert.NoError(err)

-	decoded, err := decodeConfig(p)

-	assert.NoError(err)

+	assert.Equal(5, len(decoded))

-	assert.Equal(4, len(decoded))

-

 	for _, v := range decoded {

 		assert.Equal(time.Duration(0), v.MaxAge)

 	}

@@ -133,24 +126,22 @@

 		cfg.Set("cacheDir", "/cache/thecache")

 	}

-	fs := hugofs.NewMem(cfg)

-	p, err := helpers.NewPathSpec(fs, cfg)

-	assert.NoError(err)

+	fs := afero.NewMemMapFs()

-	decoded, err := decodeConfig(p)

+	decoded, err := DecodeConfig(fs, cfg)

 	assert.NoError(err)

-	assert.Equal(4, len(decoded))

+	assert.Equal(5, len(decoded))

 	imgConfig := decoded[cacheKeyImages]

 	jsonConfig := decoded[cacheKeyGetJSON]

 	if runtime.GOOS == "windows" {

-		assert.Equal("_gen", imgConfig.Dir)

+		assert.Equal(filepath.FromSlash("_gen/images"), imgConfig.Dir)

 	} else {

-		assert.Equal("_gen", imgConfig.Dir)

-		assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir)

+		assert.Equal("_gen/images", imgConfig.Dir)

+		assert.Equal("/cache/thecache/hugoproject/filecache/getjson", jsonConfig.Dir)

 	}

 	assert.True(imgConfig.isResourceDir)

@@ -183,11 +174,9 @@

 	cfg, err := config.FromConfigString(configStr, "toml")

 	assert.NoError(err)

-	fs := hugofs.NewMem(cfg)

-	p, err := helpers.NewPathSpec(fs, cfg)

-	assert.NoError(err)

+	fs := afero.NewMemMapFs()

-	_, err = decodeConfig(p)

+	_, err = DecodeConfig(fs, cfg)

 	assert.Error(err)

 }

--- a/cache/filecache/filecache_pruner.go

+++ b/cache/filecache/filecache_pruner.go

@@ -28,53 +28,100 @@

 func (c Caches) Prune() (int, error) {

 	counter := 0

 	for k, cache := range c {

-		err := afero.Walk(cache.Fs, "", func(name string, info os.FileInfo, err error) error {

-			if info == nil {

-				return nil

-			}

-			name = cleanID(name)

+		count, err := cache.Prune(false)

-			if info.IsDir() {

-				f, err := cache.Fs.Open(name)

-				if err != nil {

-					// This cache dir may not exist.

-					return nil

-				}

-				defer f.Close()

-				_, err = f.Readdirnames(1)

-				if err == io.EOF {

-					// Empty dir.

-					return cache.Fs.Remove(name)

-				}

+		if err != nil {

+			return counter, errors.Wrapf(err, "failed to prune cache %q", k)

+		}

+		counter += count

+

+	}

+

+	return counter, nil

+}

+

+// Prune removes expired and unused items from this cache.

+// If force is set, everything will be removed not considering expiry time.

+func (c *Cache) Prune(force bool) (int, error) {

+	if c.pruneAllRootDir != "" {

+		return c.pruneRootDir(force)

+	}

+

+	counter := 0

+

+	err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error {

+		if info == nil {

+			return nil

+		}

+

+		name = cleanID(name)

+

+		if info.IsDir() {

+			f, err := c.Fs.Open(name)

+			if err != nil {

+				// This cache dir may not exist.

 				return nil

 			}

+			defer f.Close()

+			_, err = f.Readdirnames(1)

+			if err == io.EOF {

+				// Empty dir.

+				return c.Fs.Remove(name)

+			}

-			shouldRemove := cache.isExpired(info.ModTime())

+			return nil

+		}

-			if !shouldRemove && len(cache.nlocker.seen) > 0 {

-				// Remove it if it's not been touched/used in the last build.

-				_, seen := cache.nlocker.seen[name]

-				shouldRemove = !seen

-			}

+		shouldRemove := force || c.isExpired(info.ModTime())

-			if shouldRemove {

-				err := cache.Fs.Remove(name)

-				if err == nil {

-					counter++

-				}

-				return err

+		if !shouldRemove && len(c.nlocker.seen) > 0 {

+			// Remove it if it's not been touched/used in the last build.

+			_, seen := c.nlocker.seen[name]

+			shouldRemove = !seen

+		}

+

+		if shouldRemove {

+			err := c.Fs.Remove(name)

+			if err == nil {

+				counter++

 			}

+			return err

+		}

-			return nil

-		})

+		return nil

+	})

-		if err != nil {

-			return counter, errors.Wrapf(err, "failed to prune cache %q", k)

+	return counter, err

+}

+

+func (c *Cache) pruneRootDir(force bool) (int, error) {

+

+	info, err := c.Fs.Stat(c.pruneAllRootDir)

+	if err != nil {

+		if os.IsNotExist(err) {

+			return 0, nil

 		}

+		return 0, err

+	}

+	if !force && !c.isExpired(info.ModTime()) {

+		return 0, nil

 	}

-	return counter, nil

+	counter := 0

+	// Module cache has 0555 directories; make them writable in order to remove content.

+	afero.Walk(c.Fs, c.pruneAllRootDir, func(path string, info os.FileInfo, err error) error {

+		if err != nil {

+			return nil

+		}

+		if info.IsDir() {

+			counter++

+			c.Fs.Chmod(path, 0777)

+		}

+		return nil

+	})

+	return 1, c.Fs.RemoveAll(c.pruneAllRootDir)

+

 }

--- a/cache/filecache/filecache_pruner_test.go

+++ b/cache/filecache/filecache_pruner_test.go

@@ -18,9 +18,7 @@

 	"testing"

 	"time"

-	"github.com/gohugoio/hugo/config"

-	"github.com/gohugoio/hugo/helpers"

-	"github.com/gohugoio/hugo/hugofs"

+	"github.com/spf13/afero"

 	"github.com/stretchr/testify/require"

 )

@@ -54,14 +52,9 @@

 dir = ":resourceDir/_gen"

 `

-	cfg, err := config.FromConfigString(configStr, "toml")

-	assert.NoError(err)

-

 	for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} {

 		msg := fmt.Sprintf("cache: %s", name)

-		fs := hugofs.NewMem(cfg)

-		p, err := helpers.NewPathSpec(fs, cfg)

-		assert.NoError(err)

+		p := newPathsSpec(t, afero.NewMemMapFs(), configStr)

 		caches, err := NewCaches(p)

 		assert.NoError(err)

 		cache := caches[name]

--- a/cache/filecache/filecache_test.go

+++ b/cache/filecache/filecache_test.go

@@ -25,6 +25,9 @@

 	"testing"

 	"time"

+	"github.com/gohugoio/hugo/langs"

+	"github.com/gohugoio/hugo/modules"

+

 	"github.com/gohugoio/hugo/common/hugio"

 	"github.com/gohugoio/hugo/config"

 	"github.com/gohugoio/hugo/helpers"

@@ -83,13 +86,8 @@

 		configStr = replacer.Replace(configStr)

 		configStr = strings.Replace(configStr, "\\", winPathSep, -1)

-		cfg, err := config.FromConfigString(configStr, "toml")

-		assert.NoError(err)

+		p := newPathsSpec(t, osfs, configStr)

-		fs := hugofs.NewFrom(osfs, cfg)

-		p, err := helpers.NewPathSpec(fs, cfg)

-		assert.NoError(err)

-

 		caches, err := NewCaches(p)

 		assert.NoError(err)

@@ -207,11 +205,7 @@

 `

-	cfg, err := config.FromConfigString(configStr, "toml")

-	assert.NoError(err)

-	fs := hugofs.NewMem(cfg)

-	p, err := helpers.NewPathSpec(fs, cfg)

-	assert.NoError(err)

+	p := newPathsSpec(t, afero.NewMemMapFs(), configStr)

 	caches, err := NewCaches(p)

 	assert.NoError(err)

@@ -254,4 +248,52 @@

 	assert := require.New(t)

 	assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("/a/b//c.txt")))

 	assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("a/b//c.txt")))

+}

+

+func initConfig(fs afero.Fs, cfg config.Provider) error {

+	if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {

+		return err

+	}

+

+	modConfig, err := modules.DecodeConfig(cfg)

+	if err != nil {

+		return err

+	}

+

+	workingDir := cfg.GetString("workingDir")

+	themesDir := cfg.GetString("themesDir")

+	if !filepath.IsAbs(themesDir) {

+		themesDir = filepath.Join(workingDir, themesDir)

+	}

+	modulesClient := modules.NewClient(modules.ClientConfig{

+		Fs:           fs,

+		WorkingDir:   workingDir,

+		ThemesDir:    themesDir,

+		ModuleConfig: modConfig,

+		IgnoreVendor: true,

+	})

+

+	moduleConfig, err := modulesClient.Collect()

+	if err != nil {

+		return err

+	}

+

+	if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil {

+		return err

+	}

+

+	cfg.Set("allModules", moduleConfig.ActiveModules)

+

+	return nil

+}

+

+func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec {

+	assert := require.New(t)

+	cfg, err := config.FromConfigString(configStr, "toml")

+	assert.NoError(err)

+	initConfig(fs, cfg)

+	p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg)

+	assert.NoError(err)

+	return p

+

 }

--- a/commands/commandeer.go

+++ b/commands/commandeer.go

@@ -16,7 +16,12 @@

 import (

 	"bytes"

 	"errors"

+	"sync"

+	"golang.org/x/sync/semaphore"

+

+	"github.com/gohugoio/hugo/modules"

+

 	"io/ioutil"

 	"github.com/gohugoio/hugo/common/herrors"

@@ -27,8 +32,6 @@

 	"os"

 	"path/filepath"

 	"regexp"

-	"strings"

-	"sync"

 	"time"

 	"github.com/gohugoio/hugo/common/loggers"

@@ -88,6 +91,8 @@

 	configured bool

 	paused     bool

+	fullRebuildSem *semaphore.Weighted

+

 	// Any error from the last build.

 	buildErr error

 }

@@ -153,6 +158,7 @@

 		doWithCommandeer:    doWithCommandeer,

 		visitedURLs:         types.NewEvictingStringQueue(10),

 		debounce:            rebuildDebouncer,

+		fullRebuildSem:      semaphore.NewWeighted(1),

 		// This will be replaced later, but we need something to log to before the configuration is read.

 		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),

 	}

@@ -282,6 +288,7 @@

 			WorkingDir:   dir,

 			Filename:     c.h.cfgFile,

 			AbsConfigDir: c.h.getConfigDir(dir),

+			Environ:      os.Environ(),

 			Environment:  environment},

 		doWithCommandeer,

 		doWithConfig)

@@ -290,7 +297,7 @@

 		if mustHaveConfigFile {

 			return err

 		}

-		if err != hugolib.ErrNoConfigFile {

+		if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) {

 			return err

 		}

@@ -387,21 +394,6 @@

 	config.Set("cacheDir", cacheDir)

 	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())

-

-	themeDir := c.hugo.PathSpec.GetFirstThemeDir()

-	if themeDir != "" {

-		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {

-			return newSystemError("Unable to find theme Directory:", themeDir)

-		}

-	}

-

-	dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)

-

-	if themeVersionMismatch {

-		name := filepath.Base(dir)

-		cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",

-			strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)

-	}

 	return nil

--- a/commands/commands.go

+++ b/commands/commands.go

@@ -54,6 +54,7 @@

 		newImportCmd(),

 		newGenCmd(),

 		createReleaser(),

+		b.newModCmd(),

 	)

 	return b

@@ -243,13 +244,20 @@

 	return hugo.EnvironmentProduction

 }

+func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) {

+	cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")

+	cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})

+	cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment")

+	cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory")

+	cmd.PersistentFlags().BoolP("ignoreVendor", "", false, "ignores any _vendor directory")

+}

+

 func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {

+	cc.handleCommonBuilderFlags(cmd)

 	cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories")

 	cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft")

 	cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future")

 	cmd.Flags().BoolP("buildExpired", "E", false, "include expired content")

-	cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")

-	cmd.Flags().StringVarP(&cc.environment, "environment", "e", "", "build environment")

 	cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")

 	cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory")

 	cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")

@@ -256,7 +264,6 @@

 	cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory")

 	cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to")

 	cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)")

-	cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory")

 	cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/")

 	cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages")

 	cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build")

--- a/commands/convert.go

+++ b/commands/convert.go

@@ -30,7 +30,6 @@

 	"github.com/gohugoio/hugo/parser/metadecoders"

 	"github.com/gohugoio/hugo/parser/pageparser"

-	src "github.com/gohugoio/hugo/source"

 	"github.com/pkg/errors"

 	"github.com/gohugoio/hugo/hugolib"

@@ -152,8 +151,8 @@

 	site.Log.INFO.Println("Attempting to convert", p.File().Filename())

-	f, _ := p.File().(src.ReadableFile)

-	file, err := f.Open()

+	f := p.File()

+	file, err := f.FileInfo().Meta().Open()

 	if err != nil {

 		site.Log.ERROR.Println(errMsg)

 		file.Close()

--- a/commands/hugo.go

+++ b/commands/hugo.go

@@ -16,12 +16,12 @@

 package commands

 import (

+	"context"

 	"fmt"

 	"io/ioutil"

 	"os/signal"

 	"runtime/pprof"

 	"runtime/trace"

-	"sort"

 	"sync/atomic"

 	"github.com/gohugoio/hugo/hugofs"

@@ -28,7 +28,6 @@

 	"github.com/gohugoio/hugo/resources/page"

-	"github.com/gohugoio/hugo/common/hugo"

 	"github.com/pkg/errors"

 	"github.com/gohugoio/hugo/common/herrors"

@@ -49,7 +48,6 @@

 	"github.com/gohugoio/hugo/config"

-	"github.com/gohugoio/hugo/parser/metadecoders"

 	flag "github.com/spf13/pflag"

 	"github.com/fsnotify/fsnotify"

@@ -196,6 +194,7 @@

 		"forceSyncStatic",

 		"noTimes",

 		"noChmod",

+		"ignoreVendor",

 		"templateMetrics",

 		"templateMetricsHints",

@@ -291,6 +290,7 @@

 }

 func (c *commandeer) fullBuild() error {

+

 	var (

 		g         errgroup.Group

 		langCount map[string]uint64

@@ -309,13 +309,9 @@

 		cnt, err := c.copyStatic()

 		if err != nil {

-			if !os.IsNotExist(err) {

-				return errors.Wrap(err, "Error copying static files")

-			}

-			c.logger.INFO.Println("No Static directory found")

+			return errors.Wrap(err, "Error copying static files")

 		}

 		langCount = cnt

-		langCount = cnt

 		return nil

 	}

 	buildSitesFunc := func() error {

@@ -503,7 +499,11 @@

 		if err != nil {

 			return err

 		}

-		c.logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir")))

+

+		baseWatchDir := c.Cfg.GetString("workingDir")

+		rootWatchDirs := getRootWatchDirsStr(baseWatchDir, watchDirs)

+

+		c.logger.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)

 		c.logger.FEEDBACK.Println("Press Ctrl+C to stop")

 		watcher, err := c.newWatcher(watchDirs...)

 		checkErr(c.Logger, err)

@@ -547,7 +547,11 @@

 }

 func (c *commandeer) copyStatic() (map[string]uint64, error) {

-	return c.doWithPublishDirs(c.copyStaticTo)

+	m, err := c.doWithPublishDirs(c.copyStaticTo)

+	if err == nil || os.IsNotExist(err) {

+		return m, nil

+	}

+	return m, err

 }

 func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) {

@@ -566,6 +570,7 @@

 		if err != nil {

 			return langCount, err

 		}

+

 		if lang == "" {

 			// Not multihost

 			for _, l := range c.languages {

@@ -594,6 +599,16 @@

 	return f, err

 }

+func chmodFilter(dst, src os.FileInfo) bool {

+	// Hugo publishes data from multiple sources, potentially

+	// with overlapping directory structures. We cannot sync permissions

+	// for directories as that would mean that we might end up with write-protected

+	// directories inside /public.

+	// One example of this would be syncing from the Go Module cache,

+	// which have 0555 directories.

+	return src.IsDir()

+}

+

 func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) {

 	publishDir := c.hugo.PathSpec.PublishDir

 	// If root, remove the second '/'

@@ -610,6 +625,7 @@

 	syncer := fsync.NewSyncer()

 	syncer.NoTimes = c.Cfg.GetBool("noTimes")

 	syncer.NoChmod = c.Cfg.GetBool("noChmod")

+	syncer.ChmodFilter = chmodFilter

 	syncer.SrcFs = fs

 	syncer.DestFs = c.Fs.Destination

 	// Now that we are using a unionFs for the static directories

@@ -652,120 +668,39 @@

 // getDirList provides NewWatcher() with a list of directories to watch for changes.

 func (c *commandeer) getDirList() ([]string, error) {

-	var a []string

+	var dirnames []string

-	// To handle nested symlinked content dirs

-	var seen = make(map[string]bool)

-	var nested []string

+	walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {

+		if err != nil {

+			c.logger.ERROR.Println("walker: ", err)

+			return nil

+		}

-	newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error {

-		return func(path string, fi os.FileInfo, err error) error {

-			if err != nil {

-				if os.IsNotExist(err) {

-					return nil

-				}

-

-				c.logger.ERROR.Println("Walker: ", err)

-				return nil

+		if fi.IsDir() {

+			if fi.Name() == ".git" ||

+				fi.Name() == "node_modules" || fi.Name() == "bower_components" {

+				return filepath.SkipDir

 			}

-			// Skip .git directories.

-			// Related to https://github.com/gohugoio/hugo/issues/3468.

-			if fi.Name() == ".git" {

-				return nil

-			}

-

-			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {

-				link, err := filepath.EvalSymlinks(path)

-				if err != nil {

-					c.logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err)

-					return nil

-				}

-				linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link)

-				if err != nil {

-					c.logger.ERROR.Printf("Cannot stat %q: %s", link, err)

-					return nil

-				}

-				if !allowSymbolicDirs && !linkfi.Mode().IsRegular() {

-					c.logger.ERROR.Printf("Symbolic links for directories not supported, skipping %q", path)

-					return nil

-				}

-

-				if allowSymbolicDirs && linkfi.IsDir() {

-					// afero.Walk will not walk symbolic links, so wee need to do it.

-					if !seen[path] {

-						seen[path] = true

-						nested = append(nested, path)

-					}

-					return nil

-				}

-

-				fi = linkfi

-			}

-

-			if fi.IsDir() {

-				if fi.Name() == ".git" ||

-					fi.Name() == "node_modules" || fi.Name() == "bower_components" {

-					return filepath.SkipDir

-				}

-				a = append(a, path)

-			}

-			return nil

+			dirnames = append(dirnames, fi.Meta().Filename())

 		}

-	}

-	symLinkWalker := newWalker(true)

-	regularWalker := newWalker(false)

+		return nil

-	// SymbolicWalk will log anny ERRORs

-	// Also note that the Dirnames fetched below will contain any relevant theme

-	// directories.

-	for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames {

-		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker)

 	}

-	for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames {

-		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)

-	}

+	watchDirs := c.hugo.PathSpec.BaseFs.WatchDirs()

+	for _, watchDir := range watchDirs {

-	for _, staticDir := range c.hugo.PathSpec.BaseFs.I18n.Dirnames {

-		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)

-	}

-

-	for _, staticDir := range c.hugo.PathSpec.BaseFs.Layouts.Dirnames {

-		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)

-	}

-

-	for _, staticFilesystem := range c.hugo.PathSpec.BaseFs.Static {

-		for _, staticDir := range staticFilesystem.Dirnames {

-			_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)

+		w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: watchDir, WalkFn: walkFn})

+		if err := w.Walk(); err != nil {

+			c.logger.ERROR.Println("walker: ", err)

 		}

 	}

-	for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames {

-		_ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker)

-	}

+	dirnames = helpers.UniqueStringsSorted(dirnames)

-	if len(nested) > 0 {

-		for {

-

-			toWalk := nested

-			nested = nested[:0]

-

-			for _, d := range toWalk {

-				_ = helpers.SymbolicWalk(c.Fs.Source, d, symLinkWalker)

-			}

-

-			if len(nested) == 0 {

-				break

-			}

-		}

-	}

-

-	a = helpers.UniqueStrings(a)

-	sort.Strings(a)

-

-	return a, nil

+	return dirnames, nil

 }

 func (c *commandeer) buildSites() (err error) {

@@ -812,26 +747,60 @@

 	return c.hugo.Build(hugolib.BuildCfg{RecentlyVisited: visited, PartialReRender: true})

 }

-func (c *commandeer) fullRebuild() {

-	c.commandeerHugoState = &commandeerHugoState{}

-	err := c.loadConfig(true, true)

-	if err != nil {

-		// Set the processing on pause until the state is recovered.

-		c.paused = true

-		c.handleBuildErr(err, "Failed to reload config")

-

-	} else {

-		c.paused = false

+func (c *commandeer) fullRebuild(changeType string) {

+	if changeType == configChangeGoMod {

+		// go.mod may be changed during the build itself, and

+		// we really want to prevent superfluous builds.

+		if !c.fullRebuildSem.TryAcquire(1) {

+			return

+		}

+		c.fullRebuildSem.Release(1)

 	}

-	if !c.paused {

-		err := c.buildSites()

+	c.fullRebuildSem.Acquire(context.Background(), 1)

+

+	go func() {

+

+		defer c.fullRebuildSem.Release(1)

+

+		c.printChangeDetected(changeType)

+

+		defer func() {

+

+			// Allow any file system events to arrive back.

+			// This will block any rebuild on config changes for the

+			// duration of the sleep.

+			time.Sleep(2 * time.Second)

+		}()

+

+		defer c.timeTrack(time.Now(), "Total")

+

+		c.commandeerHugoState = &commandeerHugoState{}

+		err := c.loadConfig(true, true)

 		if err != nil {

-			c.logger.ERROR.Println(err)

-		} else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {

-			livereload.ForceRefresh()

+			// Set the processing on pause until the state is recovered.

+			c.paused = true

+			c.handleBuildErr(err, "Failed to reload config")

+

+		} else {

+			c.paused = false

 		}

-	}

+

+		if !c.paused {

+			_, err := c.copyStatic()

+			if err != nil {

+				c.logger.ERROR.Println(err)

+				return

+			}

+

+			err = c.buildSites()

+			if err != nil {

+				c.logger.ERROR.Println(err)

+			} else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {

+				livereload.ForceRefresh()

+			}

+		}

+	}()

 }

 // newWatcher creates a new watcher to watch filesystem events.

@@ -886,13 +855,38 @@

 	return watcher, nil

 }

+func (c *commandeer) printChangeDetected(typ string) {

+	msg := "\nChange"

+	if typ != "" {

+		msg += " of " + typ

+	}

+	msg += " detected, rebuilding site."

+

+	c.logger.FEEDBACK.Println(msg)

+	const layout = "2006-01-02 15:04:05.000 -0700"

+	c.logger.FEEDBACK.Println(time.Now().Format(layout))

+}

+

+const (

+	configChangeConfig = "config file"

+	configChangeGoMod  = "go.mod file"

+)

+

 func (c *commandeer) handleEvents(watcher *watcher.Batcher,

 	staticSyncer *staticSyncer,

 	evs []fsnotify.Event,

 	configSet map[string]bool) {

+	var isHandled bool

+

 	for _, ev := range evs {

 		isConfig := configSet[ev.Name]

+		configChangeType := configChangeConfig

+		if isConfig {

+			if strings.Contains(ev.Name, "go.mod") {

+				configChangeType = configChangeGoMod

+			}

+		}

 		if !isConfig {

 			// It may be one of the /config folders

 			dirname := filepath.Dir(ev.Name)

@@ -899,13 +893,15 @@

 			if dirname != "." && configSet[dirname] {

 				isConfig = true

 			}

-

 		}

 		if isConfig {

+			isHandled = true

+

 			if ev.Op&fsnotify.Chmod == fsnotify.Chmod {

 				continue

 			}

+

 			if ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename {

 				for _, configFile := range c.configFiles {

 					counter := 0

@@ -917,13 +913,20 @@

 						time.Sleep(100 * time.Millisecond)

 					}

 				}

+

+				// A write event will follow.

+				continue

 			}

+

 			// Config file(s) changed. Need full rebuild.

-			c.fullRebuild()

-			break

+			c.fullRebuild(configChangeType)

 		}

 	}

+	if isHandled {

+		return

+	}

+

 	if c.paused {

 		// Wait for the server to get into a consistent state before

 		// we continue with processing.

@@ -933,7 +936,9 @@

 	if len(evs) > 50 {

 		// This is probably a mass edit of the content dir.

 		// Schedule a full rebuild for when it slows down.

-		c.debounce(c.fullRebuild)

+		c.debounce(func() {

+			c.fullRebuild("")

+		})

 		return

 	}

@@ -1015,7 +1020,7 @@

 			continue

 		}

-		walkAdder := func(path string, f os.FileInfo, err error) error {

+		walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error {

 			if f.IsDir() {

 				c.logger.FEEDBACK.Println("adding created directory to watchlist", path)

 				if err := watcher.Add(path); err != nil {

@@ -1046,9 +1051,7 @@

 	}

 	if len(staticEvents) > 0 {

-		c.logger.FEEDBACK.Println("\nStatic file changes detected")

-		const layout = "2006-01-02 15:04:05.000 -0700"

-		c.logger.FEEDBACK.Println(time.Now().Format(layout))

+		c.printChangeDetected("Static files")

 		if c.Cfg.GetBool("forceSyncStatic") {

 			c.logger.FEEDBACK.Printf("Syncing all static files\n")

@@ -1087,10 +1090,7 @@

 		doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload")

 		onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents)

-		c.logger.FEEDBACK.Println("\nChange detected, rebuilding site")

-		const layout = "2006-01-02 15:04:05.000 -0700"

-		c.logger.FEEDBACK.Println(time.Now().Format(layout))

-

+		c.printChangeDetected("")

 		c.changeDetector.PrepareNew()

 		if err := c.rebuildSites(dynamicEvents); err != nil {

 			c.handleBuildErr(err, "Rebuild failed")

@@ -1166,42 +1166,4 @@

 	}

 	return name

-}

-

-// isThemeVsHugoVersionMismatch returns whether the current Hugo version is

-// less than any of the themes' min_version.

-func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mismatch bool, requiredMinVersion string) {

-	if !c.hugo.PathSpec.ThemeSet() {

-		return

-	}

-

-	for _, absThemeDir := range c.hugo.BaseFs.AbsThemeDirs {

-

-		path := filepath.Join(absThemeDir, "theme.toml")

-

-		exists, err := helpers.Exists(path, fs)

-

-		if err != nil || !exists {

-			continue

-		}

-

-		b, err := afero.ReadFile(fs, path)

-		if err != nil {

-			continue

-		}

-

-		tomlMeta, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.TOML)

-		if err != nil {

-			continue

-		}

-

-		if minVersion, ok := tomlMeta["min_version"]; ok {

-			if hugo.CompareVersion(minVersion) > 0 {

-				return absThemeDir, true, fmt.Sprint(minVersion)

-			}

-		}

-

-	}

-

-	return

 }

--- a/commands/import_jekyll.go

+++ b/commands/import_jekyll.go

@@ -17,7 +17,6 @@

 	"bytes"

 	"errors"

 	"fmt"

-	"io"

 	"io/ioutil"

 	"os"

 	"path/filepath"

@@ -27,6 +26,8 @@

 	"time"

 	"unicode"

+	"github.com/gohugoio/hugo/common/hugio"

+

 	"github.com/gohugoio/hugo/parser/metadecoders"

 	"github.com/gohugoio/hugo/helpers"

@@ -113,7 +114,7 @@

 	jww.FEEDBACK.Println("Importing...")

 	fileCount := 0

-	callback := func(path string, fi os.FileInfo, err error) error {

+	callback := func(path string, fi hugofs.FileMetaInfo, err error) error {

 		if err != nil {

 			return err

 		}

@@ -302,70 +303,14 @@

 	return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs)

 }

-func copyFile(source string, dest string) error {

-	sf, err := os.Open(source)

-	if err != nil {

-		return err

-	}

-	defer sf.Close()

-	df, err := os.Create(dest)

-	if err != nil {

-		return err

-	}

-	defer df.Close()

-	_, err = io.Copy(df, sf)

-	if err == nil {

-		si, err := os.Stat(source)

-		if err != nil {

-			err = os.Chmod(dest, si.Mode())

+func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) {

+	fs := hugofs.Os

-			if err != nil {

-				return err

-			}

-		}

-

-	}

-	return nil

-}

-

-func copyDir(source string, dest string) error {

-	fi, err := os.Stat(source)

+	fi, err := fs.Stat(jekyllRoot)

 	if err != nil {

 		return err

 	}

 	if !fi.IsDir() {

-		return errors.New(source + " is not a directory")

-	}

-	err = os.MkdirAll(dest, fi.Mode())

-	if err != nil {

-		return err

-	}

-	entries, _ := ioutil.ReadDir(source)

-	for _, entry := range entries {

-		sfp := filepath.Join(source, entry.Name())

-		dfp := filepath.Join(dest, entry.Name())

-		if entry.IsDir() {

-			err = copyDir(sfp, dfp)

-			if err != nil {

-				jww.ERROR.Println(err)

-			}

-		} else {

-			err = copyFile(sfp, dfp)

-			if err != nil {

-				jww.ERROR.Println(err)

-			}

-		}

-

-	}

-	return nil

-}

-

-func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) {

-	fi, err := os.Stat(jekyllRoot)

-	if err != nil {

-		return err

-	}

-	if !fi.IsDir() {

 		return errors.New(jekyllRoot + " is not a directory")

 	}

 	err = os.MkdirAll(dest, fi.Mode())

@@ -383,7 +328,7 @@

 		if entry.IsDir() {

 			if entry.Name()[0] != '_' && entry.Name()[0] != '.' {

 				if _, ok := jekyllPostDirs[entry.Name()]; !ok {

-					err = copyDir(sfp, dfp)

+					err = hugio.CopyDir(fs, sfp, dfp, nil)

 					if err != nil {

 						jww.ERROR.Println(err)

 					}

@@ -402,7 +347,7 @@

 			}

 			if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' {

-				err = copyFile(sfp, dfp)

+				err = hugio.CopyFile(fs, sfp, dfp)

 				if err != nil {

 					jww.ERROR.Println(err)

 				}

--- a/commands/list_test.go

+++ b/commands/list_test.go

@@ -62,6 +62,7 @@

 	}, header)

 	record, err := r.Read()

+	assert.NoError(err)

 	assert.Equal([]string{

 		filepath.Join("content", "p1.md"), "", "P1",

 		"0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z",

--- /dev/null

+++ b/commands/mod.go

@@ -1,0 +1,189 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package commands

+

+import (

+	"os"

+	"strings"

+

+	"github.com/gohugoio/hugo/modules"

+	"github.com/spf13/cobra"

+)

+

+var _ cmder = (*modCmd)(nil)

+

+type modCmd struct {

+	*baseBuilderCmd

+}

+

+func (b *commandsBuilder) newModCmd() *modCmd {

+	c := &modCmd{}

+

+	const commonUsage = `

+Note that Hugo will always start out by resolving the components defined in the site

+configuration, provided by a _vendor directory (if no --ignoreVendor flag provided),

+Go Modules, or a folder inside the themes directory, in that order.

+

+See https://gohugo.io/hugo-modules/ for more information.

+

+`

+

+	cmd := &cobra.Command{

+		Use:   "mod",

+		Short: "Various Hugo Modules helpers.",

+		Long: `Various helpers to help manage the modules in your project's dependency graph.

+

+Most operations here requires a Go version installed on your system (>= Go 1.12) and the relevant VCS client (typically Git).

+This is not needed if you only operate on modules inside /themes or if you have vendored them via "hugo mod vendor".

+

+` + commonUsage,

+

+		RunE: nil,

+	}

+

+	cmd.AddCommand(

+		&cobra.Command{

+			Use:                "get",

+			DisableFlagParsing: true,

+			Short:              "Resolves dependencies in your current Hugo Project.",

+			Long: `

+Resolves dependencies in your current Hugo Project.

+

+Some examples:

+

+Install the latest version possible for a given module:

+

+    hugo mod get github.com/gohugoio/testshortcodes

+    

+Install a specific version:

+

+    hugo mod get github.com/gohugoio/[email protected]

+

+Install the latest versions of all module dependencies:

+

+    hugo mod get -u

+

+Run "go help get" for more information. All flags available for "go get" is also relevant here.

+` + commonUsage,

+			RunE: func(cmd *cobra.Command, args []string) error {

+				return c.withModsClient(false, func(c *modules.Client) error {

+					// We currently just pass on the flags we get to Go and

+					// need to do the flag handling manually.

+					if len(args) == 1 && strings.Contains(args[0], "-h") {

+						return cmd.Help()

+					}

+					return c.Get(args...)

+				})

+			},

+		},

+		&cobra.Command{

+			Use:   "graph",

+			Short: "Print a module dependency graph.",

+			Long: `Print a module dependency graph with information about module status (disabled, vendored).

+Note that for vendored modules, that is the version listed and not the one from go.mod.

+`,

+			RunE: func(cmd *cobra.Command, args []string) error {

+				return c.withModsClient(true, func(c *modules.Client) error {

+					return c.Graph(os.Stdout)

+				})

+			},

+		},

+		&cobra.Command{

+			Use:   "init",

+			Short: "Initialize this project as a Hugo Module.",

+			Long: `Initialize this project as a Hugo Module.

+It will try to guess the module path, but you may help by passing it as an argument, e.g:

+

+    hugo mod init github.com/gohugoio/testshortcodes

+

+Note that Hugo Modules supports multi-module projects, so you can initialize a Hugo Module

+inside a subfolder on GitHub, as one example.

+`,

+			RunE: func(cmd *cobra.Command, args []string) error {

+				var path string

+				if len(args) >= 1 {

+					path = args[0]

+				}

+				return c.withModsClient(false, func(c *modules.Client) error {

+					return c.Init(path)

+				})

+			},

+		},

+		&cobra.Command{

+			Use:   "vendor",

+			Short: "Vendor all module dependencies into the _vendor directory.",

+			Long: `Vendor all module dependencies into the _vendor directory.

+

+If a module is vendored, that is where Hugo will look for it's dependencies.

+`,

+			RunE: func(cmd *cobra.Command, args []string) error {

+				return c.withModsClient(true, func(c *modules.Client) error {

+					return c.Vendor()

+				})

+			},

+		},

+		&cobra.Command{

+			Use:   "tidy",

+			Short: "Remove unused entries in go.mod and go.sum.",

+			RunE: func(cmd *cobra.Command, args []string) error {

+				return c.withModsClient(true, func(c *modules.Client) error {

+					return c.Tidy()

+				})

+			},

+		},

+		&cobra.Command{

+			Use:   "clean",

+			Short: "Delete the entire Hugo Module cache.",

+			Long: `Delete the entire Hugo Module cache.

+

+Note that after you run this command, all of your dependencies will be re-downloaded next time you run "hugo".

+

+Also note that if you configure a positive maxAge for the "modules" file cache, it will also be cleaned as part of "hugo --gc".

+ 

+`,

+			RunE: func(cmd *cobra.Command, args []string) error {

+				com, err := c.initConfig(true)

+				if err != nil {

+					return err

+				}

+

+				_, err = com.hugo.FileCaches.ModulesCache().Prune(true)

+				return err

+

+			},

+		},

+	)

+

+	c.baseBuilderCmd = b.newBuilderCmd(cmd)

+

+	return c

+

+}

+

+func (c *modCmd) withModsClient(failOnMissingConfig bool, f func(*modules.Client) error) error {

+	com, err := c.initConfig(failOnMissingConfig)

+	if err != nil {

+		return err

+	}

+

+	return f(com.hugo.ModulesClient)

+}

+

+func (c *modCmd) initConfig(failOnNoConfig bool) (*commandeer, error) {

+	com, err := initializeConfig(failOnNoConfig, false, &c.hugoBuilderCommon, c, nil)

+	if err != nil {

+		return nil, err

+	}

+	return com, nil

+}

--- a/commands/new.go

+++ b/commands/new.go

@@ -53,8 +53,6 @@

 	cc := &newCmd{baseBuilderCmd: b.newBuilderCmd(cmd)}

 	cmd.Flags().StringVarP(&cc.contentType, "kind", "k", "", "content type to create")

-	cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")

-	cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})

 	cmd.Flags().StringVar(&cc.contentEditor, "editor", "", "edit new content with this editor, if provided")

 	cmd.AddCommand(newNewSiteCmd().getCommand())

@@ -120,8 +118,8 @@

 	createpath := filepath.FromSlash(path)

 	if h != nil {

-		for _, s := range h.Sites {

-			createpath = strings.TrimPrefix(createpath, s.PathSpec.ContentDir)

+		for _, dir := range h.BaseFs.Content.Dirs {

+			createpath = strings.TrimPrefix(createpath, dir.Meta().Filename())

 		}

 	}

--- a/commands/server.go

+++ b/commands/server.go

@@ -256,15 +256,11 @@

 			return err

 		}

-		baseWatchDir := c.Cfg.GetString("workingDir")

-		relWatchDirs := make([]string, len(watchDirs))

-		for i, dir := range watchDirs {

-			relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir)

-		}

+		watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs)

-		rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(relWatchDirs)), ",")

-

-		jww.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)

+		for _, group := range watchGroups {

+			jww.FEEDBACK.Printf("Watching for changes in %s\n", group)

+		}

 		watcher, err := c.newWatcher(watchDirs...)

 		if err != nil {

@@ -277,6 +273,15 @@

 	return c.serve(sc)

+}

+

+func getRootWatchDirsStr(baseDir string, watchDirs []string) string {

+	relWatchDirs := make([]string, len(watchDirs))

+	for i, dir := range watchDirs {

+		relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseDir)

+	}

+

+	return strings.Join(helpers.UniqueStringsSorted(helpers.ExtractRootPaths(relWatchDirs)), ",")

 }

 type fileServer struct {

--- a/commands/static_syncer.go

+++ b/commands/static_syncer.go

@@ -53,6 +53,7 @@

 		syncer := fsync.NewSyncer()

 		syncer.NoTimes = c.Cfg.GetBool("noTimes")

 		syncer.NoChmod = c.Cfg.GetBool("noChmod")

+		syncer.ChmodFilter = chmodFilter

 		syncer.SrcFs = sourceFs.Fs

 		syncer.DestFs = c.Fs.Destination

--- a/common/herrors/errors.go

+++ b/common/herrors/errors.go

@@ -19,6 +19,7 @@

 	"fmt"

 	"io"

 	"os"

+	"runtime/debug"

 	_errors "github.com/pkg/errors"

 )

@@ -44,6 +45,16 @@

 			fmt.Fprintf(w, "%+s:%d\n", f, f)

 		}

 	}

+}

+

+// Recover is a helper function that can be used to capture panics.

+// Put this at the top of a method/function that crashes in a template:

+//     defer herrors.Recover()

+func Recover() {

+	if r := recover(); r != nil {

+		fmt.Println("stacktrace from panic: \n" + string(debug.Stack()))

+	}

+

 }

 // ErrFeatureNotAvailable denotes that a feature is unavailable.

--- /dev/null

+++ b/common/hugio/copy.go

@@ -1,0 +1,90 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugio

+

+import (

+	"io"

+	"io/ioutil"

+	"os"

+	"path/filepath"

+

+	"github.com/pkg/errors"

+

+	"github.com/spf13/afero"

+)

+

+// CopyFile copies a file.

+func CopyFile(fs afero.Fs, from, to string) error {

+	sf, err := os.Open(from)

+	if err != nil {

+		return err

+	}

+	defer sf.Close()

+	df, err := os.Create(to)

+	if err != nil {

+		return err

+	}

+	defer df.Close()

+	_, err = io.Copy(df, sf)

+	if err == nil {

+		si, err := os.Stat(from)

+		if err != nil {

+			err = os.Chmod(to, si.Mode())

+

+			if err != nil {

+				return err

+			}

+		}

+

+	}

+	return nil

+}

+

+// CopyDir copies a directory.

+func CopyDir(fs afero.Fs, from, to string, shouldCopy func(filename string) bool) error {

+	fi, err := os.Stat(from)

+	if err != nil {

+		return err

+	}

+

+	if !fi.IsDir() {

+		return errors.Errorf("%q is not a directory", from)

+	}

+

+	err = fs.MkdirAll(to, 0777) // before umask

+	if err != nil {

+		return err

+	}

+

+	entries, _ := ioutil.ReadDir(from)

+	for _, entry := range entries {

+		fromFilename := filepath.Join(from, entry.Name())

+		toFilename := filepath.Join(to, entry.Name())

+		if entry.IsDir() {

+			if shouldCopy != nil && !shouldCopy(fromFilename) {

+				continue

+			}

+			if err := CopyDir(fs, fromFilename, toFilename, shouldCopy); err != nil {

+				return err

+			}

+		} else {

+			if err := CopyFile(fs, fromFilename, toFilename); err != nil {

+				return err

+			}

+		}

+

+	}

+

+	return nil

+}

--- a/common/hugo/vars_extended.go

+++ b/common/hugo/vars_extended.go

@@ -15,4 +15,4 @@

 package hugo

-var isExtended = true

+var IsExtended = true

--- a/common/hugo/vars_regular.go

+++ b/common/hugo/vars_regular.go

@@ -15,4 +15,4 @@

 package hugo

-var isExtended = false

+var IsExtended = false

--- a/common/hugo/version.go

+++ b/common/hugo/version.go

@@ -15,6 +15,7 @@

 import (

 	"fmt"

+	"strconv"

 	"runtime"

 	"strings"

@@ -133,7 +134,7 @@

 	if commitHash != "" {

 		version += "-" + strings.ToUpper(commitHash)

 	}

-	if isExtended {

+	if IsExtended {

 		version += "/extended"

 	}

@@ -234,4 +235,17 @@

 		return -1

 	}

 	return 1

+}

+

+func GoMinorVersion() int {

+	return goMinorVersion(runtime.Version())

+}

+

+func goMinorVersion(version string) int {

+	if strings.HasPrefix(version, "devel") {

+		return 9999 // magic

+	}

+	i, _ := strconv.Atoi(strings.Split(version, ".")[1])

+	return i

+

 }

--- a/common/hugo/version_test.go

+++ b/common/hugo/version_test.go

@@ -77,3 +77,9 @@

 	require.Equal(t, "0.25-DEV", MustParseVersion("0.25-DEV").String())

 }

+

+func TestGoMinorVersion(t *testing.T) {

+	assert := require.New(t)

+	assert.Equal(12, goMinorVersion("go1.12.5"))

+	assert.True(GoMinorVersion() >= 11)

+}

--- a/common/loggers/loggers.go

+++ b/common/loggers/loggers.go

@@ -40,6 +40,7 @@

 type Logger struct {

 	*jww.Notepad

 	ErrorCounter *jww.Counter

+	WarnCounter  *jww.Counter

 	// This is only set in server mode.

 	errors *bytes.Buffer

@@ -143,9 +144,10 @@

 func newLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger {

 	errorCounter := &jww.Counter{}

+	warnCounter := &jww.Counter{}

 	outHandle, logHandle = getLogWriters(outHandle, logHandle)

-	listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError)}

+	listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError), jww.LogCounter(warnCounter, jww.LevelWarn)}

 	var errorBuff *bytes.Buffer

 	if saveErrors {

 		errorBuff = new(bytes.Buffer)

@@ -164,6 +166,7 @@

 	return &Logger{

 		Notepad:      jww.NewNotepad(stdoutThreshold, logThreshold, outHandle, logHandle, "", log.Ldate|log.Ltime, listeners...),

 		ErrorCounter: errorCounter,

+		WarnCounter:  warnCounter,

 		errors:       errorBuff,

 	}

 }

--- /dev/null

+++ b/common/maps/params.go

@@ -1,0 +1,96 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package maps

+

+import (

+	"strings"

+

+	"github.com/pkg/errors"

+

+	"github.com/spf13/cast"

+)

+

+// GetNestedParam gets the first match of the keyStr in the candidates given.

+// It will first try the exact match and then try to find it as a nested map value,

+// using the given separator, e.g. "mymap.name".

+// It assumes that all the maps given have lower cased keys.

+func GetNestedParam(keyStr, separator string, candidates ...map[string]interface{}) (interface{}, error) {

+	keyStr = strings.ToLower(keyStr)

+

+	lookupFn := func(key string) interface{} {

+		for _, m := range candidates {

+			if v, ok := m[key]; ok {

+				return v

+			}

+		}

+

+		return nil

+	}

+

+	v, _, _, err := GetNestedParamFn(keyStr, separator, lookupFn)

+	return v, err

+}

+

+func GetNestedParamFn(keyStr, separator string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) {

+	result, _ := traverseDirectParams(keyStr, lookupFn)

+	if result != nil {

+		return result, keyStr, nil, nil

+	}

+

+	keySegments := strings.Split(keyStr, separator)

+	if len(keySegments) == 1 {

+		return nil, keyStr, nil, nil

+	}

+

+	return traverseNestedParams(keySegments, lookupFn)

+}

+

+func traverseDirectParams(keyStr string, lookupFn func(key string) interface{}) (interface{}, error) {

+	return lookupFn(keyStr), nil

+}

+

+func traverseNestedParams(keySegments []string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) {

+	firstKey, rest := keySegments[0], keySegments[1:]

+	result := lookupFn(firstKey)

+	if result == nil || len(rest) == 0 {

+		return result, firstKey, nil, nil

+	}

+

+	switch m := result.(type) {

+	case map[string]interface{}:

+		v, key, owner := traverseParams(rest, m)

+		return v, key, owner, nil

+	default:

+		return nil, "", nil, errors.Errorf("unsupported Params type: %T", result)

+	}

+}

+

+func traverseParams(keys []string, m map[string]interface{}) (interface{}, string, map[string]interface{}) {

+	// Shift first element off.

+	firstKey, rest := keys[0], keys[1:]

+	result := m[firstKey]

+

+	// No point in continuing here.

+	if result == nil {

+		return result, "", nil

+	}

+

+	if len(rest) == 0 {

+		// That was the last key.

+		return result, firstKey, m

+	}

+

+	// That was not the last key.

+	return traverseParams(rest, cast.ToStringMap(result))

+}

--- /dev/null

+++ b/common/maps/params_test.go

@@ -1,0 +1,45 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package maps

+

+import (

+	"testing"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestGetNestedParam(t *testing.T) {

+

+	m := map[string]interface{}{

+		"first":           1,

+		"with_underscore": 2,

+		"nested": map[string]interface{}{

+			"color": "blue",

+		},

+	}

+

+	assert := require.New(t)

+

+	must := func(keyStr, separator string, candidates ...map[string]interface{}) interface{} {

+		v, err := GetNestedParam(keyStr, separator, candidates...)

+		assert.NoError(err)

+		return v

+	}

+

+	assert.Equal(1, must("first", "_", m))

+	assert.Equal(1, must("First", "_", m))

+	assert.Equal(2, must("with_underscore", "_", m))

+	assert.Equal("blue", must("nested_color", "_", m))

+

+}

--- a/config/configLoader.go

+++ b/config/configLoader.go

@@ -120,8 +120,6 @@

 func newViper() *viper.Viper {

 	v := viper.New()

-	v.AutomaticEnv()

-	v.SetEnvPrefix("hugo")

 	return v

 }

--- a/config/configProvider.go

+++ b/config/configProvider.go

@@ -35,10 +35,14 @@

 // we do not attempt to split it into fields.

 func GetStringSlicePreserveString(cfg Provider, key string) []string {

 	sd := cfg.Get(key)

-	if sds, ok := sd.(string); ok {

+	return toStringSlicePreserveString(sd)

+}

+

+func toStringSlicePreserveString(v interface{}) []string {

+	if sds, ok := v.(string); ok {

 		return []string{sds}

 	}

-	return cast.ToStringSlice(sd)

+	return cast.ToStringSlice(v)

 }

 // SetBaseTestDefaults provides some common config defaults used in tests.

--- a/config/env.go

+++ b/config/env.go

@@ -17,6 +17,7 @@

 	"os"

 	"runtime"

 	"strconv"

+	"strings"

 )

 // GetNumWorkerMultiplier returns the base value used to calculate the number

@@ -30,4 +31,27 @@

 		}

 	}

 	return runtime.NumCPU()

+}

+

+// SetEnvVars sets vars on the form key=value in the oldVars slice.

+func SetEnvVars(oldVars *[]string, keyValues ...string) {

+	for i := 0; i < len(keyValues); i += 2 {

+		setEnvVar(oldVars, keyValues[i], keyValues[i+1])

+	}

+}

+

+func SplitEnvVar(v string) (string, string) {

+	parts := strings.Split(v, "=")

+	return parts[0], parts[1]

+}

+

+func setEnvVar(vars *[]string, key, value string) {

+	for i := range *vars {

+		if strings.HasPrefix((*vars)[i], key+"=") {

+			(*vars)[i] = key + "=" + value

+			return

+		}

+	}

+	// New var.

+	*vars = append(*vars, key+"="+value)

 }

--- /dev/null

+++ b/config/env_test.go

@@ -1,0 +1,32 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package config

+

+import (

+	"testing"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestSetEnvVars(t *testing.T) {

+	t.Parallel()

+	assert := require.New(t)

+	vars := []string{"FOO=bar", "HUGO=cool", "BAR=foo"}

+	SetEnvVars(&vars, "HUGO", "rocking!", "NEW", "bar")

+	assert.Equal([]string{"FOO=bar", "HUGO=rocking!", "BAR=foo", "NEW=bar"}, vars)

+

+	key, val := SplitEnvVar("HUGO=rocks")

+	assert.Equal("HUGO", key)

+	assert.Equal("rocks", val)

+}

--- a/create/content.go

+++ b/create/content.go

@@ -25,6 +25,8 @@

 	"path/filepath"

 	"strings"

+	"github.com/gohugoio/hugo/hugofs/files"

+

 	"github.com/gohugoio/hugo/hugofs"

 	"github.com/gohugoio/hugo/helpers"

@@ -50,7 +52,10 @@

 	if isDir {

-		langFs := hugofs.NewLanguageFs(s.Language().Lang, sites.LanguageSet(), archetypeFs)

+		langFs, err := hugofs.NewLanguageFs(sites.LanguageSet(), archetypeFs)

+		if err != nil {

+			return err

+		}

 		cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename)

 		if err != nil {

@@ -64,7 +69,7 @@

 		}

 		name := filepath.Base(targetPath)

-		return newContentFromDir(archetypeFilename, sites, archetypeFs, sourceFs, cm, name, contentPath)

+		return newContentFromDir(archetypeFilename, sites, sourceFs, cm, name, contentPath)

 	}

 	// Building the sites can be expensive, so only do it if really needed.

@@ -111,9 +116,9 @@

 	return nil

 }

-func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib.Site {

+func targetSite(sites *hugolib.HugoSites, fi hugofs.FileMetaInfo) *hugolib.Site {

 	for _, s := range sites.Sites {

-		if fi.Lang() == s.Language().Lang {

+		if fi.Meta().Lang() == s.Language().Lang {

 			return s

 		}

 	}

@@ -123,13 +128,14 @@

 func newContentFromDir(

 	archetypeDir string,

 	sites *hugolib.HugoSites,

-	sourceFs, targetFs afero.Fs,

+	targetFs afero.Fs,

 	cm archetypeMap, name, targetPath string) error {

 	for _, f := range cm.otherFiles {

-		filename := f.Filename()

+		meta := f.Meta()

+		filename := meta.Path()

 		// Just copy the file to destination.

-		in, err := sourceFs.Open(filename)

+		in, err := meta.Open()

 		if err != nil {

 			return errors.Wrap(err, "failed to open non-content file")

 		}

@@ -156,7 +162,7 @@

 	}

 	for _, f := range cm.contentFiles {

-		filename := f.Filename()

+		filename := f.Meta().Path()

 		s := targetSite(sites, f)

 		targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir))

@@ -177,9 +183,9 @@

 type archetypeMap struct {

 	// These needs to be parsed and executed as Go templates.

-	contentFiles []*hugofs.LanguageFileInfo

+	contentFiles []hugofs.FileMetaInfo

 	// These are just copied to destination.

-	otherFiles []*hugofs.LanguageFileInfo

+	otherFiles []hugofs.FileMetaInfo

 	// If the templates needs a fully built site. This can potentially be

 	// expensive, so only do when needed.

 	siteUsed bool

@@ -192,7 +198,7 @@

 	var m archetypeMap

-	walkFn := func(filename string, fi os.FileInfo, err error) error {

+	walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {

 		if err != nil {

 			return err

@@ -202,12 +208,12 @@

 			return nil

 		}

-		fil := fi.(*hugofs.LanguageFileInfo)

+		fil := fi.(hugofs.FileMetaInfo)

-		if hugolib.IsContentFile(filename) {

+		if files.IsContentFile(path) {

 			m.contentFiles = append(m.contentFiles, fil)

 			if !m.siteUsed {

-				m.siteUsed, err = usesSiteVar(fs, filename)

+				m.siteUsed, err = usesSiteVar(fs, path)

 				if err != nil {

 					return err

 				}

@@ -220,7 +226,15 @@

 		return nil

 	}

-	if err := helpers.SymbolicWalk(fs, archetypeDir, walkFn); err != nil {

+	walkCfg := hugofs.WalkwayConfig{

+		WalkFn: walkFn,

+		Fs:     fs,

+		Root:   archetypeDir,

+	}

+

+	w := hugofs.NewWalkway(walkCfg)

+

+	if err := w.Walk(); err != nil {

 		return m, errors.Wrapf(err, "failed to walk archetype dir %q", archetypeDir)

 	}

@@ -254,17 +268,29 @@

 		}

 	}

-	for _, ss := range sites.Sites {

-		contentDir := ss.PathSpec.ContentDir

+	var dirLang string

+

+	for _, dir := range sites.BaseFs.Content.Dirs {

+		meta := dir.Meta()

+		contentDir := meta.Filename()

+

 		if !strings.HasSuffix(contentDir, helpers.FilePathSeparator) {

 			contentDir += helpers.FilePathSeparator

 		}

+

 		if strings.HasPrefix(targetPath, contentDir) {

-			siteContentDir = ss.PathSpec.ContentDir

-			if s == nil {

+			siteContentDir = contentDir

+			dirLang = meta.Lang()

+			break

+		}

+	}

+

+	if s == nil && dirLang != "" {

+		for _, ss := range sites.Sites {

+			if ss.Lang() == dirLang {

 				s = ss

+				break

 			}

-			break

 		}

 	}

@@ -280,12 +306,22 @@

 		}

 	}

+	if siteContentDir == "" {

+

+	}

+

 	if siteContentDir != "" {

 		pp := filepath.Join(siteContentDir, strings.TrimPrefix(targetPath, siteContentDir))

 		return s.PathSpec.AbsPathify(pp), s

-

 	} else {

-		return s.PathSpec.AbsPathify(filepath.Join(first.PathSpec.ContentDir, targetPath)), s

+		var contentDir string

+		for _, dir := range sites.BaseFs.Content.Dirs {

+			contentDir = dir.Meta().Filename()

+			if dir.Meta().Lang() == s.Lang() {

+				break

+			}

+		}

+		return s.PathSpec.AbsPathify(filepath.Join(contentDir, targetPath)), s

 	}

 }

--- a/create/content_template_handler.go

+++ b/create/content_template_handler.go

@@ -90,7 +90,10 @@

 		err               error

 	)

-	f := s.SourceSpec.NewFileInfo("", targetPath, false, nil)

+	f, err := s.SourceSpec.NewFileInfoFrom(targetPath, targetPath)

+	if err != nil {

+		return nil, err

+	}

 	if name == "" {

 		name = f.TranslationBaseName()

--- a/create/content_test.go

+++ b/create/content_test.go

@@ -35,7 +35,6 @@

 )

 func TestNewContent(t *testing.T) {

-	assert := require.New(t)

 	cases := []struct {

 		kind     string

@@ -50,12 +49,12 @@

 		{"product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter

 		{"lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}},

 		{"lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}},

-		{"lang", "post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}},

+		{"lang", "content/post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}},

 		{"lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}},

 		{"lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}},

 		{"lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},

 		{"lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},

-		{"lang", "post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}},

+		{"lang", "content/post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}},

 		{"shortcodes", "shortcodes/go.md", []string{

 			`title = "GO"`,

 			"{{< myshortcode >}}",

@@ -64,37 +63,43 @@

 	}

 	for i, c := range cases {

-		cfg, fs := newTestCfg(assert)

-		assert.NoError(initFs(fs))

-		h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})

-		assert.NoError(err)

+		c := c

+		t.Run(fmt.Sprintf("%s-%d", c.kind, i), func(t *testing.T) {

+			t.Parallel()

+			assert := require.New(t)

+			mm := afero.NewMemMapFs()

+			assert.NoError(initFs(mm))

+			cfg, fs := newTestCfg(assert, mm)

+			h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})

+			assert.NoError(err)

-		assert.NoError(create.NewContent(h, c.kind, c.path))

+			assert.NoError(create.NewContent(h, c.kind, c.path))

-		fname := filepath.FromSlash(c.path)

-		if !strings.HasPrefix(fname, "content") {

-			fname = filepath.Join("content", fname)

-		}

-		content := readFileFromFs(t, fs.Source, fname)

-		for _, v := range c.expected {

-			found := strings.Contains(content, v)

-			if !found {

-				t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)

+			fname := filepath.FromSlash(c.path)

+			if !strings.HasPrefix(fname, "content") {

+				fname = filepath.Join("content", fname)

 			}

-		}

+			content := readFileFromFs(t, fs.Source, fname)

+			for _, v := range c.expected {

+				found := strings.Contains(content, v)

+				if !found {

+					t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)

+				}

+			}

+		})

+

 	}

 }

 func TestNewContentFromDir(t *testing.T) {

+	mm := afero.NewMemMapFs()

 	assert := require.New(t)

-	cfg, fs := newTestCfg(assert)

-	assert.NoError(initFs(fs))

 	archetypeDir := filepath.Join("archetypes", "my-bundle")

-	assert.NoError(fs.Source.Mkdir(archetypeDir, 0755))

+	assert.NoError(mm.MkdirAll(archetypeDir, 0755))

 	archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle")

-	assert.NoError(fs.Source.Mkdir(archetypeThemeDir, 0755))

+	assert.NoError(mm.MkdirAll(archetypeThemeDir, 0755))

 	contentFile := `

 File: %s

@@ -103,16 +108,19 @@

 i18n: {{ T "hugo" }}

 `

-	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))

-	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755))

+	assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))

+	assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755))

-	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755))

-	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))

-	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755))

+	assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755))

+	assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))

+	assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755))

-	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))

-	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))

+	assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))

+	assert.NoError(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))

+	assert.NoError(initFs(mm))

+	cfg, fs := newTestCfg(assert, mm)

+

 	h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})

 	assert.NoError(err)

 	assert.Equal(2, len(h.Sites))

@@ -135,7 +143,7 @@

 }

-func initFs(fs *hugofs.Fs) error {

+func initFs(fs afero.Fs) error {

 	perm := os.FileMode(0755)

 	var err error

@@ -146,8 +154,8 @@

 		filepath.Join("themes", "sample", "archetypes"),

 	}

 	for _, dir := range dirs {

-		err = fs.Source.Mkdir(dir, perm)

-		if err != nil {

+		err = fs.Mkdir(dir, perm)

+		if err != nil && !os.IsExist(err) {

 			return err

 		}

 	}

@@ -198,7 +206,7 @@

 `,

 		},

 	} {

-		f, err := fs.Source.Create(v.path)

+		f, err := fs.Create(v.path)

 		if err != nil {

 			return err

 		}

@@ -221,6 +229,7 @@

 // TODO(bep) extract common testing package with this and some others

 func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {

+	t.Helper()

 	filename = filepath.FromSlash(filename)

 	b, err := afero.ReadFile(fs, filename)

 	if err != nil {

@@ -238,12 +247,11 @@

 	return string(b)

 }

-func newTestCfg(assert *require.Assertions) (*viper.Viper, *hugofs.Fs) {

+func newTestCfg(assert *require.Assertions, mm afero.Fs) (*viper.Viper, *hugofs.Fs) {

 	cfg := `

 theme = "mytheme"

-	

 [languages]

 [languages.en]

 weight = 1

@@ -254,8 +262,13 @@

 contentDir = "content_nn"

 `

+	if mm == nil {

+		mm = afero.NewMemMapFs()

+	}

-	mm := afero.NewMemMapFs()

+	mm.MkdirAll(filepath.FromSlash("content_nn"), 0777)

+

+	mm.MkdirAll(filepath.FromSlash("themes/mytheme"), 0777)

 	assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo]

 other = "Hugo Rocks!"`), 0755))

--- a/deps/deps.go

+++ b/deps/deps.go

@@ -159,11 +159,11 @@

 func (d *Deps) LoadResources() error {

 	// Note that the translations need to be loaded before the templates.

 	if err := d.translationProvider.Update(d); err != nil {

-		return err

+		return errors.Wrap(err, "loading translations")

 	}

 	if err := d.templateProvider.Update(d); err != nil {

-		return err

+		return errors.Wrap(err, "loading templates")

 	}

 	return nil

@@ -210,7 +210,7 @@

 	ps, err := helpers.NewPathSpec(fs, cfg.Language)

 	if err != nil {

-		return nil, err

+		return nil, errors.Wrap(err, "create PathSpec")

 	}

 	fileCaches, err := filecache.NewCaches(ps)

--- a/go.mod

+++ b/go.mod

@@ -12,18 +12,18 @@

 	github.com/bep/debounce v1.2.0

 	github.com/bep/gitmap v1.1.0

 	github.com/bep/go-tocss v0.6.0

-	github.com/cpuguy83/go-md2man v1.0.8 // indirect

 	github.com/disintegration/imaging v1.6.0

 	github.com/dustin/go-humanize v1.0.0

 	github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385

 	github.com/fortytw2/leaktest v1.3.0

 	github.com/fsnotify/fsnotify v1.4.7

+	github.com/go-errors/errors v1.0.1

 	github.com/gobwas/glob v0.2.3

+	github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95

 	github.com/google/go-cmp v0.3.0

 	github.com/gorilla/websocket v1.4.0

 	github.com/hashicorp/go-immutable-radix v1.0.0

 	github.com/hashicorp/go-uuid v1.0.1 // indirect

-	github.com/inconshreveable/mousetrap v1.0.0 // indirect

 	github.com/jdkato/prose v1.1.0

 	github.com/kyokomi/emoji v1.5.1

 	github.com/magefile/mage v1.4.0

@@ -30,23 +30,25 @@

 	github.com/magiconair/properties v1.8.1 // indirect

 	github.com/markbates/inflect v1.0.0

 	github.com/mattn/go-isatty v0.0.8

-	github.com/mattn/go-runewidth v0.0.3 // indirect

 	github.com/miekg/mmark v1.3.6

 	github.com/mitchellh/hashstructure v1.0.0

 	github.com/mitchellh/mapstructure v1.1.2

 	github.com/muesli/smartcrop v0.0.0-20180228075044-f6ebaa786a12

+	github.com/ncw/rclone v1.48.0

 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect

 	github.com/nicksnyder/go-i18n v1.10.0

 	github.com/niklasfasching/go-org v0.1.1

 	github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84

+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7

 	github.com/pelletier/go-toml v1.4.0 // indirect

 	github.com/pkg/errors v0.8.1

+	github.com/rogpeppe/go-internal v1.3.0

 	github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691

 	github.com/sanity-io/litter v1.1.0

 	github.com/spf13/afero v1.2.2

 	github.com/spf13/cast v1.3.0

-	github.com/spf13/cobra v0.0.3

-	github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05

+	github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267

+	github.com/spf13/fsync v0.9.0

 	github.com/spf13/jwalterweatherman v1.1.0

 	github.com/spf13/pflag v1.0.3

 	github.com/spf13/viper v1.4.0

@@ -67,3 +69,5 @@

 )

 replace github.com/markbates/inflect => github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6

+

+go 1.13

--- a/go.sum

+++ b/go.sum

@@ -1,3 +1,4 @@

+bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=

 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=

 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=

 cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=

@@ -19,6 +20,7 @@

 github.com/Azure/azure-service-bus-go v0.4.1/go.mod h1:d9ho9e/06euiTwGpKxmlbpPhFUsfCsq6a4tZ68r51qI=

 github.com/Azure/azure-storage-blob-go v0.6.0 h1:SEATKb3LIHcaSIX+E6/K4kJpwfuozFEsmt5rS56N6CE=

 github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=

+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=

 github.com/Azure/go-autorest v11.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=

 github.com/Azure/go-autorest v11.1.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=

 github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=

@@ -35,6 +37,11 @@

 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=

 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=

 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=

+github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=

+github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=

+github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=

+github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=

+github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=

 github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=

 github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=

 github.com/alecthomas/chroma v0.6.4 h1:Gn37/7W4a1qkmKLzfUpDy2rt3jt4X8CWycb4Gm7L360=

@@ -48,10 +55,12 @@

 github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=

 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=

 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

+github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=

 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=

 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=

 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=

 github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=

+github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=

 github.com/aws/aws-sdk-go v1.19.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=

 github.com/aws/aws-sdk-go v1.19.40 h1:omRrS4bCM/IbzU6UEb8Ojg1PvlElZzYZkOh8vWWgFMc=

 github.com/aws/aws-sdk-go v1.19.40/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=

@@ -63,6 +72,7 @@

 github.com/bep/gitmap v1.1.0/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY=

 github.com/bep/go-tocss v0.6.0 h1:lJf+nIjsQDpifUr+NgHi9QMBnrr9cFvMvEBT+uV9Q9E=

 github.com/bep/go-tocss v0.6.0/go.mod h1:d9d3crzlTl+PUZLFzBUjfFCpp68K+ku10mzTlnqU/+A=

+github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=

 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=

@@ -69,13 +79,17 @@

 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=

 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=

 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=

+github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=

 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=

 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=

 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=

 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=

 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=

 github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M=

 github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=

+github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=

+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=

 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=

 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=

 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

@@ -86,8 +100,10 @@

 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=

 github.com/disintegration/imaging v1.6.0 h1:nVPXRUUQ36Z7MNf0O77UzgnOb1mkMMor7lmJMJXc/mA=

 github.com/disintegration/imaging v1.6.0/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ=

+github.com/djherbis/times v1.2.0/go.mod h1:CGMZlo255K5r4Yw0b9RRfFQpM2y7uOmxg4jm9HsaVf8=

 github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=

 github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=

+github.com/dropbox/dropbox-sdk-go-unofficial v5.4.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY=

 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=

 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=

 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=

@@ -95,6 +111,7 @@

 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=

 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=

 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=

+github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=

 github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw+Q=

 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=

 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=

@@ -102,6 +119,7 @@

 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=

 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=

 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=

+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=

 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=

 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=

 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=

@@ -110,9 +128,14 @@

 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=

 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=

 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=

+github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9/go.mod h1:GpOj6zuVBG3Inr9qjEnuVTgBlk2lZ1S9DcoFiXWyKss=

+github.com/goftp/server v0.0.0-20190304020633-eabccc535b5a/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE=

+github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=

 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

+github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 h1:sgew0XCnZwnzpWxTt3V8LLiCO7OQi3C6dycaE67wfkU=

+github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95/go.mod h1:bOlVlCa1/RajcHpXkrUXPSHB/Re1UnlXxD1Qp8SKOd8=

 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=

 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

@@ -130,6 +153,7 @@

 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=

 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=

 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=

+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=

 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=

 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=

 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=

@@ -143,6 +167,8 @@

 github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=

 github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc=

 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=

+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=

+github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=

 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=

 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=

 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=

@@ -168,6 +194,7 @@

 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=

 github.com/jdkato/prose v1.1.0 h1:LpvmDGwbKGTgdCH3a8VJL56sr7p/wOFPw/R4lM4PfFg=

 github.com/jdkato/prose v1.1.0/go.mod h1:jkF0lkxaX5PFSlk9l4Gh9Y+T57TqUZziWT7uZbW5ADg=

+github.com/jlaffaye/ftp v0.0.0-20190519203911-8f5b34ce006f/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=

 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=

 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

@@ -174,10 +201,15 @@

 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=

 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=

 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=

+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=

 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=

+github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=

 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

+github.com/koofr/go-httpclient v0.0.0-20180104120329-03786175608a/go.mod h1:3xszwh+rNrYk1r9SStc4iJ326gne1OaBcrdB1ACsbzI=

+github.com/koofr/go-koofrclient v0.0.0-20190131164641-7f327592caff/go.mod h1:MRAz4Gsxd+OzrZ0owwrUHc0zLESL+1Y5syqK/sJxK2A=

+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=

 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=

 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=

 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

@@ -187,6 +219,7 @@

 github.com/kyokomi/emoji v1.5.1 h1:qp9dub1mW7C4MlvoRENH6EAENb9skEFOvIEbp1Waj38=

 github.com/kyokomi/emoji v1.5.1/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=

 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=

+github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=

 github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk=

 github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=

 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=

@@ -199,13 +232,18 @@

 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=

 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=

 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=

+github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=

+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=

 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=

 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=

 github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=

 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=

+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=

+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=

 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=

 github.com/miekg/mmark v1.3.6 h1:t47x5vThdwgLJzofNsbsAl7gmIiJ7kbDQN5BxwBmwvY=

 github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw=

+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=

 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=

 github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=

 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=

@@ -214,6 +252,10 @@

 github.com/muesli/smartcrop v0.0.0-20180228075044-f6ebaa786a12 h1:l0X/8IDy2UoK+oXcQFMRSIOcyuYb5iEPytPGplnM41Y=

 github.com/muesli/smartcrop v0.0.0-20180228075044-f6ebaa786a12/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI=

 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

+github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=

+github.com/ncw/rclone v1.48.0 h1:Rc7A4YEQDeMPgnc1IzA6PsJ4YikyP+zS68rgGMYKJ7o=

+github.com/ncw/rclone v1.48.0/go.mod h1:CXDUKN1OQ3Y2ya1Ma6jTZ7m9ZarGzF3ZTHsdPLHWWzY=

+github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=

 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=

 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=

 github.com/nicksnyder/go-i18n v1.10.0 h1:5AzlPKvXBH4qBzmZ09Ua9Gipyruv6uApMcrNZdo96+Q=

@@ -220,23 +262,33 @@

 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=

 github.com/niklasfasching/go-org v0.1.1 h1:yoU+dQLuc8P1Y0nxY3DV9SoBQTXSy9CogEvhOXSq9/Q=

 github.com/niklasfasching/go-org v0.1.1/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU=

+github.com/nsf/termbox-go v0.0.0-20190325093121-288510b9734e/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=

 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=

+github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd/go.mod h1:4soZNh0zW0LtYGdQ416i0jO0EIqMGcbtaspRS4BDvRQ=

 github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 h1:fiKJgB4JDUd43CApkmCeTSQlWjtTtABrU2qsgbuP0BI=

 github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=

 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=

+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=

+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=

+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=

 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=

 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=

+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=

+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=

 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=

 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

 github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=

 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=

+github.com/pengsrc/go-shared v0.2.0/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=

 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=

 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=

 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

+github.com/pkg/sftp v1.10.1-0.20190523025818-e98a7bef6829/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=

 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

@@ -253,7 +305,13 @@

 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=

 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=

 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=

+github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46 h1:w2CpS5muK+jyydnmlkqpAhzKmHmMBzBkfYUDjQNS1Dk=

+github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46/go.mod h1:U2bmx0hDj8EyDdcxmD5t3XHDnBFnyNNc22n1R4008eM=

 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=

+github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk=

+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=

+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=

 github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691 h1:auJkuUc4uOuZNoH9jGLvqVaDLiuCOh/LY+Qw5NBFo4I=

 github.com/russross/blackfriday v1.5.3-0.20190124082335-a477dd164691/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=

 github.com/sanity-io/litter v1.1.0 h1:BllcKWa3VbZmOZbDCoszYLk7zCsKHz5Beossi8SUcTc=

@@ -260,7 +318,14 @@

 github.com/sanity-io/litter v1.1.0/go.mod h1:CJ0VCw2q4qKU7LaQr3n7UOSHzgEMgcGco7N/SkZQPjw=

 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=

 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=

+github.com/sevlyar/go-daemon v0.1.4/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE=

+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=

+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=

 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

+github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=

+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=

+github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=

+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=

 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=

 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=

 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=

@@ -271,8 +336,12 @@

 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=

 github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=

 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=

+github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267 h1:I9j1PLS64+NgCtkgbomGInboj1NFH1KF1tkVKlt3yF4=

+github.com/spf13/cobra v0.0.4-0.20190321000552-67fc4837d267/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=

 github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05 h1:pQHm7pxjSgC54M1rtLSLmju25phy6RgYf3p4O6XanYE=

 github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05/go.mod h1:jdsEoy1w+v0NpuwXZEaRAH6ADTDmzfRnE2eVwshwFrM=

+github.com/spf13/fsync v0.9.0 h1:f9CEt3DOB2mnHxZaftmEOFWjABEvKM/xpf3cUwJrGOY=

+github.com/spf13/fsync v0.9.0/go.mod h1:fNtJEfG3HiltN3y4cPOz6MLjos9+2pIEqLIgszqhp/0=

 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=

 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=

 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=

@@ -279,6 +348,7 @@

 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=

 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=

 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=

+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=

 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=

 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=

 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=

@@ -288,6 +358,7 @@

 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=

 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

+github.com/t3rm1n4l/go-mega v0.0.0-20190430100803-72151b53bb44/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=

 github.com/tdewolff/minify/v2 v2.3.7 h1:nhk7MKYRdTDwTxqEQZKLDkLe04tDHht8mBI+VJrsYvk=

 github.com/tdewolff/minify/v2 v2.3.7/go.mod h1:DD1stRlSx6JsHfl1+E/HVMQeXiec9rD1UQ0epklIZLc=

 github.com/tdewolff/parse/v2 v2.3.5 h1:/uS8JfhwVJsNkEh769GM5ENv6L9LOh2Z9uW3tCdlhs0=

@@ -300,8 +371,10 @@

 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=

 github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=

 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=

+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=

 github.com/wellington/go-libsass v0.9.3-0.20181113175235-c63644206701 h1:9vG9vvVNVupO4Y7uwFkRgIMNe9rdaJMCINDe8vhAhLo=

 github.com/wellington/go-libsass v0.9.3-0.20181113175235-c63644206701/go.mod h1:mxgxgam0N0E+NAUMHLcu20Ccfc3mVpDkyrLDayqfiTs=

+github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=

 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=

 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=

 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=

@@ -308,6 +381,7 @@

 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

 github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA=

 github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0=

+github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=

 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

 go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=

 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=

@@ -325,8 +399,13 @@

 gocloud.dev v0.15.0/go.mod h1:ShXCyJaGrJu9y/7a6+DSCyBb9MFGZ1P5wwPa0Wu6w34=

 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

 golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

+golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

 golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=

+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI=

 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=

@@ -355,6 +434,8 @@

 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=

 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=

+golang.org/x/net v0.0.0-20190606173856-1492cefac77f h1:IWHgpgFqnL5AhBUBZSgBdjl2vkQUEzcY+JNKWfcgAU0=

+golang.org/x/net v0.0.0-20190606173856-1492cefac77f/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=

 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=

 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

@@ -379,11 +460,16 @@

 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

 golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

+golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

+golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1 h1:R4dVlxdmKenVdMRS/tTspEpSTRWINYrHD8ySIU9yCIU=

+golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=

 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=

@@ -394,6 +480,7 @@

 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=

 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

@@ -402,7 +489,9 @@

 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

 golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

+golang.org/x/tools v0.0.0-20190606174628-0139d5756a7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=

 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=

 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

@@ -413,6 +502,7 @@

 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=

 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

 google.golang.org/appengine v1.6.0 h1:Tfd7cKwKbFRsI8RMAD3oqqw7JPFRrvFlOsfbgVkjOOw=

 google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

@@ -427,6 +517,7 @@

 google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=

 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=

 google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=

+google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=

 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=

 google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0=

 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

@@ -435,6 +526,7 @@

 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=

 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

--- a/helpers/content.go

+++ b/helpers/content.go

@@ -511,12 +511,6 @@

 	return n

 }

-// Old implementation only kept for benchmark comparison.

-// TODO(bep) remove

-func totalWordsOld(s string) int {

-	return len(strings.Fields(s))

-}

-

 // TruncateWordsByRune truncates words by runes.

 func (c *ContentSpec) TruncateWordsByRune(in []string) (string, bool) {

 	words := make([]string, len(in))

--- a/helpers/content_test.go

+++ b/helpers/content_test.go

@@ -506,13 +506,3 @@

 		}

 	}

 }

-

-func BenchmarkTotalWordsOld(b *testing.B) {

-	b.ResetTimer()

-	for i := 0; i < b.N; i++ {

-		wordCount := totalWordsOld(totalWordsBenchmarkString)

-		if wordCount != 400 {

-			b.Fatal("Wordcount error")

-		}

-	}

-}

--- a/helpers/docshelper.go

+++ b/helpers/docshelper.go

@@ -36,8 +36,7 @@

 				}

 			}

-			sort.Strings(aliases)

-			aliases = UniqueStrings(aliases)

+			aliases = UniqueStringsSorted(aliases)

 			lexerEntry := struct {

 				Name    string

--- a/helpers/general.go

+++ b/helpers/general.go

@@ -22,15 +22,16 @@

 	"net"

 	"os"

 	"path/filepath"

+	"sort"

 	"strings"

 	"sync"

 	"unicode"

 	"unicode/utf8"

-	"github.com/gohugoio/hugo/common/hugo"

-

 	"github.com/gohugoio/hugo/hugofs"

+	"github.com/gohugoio/hugo/common/hugo"

+

 	"github.com/spf13/afero"

 	"github.com/jdkato/prose/transform"

@@ -106,7 +107,7 @@

 // UniqueStrings returns a new slice with any duplicates removed.

 func UniqueStrings(s []string) []string {

-	var unique []string

+	unique := make([]string, 0, len(s))

 	set := map[string]interface{}{}

 	for _, val := range s {

 		if _, ok := set[val]; !ok {

@@ -117,6 +118,40 @@

 	return unique

 }

+// UniqueStringsReuse returns a slice with any duplicates removed.

+// It will modify the input slice.

+func UniqueStringsReuse(s []string) []string {

+	set := map[string]interface{}{}

+	result := s[:0]

+	for _, val := range s {

+		if _, ok := set[val]; !ok {

+			result = append(result, val)

+			set[val] = val

+		}

+	}

+	return result

+}

+

+// UniqueStringsReuse returns a sorted slice with any duplicates removed.

+// It will modify the input slice.

+func UniqueStringsSorted(s []string) []string {

+	if len(s) == 0 {

+		return nil

+	}

+	ss := sort.StringSlice(s)

+	ss.Sort()

+	i := 0

+	for j := 1; j < len(s); j++ {

+		if !ss.Less(i, j) {

+			continue

+		}

+		i++

+		s[i] = s[j]

+	}

+

+	return s[:i+1]

+}

+

 // ReaderToBytes takes an io.Reader argument, reads from it

 // and returns bytes.

 func ReaderToBytes(lines io.Reader) []byte {

@@ -459,17 +494,15 @@

 	if fs == nil {

 		return

 	}

+

 	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {

-		if info != nil && !info.IsDir() {

-			s := path

-			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {

-				s = s + "\tLANG: " + lang.Lang()

-			}

-			if fp, ok := info.(hugofs.FilePather); ok {

-				s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir()

-			}

-			fmt.Fprintln(w, "    ", s)

+		var filename string

+		var meta interface{}

+		if fim, ok := info.(hugofs.FileMetaInfo); ok {

+			filename = fim.Meta().Filename()

+			meta = fim.Meta()

 		}

+		fmt.Fprintf(w, "    %q %q\t\t%v\n", path, filename, meta)

 		return nil

 	})

 }

--- a/helpers/general_test.go

+++ b/helpers/general_test.go

@@ -234,6 +234,24 @@

 	}

 }

+func TestUniqueStringsReuse(t *testing.T) {

+	in := []string{"a", "b", "a", "b", "c", "", "a", "", "d"}

+	output := UniqueStringsReuse(in)

+	expected := []string{"a", "b", "c", "", "d"}

+	if !reflect.DeepEqual(output, expected) {

+		t.Errorf("Expected %#v, got %#v\n", expected, output)

+	}

+}

+

+func TestUniqueStringsSorted(t *testing.T) {

+	assert := require.New(t)

+	in := []string{"a", "a", "b", "c", "b", "", "a", "", "d"}

+	output := UniqueStringsSorted(in)

+	expected := []string{"", "a", "b", "c", "d"}

+	assert.Equal(expected, output)

+	assert.Nil(UniqueStringsSorted(nil))

+}

+

 func TestFindAvailablePort(t *testing.T) {

 	addr, err := FindAvailablePort()

 	assert.Nil(t, err)

@@ -326,5 +344,57 @@

 			}

 		})

 	}

+

+}

+

+func BenchmarkUniqueStrings(b *testing.B) {

+	input := []string{"a", "b", "d", "e", "d", "h", "a", "i"}

+

+	b.Run("Safe", func(b *testing.B) {

+		for i := 0; i < b.N; i++ {

+			result := UniqueStrings(input)

+			if len(result) != 6 {

+				b.Fatal(fmt.Sprintf("invalid count: %d", len(result)))

+			}

+		}

+	})

+

+	b.Run("Reuse slice", func(b *testing.B) {

+		b.StopTimer()

+		inputs := make([][]string, b.N)

+		for i := 0; i < b.N; i++ {

+			inputc := make([]string, len(input))

+			copy(inputc, input)

+			inputs[i] = inputc

+		}

+		b.StartTimer()

+		for i := 0; i < b.N; i++ {

+			inputc := inputs[i]

+

+			result := UniqueStringsReuse(inputc)

+			if len(result) != 6 {

+				b.Fatal(fmt.Sprintf("invalid count: %d", len(result)))

+			}

+		}

+	})

+

+	b.Run("Reuse slice sorted", func(b *testing.B) {

+		b.StopTimer()

+		inputs := make([][]string, b.N)

+		for i := 0; i < b.N; i++ {

+			inputc := make([]string, len(input))

+			copy(inputc, input)

+			inputs[i] = inputc

+		}

+		b.StartTimer()

+		for i := 0; i < b.N; i++ {

+			inputc := inputs[i]

+

+			result := UniqueStringsSorted(inputc)

+			if len(result) != 6 {

+				b.Fatal(fmt.Sprintf("invalid count: %d", len(result)))

+			}

+		}

+	})

 }

--- a/helpers/path.go

+++ b/helpers/path.go

@@ -26,6 +26,8 @@

 	"github.com/gohugoio/hugo/config"

+	"github.com/gohugoio/hugo/hugofs"

+

 	"github.com/gohugoio/hugo/common/hugio"

 	_errors "github.com/pkg/errors"

 	"github.com/spf13/afero"

@@ -172,32 +174,6 @@

 	return f + "." + newExt

 }

-// GetFirstThemeDir gets the root directory of the first theme, if there is one.

-// If there is no theme, returns the empty string.

-func (p *PathSpec) GetFirstThemeDir() string {

-	if p.ThemeSet() {

-		return p.AbsPathify(filepath.Join(p.ThemesDir, p.Themes()[0]))

-	}

-	return ""

-}

-

-// GetThemesDir gets the absolute root theme dir path.

-func (p *PathSpec) GetThemesDir() string {

-	if p.ThemeSet() {

-		return p.AbsPathify(p.ThemesDir)

-	}

-	return ""

-}

-

-// GetRelativeThemeDir gets the relative root directory of the current theme, if there is one.

-// If there is no theme, returns the empty string.

-func (p *PathSpec) GetRelativeThemeDir() string {

-	if p.ThemeSet() {

-		return strings.TrimPrefix(filepath.Join(p.ThemesDir, p.Themes()[0]), FilePathSeparator)

-	}

-	return ""

-}

-

 func makePathRelative(inPath string, possibleDirectories ...string) (string, error) {

 	for _, currentPath := range possibleDirectories {

@@ -379,6 +355,107 @@

 	return b.Join(b.Dir(in), name, "index"+ext)

 }

+type NamedSlice struct {

+	Name  string

+	Slice []string

+}

+

+func (n NamedSlice) String() string {

+	if len(n.Slice) == 0 {

+		return n.Name

+	}

+	return fmt.Sprintf("%s%s{%s}", n.Name, FilePathSeparator, strings.Join(n.Slice, ","))

+}

+

+func ExtractAndGroupRootPaths(paths []string) []NamedSlice {

+	if len(paths) == 0 {

+		return nil

+	}

+

+	pathsCopy := make([]string, len(paths))

+	hadSlashPrefix := strings.HasPrefix(paths[0], FilePathSeparator)

+

+	for i, p := range paths {

+		pathsCopy[i] = strings.Trim(filepath.ToSlash(p), "/")

+	}

+

+	sort.Strings(pathsCopy)

+

+	pathsParts := make([][]string, len(pathsCopy))

+

+	for i, p := range pathsCopy {

+		pathsParts[i] = strings.Split(p, "/")

+	}

+

+	var groups [][]string

+

+	for i, p1 := range pathsParts {

+		c1 := -1

+

+		for j, p2 := range pathsParts {

+			if i == j {

+				continue

+			}

+

+			c2 := -1

+

+			for i, v := range p1 {

+				if i >= len(p2) {

+					break

+				}

+				if v != p2[i] {

+					break

+				}

+

+				c2 = i

+			}

+

+			if c1 == -1 || (c2 != -1 && c2 < c1) {

+				c1 = c2

+			}

+		}

+

+		if c1 != -1 {

+			groups = append(groups, p1[:c1+1])

+		} else {

+			groups = append(groups, p1)

+		}

+	}

+

+	groupsStr := make([]string, len(groups))

+	for i, g := range groups {

+		groupsStr[i] = strings.Join(g, "/")

+	}

+

+	groupsStr = UniqueStringsSorted(groupsStr)

+

+	var result []NamedSlice

+

+	for _, g := range groupsStr {

+		name := filepath.FromSlash(g)

+		if hadSlashPrefix {

+			name = FilePathSeparator + name

+		}

+		ns := NamedSlice{Name: name}

+		for _, p := range pathsCopy {

+			if !strings.HasPrefix(p, g) {

+				continue

+			}

+

+			p = strings.TrimPrefix(p, g)

+			if p != "" {

+				ns.Slice = append(ns.Slice, p)

+			}

+		}

+

+		ns.Slice = UniqueStrings(ExtractRootPaths(ns.Slice))

+

+		result = append(result, ns)

+	}

+

+	return result

+}

+

 // ExtractRootPaths extracts the root paths from the supplied list of paths.

 // The resulting root path will not contain any file separators, but there

 // may be duplicates.

@@ -425,98 +502,21 @@

 	return path, nil

 }

-// SymbolicWalk is like filepath.Walk, but it supports the root being a

-// symbolic link. It will still not follow symbolic links deeper down in

-// the file structure.

-func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {

-

-	// Sanity check

-	if root != "" && len(root) < 4 {

-		return errors.New("path is too short")

+// SymbolicWalk is like filepath.Walk, but it follows symbolic links.

+func SymbolicWalk(fs afero.Fs, root string, walker hugofs.WalkFunc) error {

+	if _, isOs := fs.(*afero.OsFs); isOs {

+		// Mainly to track symlinks.

+		fs = hugofs.NewBaseFileDecorator(fs)

 	}

-	// Handle the root first

-	fileInfo, realPath, err := getRealFileInfo(fs, root)

+	w := hugofs.NewWalkway(hugofs.WalkwayConfig{

+		Fs:     fs,

+		Root:   root,

+		WalkFn: walker,

+	})

-	if err != nil {

-		return walker(root, nil, err)

-	}

+	return w.Walk()

-	if !fileInfo.IsDir() {

-		return fmt.Errorf("cannot walk regular file %s", root)

-	}

-

-	if err := walker(realPath, fileInfo, err); err != nil && err != filepath.SkipDir {

-		return err

-	}

-

-	// Some of Hugo's filesystems represents an ordered root folder, i.e. project first, then theme folders.

-	// Make sure that order is preserved. afero.Walk will sort the directories down in the file tree,

-	// but we don't care about that.

-	rootContent, err := readDir(fs, root, false)

-

-	if err != nil {

-		return walker(root, nil, err)

-	}

-

-	for _, fi := range rootContent {

-		if err := afero.Walk(fs, filepath.Join(root, fi.Name()), walker); err != nil {

-			return err

-		}

-	}

-

-	return nil

-

-}

-

-func readDir(fs afero.Fs, dirname string, doSort bool) ([]os.FileInfo, error) {

-	f, err := fs.Open(dirname)

-	if err != nil {

-		return nil, err

-	}

-	list, err := f.Readdir(-1)

-	f.Close()

-	if err != nil {

-		return nil, err

-	}

-	if doSort {

-		sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() })

-	}

-	return list, nil

-}

-

-func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {

-	fileInfo, err := LstatIfPossible(fs, path)

-	realPath := path

-

-	if err != nil {

-		return nil, "", err

-	}

-

-	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {

-		link, err := filepath.EvalSymlinks(path)

-		if err != nil {

-			return nil, "", _errors.Wrapf(err, "Cannot read symbolic link %q", path)

-		}

-		fileInfo, err = LstatIfPossible(fs, link)

-		if err != nil {

-			return nil, "", _errors.Wrapf(err, "Cannot stat %q", link)

-		}

-		realPath = link

-	}

-	return fileInfo, realPath, nil

-}

-

-// GetRealPath returns the real file path for the given path, whether it is a

-// symlink or not.

-func GetRealPath(fs afero.Fs, path string) (string, error) {

-	_, realPath, err := getRealFileInfo(fs, path)

-

-	if err != nil {

-		return "", err

-	}

-

-	return realPath, nil

 }

 // LstatIfPossible can be used to call Lstat if possible, else Stat.

--- a/helpers/path_test.go

+++ b/helpers/path_test.go

@@ -29,8 +29,6 @@

 	"github.com/stretchr/testify/require"

-	"github.com/stretchr/testify/assert"

-

 	"github.com/gohugoio/hugo/hugofs"

 	"github.com/spf13/afero"

 	"github.com/spf13/viper"

@@ -73,18 +71,9 @@

 }

 func TestMakePathSanitized(t *testing.T) {

-	v := viper.New()

-	v.Set("contentDir", "content")

-	v.Set("dataDir", "data")

-	v.Set("i18nDir", "i18n")

-	v.Set("layoutDir", "layouts")

-	v.Set("assetDir", "assets")

-	v.Set("resourceDir", "resources")

-	v.Set("publishDir", "public")

-	v.Set("archetypeDir", "archetypes")

+	v := newTestCfg()

-	l := langs.NewDefaultLanguage(v)

-	p, _ := NewPathSpec(hugofs.NewMem(v), l)

+	p, _ := NewPathSpec(hugofs.NewMem(v), v)

 	tests := []struct {

 		input    string

@@ -166,33 +155,6 @@

 	}

 }

-func TestGetRealPath(t *testing.T) {

-	if runtime.GOOS == "windows" && os.Getenv("CI") == "" {

-		t.Skip("Skip TestGetRealPath as os.Symlink needs administrator rights on Windows")

-	}

-

-	d1, _ := ioutil.TempDir("", "d1")

-	defer os.Remove(d1)

-	fs := afero.NewOsFs()

-

-	rp1, err := GetRealPath(fs, d1)

-	require.NoError(t, err)

-	assert.Equal(t, d1, rp1)

-

-	sym := filepath.Join(os.TempDir(), "d1sym")

-	err = os.Symlink(d1, sym)

-	require.NoError(t, err)

-	defer os.Remove(sym)

-

-	rp2, err := GetRealPath(fs, sym)

-	require.NoError(t, err)

-

-	// On OS X, the temp folder is itself a symbolic link (to /private...)

-	// This has to do for now.

-	assert.True(t, strings.HasSuffix(rp2, d1))

-

-}

-

 func TestMakePathRelative(t *testing.T) {

 	type test struct {

 		inPath, path1, path2, output string

@@ -656,6 +618,29 @@

 }

 func TestPrettifyPath(t *testing.T) {

+

+}

+

+func TestExtractAndGroupRootPaths(t *testing.T) {

+	in := []string{

+		filepath.FromSlash("/a/b/c/d"),

+		filepath.FromSlash("/a/b/c/e"),

+		filepath.FromSlash("/a/b/e/f"),

+		filepath.FromSlash("/a/b"),

+		filepath.FromSlash("/a/b/c/b/g"),

+		filepath.FromSlash("/c/d/e"),

+	}

+

+	inCopy := make([]string, len(in))

+	copy(inCopy, in)

+

+	result := ExtractAndGroupRootPaths(in)

+

+	assert := require.New(t)

+	assert.Equal(filepath.FromSlash("[/a/b/{c,e} /c/d/e]"), fmt.Sprint(result))

+

+	// Make sure the original is preserved

+	assert.Equal(inCopy, in)

 }

--- a/helpers/pathspec_test.go

+++ b/helpers/pathspec_test.go

@@ -14,6 +14,7 @@

 package helpers

 import (

+	"path/filepath"

 	"testing"

 	"github.com/gohugoio/hugo/hugofs"

@@ -36,9 +37,13 @@

 	v.Set("workingDir", "thework")

 	v.Set("staticDir", "thestatic")

 	v.Set("theme", "thetheme")

+	langs.LoadLanguageSettings(v, nil)

-	p, err := NewPathSpec(hugofs.NewMem(v), l)

+	fs := hugofs.NewMem(v)

+	fs.Source.MkdirAll(filepath.FromSlash("thework/thethemes/thetheme"), 0777)

+	p, err := NewPathSpec(fs, l)

+

 	require.NoError(t, err)

 	require.True(t, p.CanonifyURLs)

 	require.True(t, p.DisablePathToLower)

@@ -50,5 +55,5 @@

 	require.Equal(t, "http://base.com", p.BaseURL.String())

 	require.Equal(t, "thethemes", p.ThemesDir)

 	require.Equal(t, "thework", p.WorkingDir)

-	require.Equal(t, []string{"thetheme"}, p.Themes())

+

 }

--- a/helpers/testhelpers_test.go

+++ b/helpers/testhelpers_test.go

@@ -5,6 +5,7 @@

 	"github.com/gohugoio/hugo/hugofs"

 	"github.com/gohugoio/hugo/langs"

+	"github.com/gohugoio/hugo/modules"

 )

 func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec {

@@ -42,6 +43,14 @@

 	v.Set("resourceDir", "resources")

 	v.Set("publishDir", "public")

 	v.Set("archetypeDir", "archetypes")

+	langs.LoadLanguageSettings(v, nil)

+	langs.LoadLanguageSettings(v, nil)

+	mod, err := modules.CreateProjectModule(v)

+	if err != nil {

+		panic(err)

+	}

+	v.Set("allModules", modules.Modules{mod})

+

 	return v

 }

--- /dev/null

+++ b/htesting/test_helpers.go

@@ -1,0 +1,39 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package htesting

+

+import (

+	"runtime"

+	"strings"

+

+	"github.com/spf13/afero"

+)

+

+// CreateTempDir creates a temp dir in the given filesystem and

+// returns the dirnam and a func that removes it when done.

+func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) {

+	tempDir, err := afero.TempDir(fs, "", prefix)

+	if err != nil {

+		return "", nil, err

+	}

+

+	_, isOsFs := fs.(*afero.OsFs)

+

+	if isOsFs && runtime.GOOS == "darwin" && !strings.HasPrefix(tempDir, "/private") {

+		// To get the entry folder in line with the rest. This its a little bit

+		// mysterious, but so be it.

+		tempDir = "/private" + tempDir

+	}

+	return tempDir, func() { fs.RemoveAll(tempDir) }, nil

+}

--- a/htesting/test_structs.go

+++ /dev/null

@@ -1,100 +1,0 @@

-// Copyright 2019 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package htesting

-

-import (

-	"html/template"

-	"time"

-

-	"github.com/gohugoio/hugo/common/hugo"

-	"github.com/gohugoio/hugo/langs"

-	"github.com/gohugoio/hugo/navigation"

-	"github.com/gohugoio/hugo/resources/page"

-	"github.com/spf13/viper"

-)

-

-type testSite struct {

-	h hugo.Info

-	l *langs.Language

-}

-

-func (t testSite) Hugo() hugo.Info {

-	return t.h

-}

-

-func (t testSite) ServerPort() int {

-	return 1313

-}

-

-func (testSite) LastChange() (t time.Time) {

-	return

-}

-

-func (t testSite) Title() string {

-	return "foo"

-}

-

-func (t testSite) Sites() page.Sites {

-	return nil

-}

-

-func (t testSite) IsServer() bool {

-	return false

-}

-

-func (t testSite) Language() *langs.Language {

-	return t.l

-}

-

-func (t testSite) Pages() page.Pages {

-	return nil

-}

-

-func (t testSite) RegularPages() page.Pages {

-	return nil

-}

-

-func (t testSite) Menus() navigation.Menus {

-	return nil

-}

-

-func (t testSite) Taxonomies() interface{} {

-	return nil

-}

-

-func (t testSite) BaseURL() template.URL {

-	return ""

-}

-

-func (t testSite) Params() map[string]interface{} {

-	return nil

-}

-

-func (t testSite) Data() map[string]interface{} {

-	return nil

-}

-

-// NewTestHugoSite creates a new minimal test site.

-func NewTestHugoSite() page.Site {

-	return testSite{

-		h: hugo.NewInfo(hugo.EnvironmentProduction),

-		l: langs.NewLanguage("en", newTestConfig()),

-	}

-}

-

-func newTestConfig() *viper.Viper {

-	v := viper.New()

-	v.Set("contentDir", "content")

-	return v

-}

--- a/hugofs/basepath_real_filename_fs.go

+++ /dev/null

@@ -1,91 +1,0 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugofs

-

-import (

-	"os"

-

-	"github.com/spf13/afero"

-)

-

-// RealFilenameInfo is a thin wrapper around os.FileInfo adding the real filename.

-type RealFilenameInfo interface {

-	os.FileInfo

-

-	// This is the real filename to the file in the underlying filesystem.

-	RealFilename() string

-}

-

-type realFilenameInfo struct {

-	os.FileInfo

-	realFilename string

-}

-

-func (f *realFilenameInfo) RealFilename() string {

-	return f.realFilename

-}

-

-// NewBasePathRealFilenameFs returns a new BasePathRealFilenameFs instance

-// using base.

-func NewBasePathRealFilenameFs(base *afero.BasePathFs) *BasePathRealFilenameFs {

-	return &BasePathRealFilenameFs{BasePathFs: base}

-}

-

-// BasePathRealFilenameFs is a thin wrapper around afero.BasePathFs that

-// provides the real filename in Stat and LstatIfPossible.

-type BasePathRealFilenameFs struct {

-	*afero.BasePathFs

-}

-

-// Stat returns the os.FileInfo structure describing a given file.  If there is

-// an error, it will be of type *os.PathError.

-func (b *BasePathRealFilenameFs) Stat(name string) (os.FileInfo, error) {

-	fi, err := b.BasePathFs.Stat(name)

-	if err != nil {

-		return nil, err

-	}

-

-	if _, ok := fi.(RealFilenameInfo); ok {

-		return fi, nil

-	}

-

-	filename, err := b.RealPath(name)

-	if err != nil {

-		return nil, &os.PathError{Op: "stat", Path: name, Err: err}

-	}

-

-	return &realFilenameInfo{FileInfo: fi, realFilename: filename}, nil

-}

-

-// LstatIfPossible returns the os.FileInfo structure describing a given file.

-// It attempts to use Lstat if supported or defers to the os.  In addition to

-// the FileInfo, a boolean is returned telling whether Lstat was called.

-func (b *BasePathRealFilenameFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

-

-	fi, ok, err := b.BasePathFs.LstatIfPossible(name)

-	if err != nil {

-		return nil, false, err

-	}

-

-	if _, ok := fi.(RealFilenameInfo); ok {

-		return fi, ok, nil

-	}

-

-	filename, err := b.RealPath(name)

-	if err != nil {

-		return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err}

-	}

-

-	return &realFilenameInfo{FileInfo: fi, realFilename: filename}, ok, nil

-}

--- /dev/null

+++ b/hugofs/decorators.go

@@ -1,0 +1,205 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"os"

+	"path/filepath"

+	"strings"

+

+	"github.com/pkg/errors"

+

+	"github.com/spf13/afero"

+)

+

+func decorateDirs(fs afero.Fs, meta FileMeta) afero.Fs {

+	ffs := &baseFileDecoratorFs{Fs: fs}

+

+	decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) {

+		if !fi.IsDir() {

+			// Leave regular files as they are.

+			return fi, nil

+		}

+

+		return decorateFileInfo(fi, fs, nil, "", "", meta), nil

+	}

+

+	ffs.decorate = decorator

+

+	return ffs

+

+}

+

+func decoratePath(fs afero.Fs, createPath func(name string) string) afero.Fs {

+

+	ffs := &baseFileDecoratorFs{Fs: fs}

+

+	decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) {

+		path := createPath(name)

+

+		return decorateFileInfo(fi, fs, nil, "", path, nil), nil

+	}

+

+	ffs.decorate = decorator

+

+	return ffs

+

+}

+

+// DecorateBasePathFs adds Path info to files and directories in the

+// provided BasePathFs, using the base as base.

+func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs {

+	basePath, _ := base.RealPath("")

+	if !strings.HasSuffix(basePath, filepathSeparator) {

+		basePath += filepathSeparator

+	}

+

+	ffs := &baseFileDecoratorFs{Fs: base}

+

+	decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) {

+		path := strings.TrimPrefix(name, basePath)

+

+		return decorateFileInfo(fi, base, nil, "", path, nil), nil

+	}

+

+	ffs.decorate = decorator

+

+	return ffs

+}

+

+// NewBaseFileDecorator decorates the given Fs to provide the real filename

+// and an Opener func. If

+func NewBaseFileDecorator(fs afero.Fs) afero.Fs {

+

+	ffs := &baseFileDecoratorFs{Fs: fs}

+

+	decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {

+		// Store away the original in case it's a symlink.

+		meta := FileMeta{metaKeyName: fi.Name()}

+		isSymlink := isSymlink(fi)

+		if isSymlink {

+			meta[metaKeyOriginalFilename] = filename

+			link, err := filepath.EvalSymlinks(filename)

+			if err != nil {

+				return nil, err

+			}

+

+			fi, err = fs.Stat(link)

+			if err != nil {

+				return nil, err

+			}

+

+			filename = link

+			meta[metaKeyIsSymlink] = true

+

+		}

+

+		opener := func() (afero.File, error) {

+			return ffs.open(filename)

+

+		}

+

+		return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil

+	}

+

+	ffs.decorate = decorator

+	return ffs

+}

+

+type baseFileDecoratorFs struct {

+	afero.Fs

+	decorate func(fi os.FileInfo, filename string) (os.FileInfo, error)

+}

+

+func (fs *baseFileDecoratorFs) Stat(name string) (os.FileInfo, error) {

+	fi, err := fs.Fs.Stat(name)

+	if err != nil {

+		return nil, err

+	}

+

+	return fs.decorate(fi, name)

+

+}

+

+func (fs *baseFileDecoratorFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

+	var (

+		fi  os.FileInfo

+		err error

+		ok  bool

+	)

+

+	if lstater, isLstater := fs.Fs.(afero.Lstater); isLstater {

+		fi, ok, err = lstater.LstatIfPossible(name)

+	} else {

+		fi, err = fs.Fs.Stat(name)

+	}

+

+	if err != nil {

+		return nil, false, err

+	}

+

+	fi, err = fs.decorate(fi, name)

+

+	return fi, ok, err

+}

+

+func (fs *baseFileDecoratorFs) Open(name string) (afero.File, error) {

+	return fs.open(name)

+}

+

+func (fs *baseFileDecoratorFs) open(name string) (afero.File, error) {

+	f, err := fs.Fs.Open(name)

+	if err != nil {

+		return nil, err

+	}

+	return &baseFileDecoratorFile{File: f, fs: fs}, nil

+}

+

+type baseFileDecoratorFile struct {

+	afero.File

+	fs *baseFileDecoratorFs

+}

+

+func (l *baseFileDecoratorFile) Readdir(c int) (ofi []os.FileInfo, err error) {

+	dirnames, err := l.File.Readdirnames(c)

+	if err != nil {

+		return nil, err

+	}

+

+	fisp := make([]os.FileInfo, 0, len(dirnames))

+

+	for _, dirname := range dirnames {

+		filename := dirname

+

+		if l.Name() != "" && l.Name() != filepathSeparator {

+			filename = filepath.Join(l.Name(), dirname)

+		}

+

+		// We need to resolve any symlink info.

+		fi, _, err := lstatIfPossible(l.fs.Fs, filename)

+		if err != nil {

+			if os.IsNotExist(err) {

+				continue

+			}

+			return nil, err

+		}

+		fi, err = l.fs.decorate(fi, filename)

+		if err != nil {

+			return nil, errors.Wrap(err, "decorate")

+		}

+		fisp = append(fisp, fi)

+	}

+

+	return fisp, err

+}

--- /dev/null

+++ b/hugofs/fileinfo.go

@@ -1,0 +1,297 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+// Package hugofs provides the file systems used by Hugo.

+package hugofs

+

+import (

+	"os"

+	"path/filepath"

+	"runtime"

+	"strings"

+	"time"

+

+	"github.com/gohugoio/hugo/hugofs/files"

+	"golang.org/x/text/unicode/norm"

+

+	"github.com/pkg/errors"

+

+	"github.com/spf13/cast"

+

+	"github.com/gohugoio/hugo/common/hreflect"

+

+	"github.com/spf13/afero"

+)

+

+const (

+	metaKeyFilename                   = "filename"

+	metaKeyOriginalFilename           = "originalFilename"

+	metaKeyName                       = "name"

+	metaKeyPath                       = "path"

+	metaKeyPathWalk                   = "pathWalk"

+	metaKeyLang                       = "lang"

+	metaKeyWeight                     = "weight"

+	metaKeyOrdinal                    = "ordinal"

+	metaKeyFs                         = "fs"

+	metaKeyOpener                     = "opener"

+	metaKeyIsOrdered                  = "isOrdered"

+	metaKeyIsSymlink                  = "isSymlink"

+	metaKeySkipDir                    = "skipDir"

+	metaKeyClassifier                 = "classifier"

+	metaKeyTranslationBaseName        = "translationBaseName"

+	metaKeyTranslationBaseNameWithExt = "translationBaseNameWithExt"

+	metaKeyTranslations               = "translations"

+	metaKeyDecoraterPath              = "decoratorPath"

+)

+

+type FileMeta map[string]interface{}

+

+func (f FileMeta) GetInt(key string) int {

+	return cast.ToInt(f[key])

+}

+

+func (f FileMeta) GetString(key string) string {

+	return cast.ToString(f[key])

+}

+

+func (f FileMeta) GetBool(key string) bool {

+	return cast.ToBool(f[key])

+}

+

+func (f FileMeta) Filename() string {

+	return f.stringV(metaKeyFilename)

+}

+

+func (f FileMeta) OriginalFilename() string {

+	return f.stringV(metaKeyOriginalFilename)

+}

+

+func (f FileMeta) SkipDir() bool {

+	return f.GetBool(metaKeySkipDir)

+}

+func (f FileMeta) TranslationBaseName() string {

+	return f.stringV(metaKeyTranslationBaseName)

+}

+

+func (f FileMeta) TranslationBaseNameWithExt() string {

+	return f.stringV(metaKeyTranslationBaseNameWithExt)

+}

+

+func (f FileMeta) Translations() []string {

+	return cast.ToStringSlice(f[metaKeyTranslations])

+}

+

+func (f FileMeta) Name() string {

+	return f.stringV(metaKeyName)

+}

+

+func (f FileMeta) Classifier() string {

+	c := f.stringV(metaKeyClassifier)

+	if c != "" {

+		return c

+	}

+

+	return files.ContentClassFile // For sorting

+}

+

+func (f FileMeta) Lang() string {

+	return f.stringV(metaKeyLang)

+}

+

+func (f FileMeta) Path() string {

+	return f.stringV(metaKeyPath)

+}

+

+func (f FileMeta) Weight() int {

+	return f.GetInt(metaKeyWeight)

+}

+

+func (f FileMeta) Ordinal() int {

+	return f.GetInt(metaKeyOrdinal)

+}

+

+func (f FileMeta) IsOrdered() bool {

+	return f.GetBool(metaKeyIsOrdered)

+}

+

+// IsSymlink returns whether this comes from a symlinked file or directory.

+func (f FileMeta) IsSymlink() bool {

+	return f.GetBool(metaKeyIsSymlink)

+}

+

+func (f FileMeta) Watch() bool {

+	if v, found := f["watch"]; found {

+		return v.(bool)

+	}

+	return false

+}

+

+func (f FileMeta) Fs() afero.Fs {

+	if v, found := f[metaKeyFs]; found {

+		return v.(afero.Fs)

+	}

+	return nil

+}

+

+func (f FileMeta) GetOpener() func() (afero.File, error) {

+	o, found := f[metaKeyOpener]

+	if !found {

+		return nil

+	}

+	return o.(func() (afero.File, error))

+}

+

+func (f FileMeta) Open() (afero.File, error) {

+	v, found := f[metaKeyOpener]

+	if !found {

+		return nil, errors.New("file opener not found")

+	}

+	return v.(func() (afero.File, error))()

+}

+

+func (f FileMeta) stringV(key string) string {

+	if v, found := f[key]; found {

+		return v.(string)

+	}

+	return ""

+}

+

+func (f FileMeta) setIfNotZero(key string, val interface{}) {

+	if !hreflect.IsTruthful(val) {

+		return

+	}

+	f[key] = val

+}

+

+type FileMetaInfo interface {

+	os.FileInfo

+	Meta() FileMeta

+}

+

+type fileInfoMeta struct {

+	os.FileInfo

+	m FileMeta

+}

+

+func (fi *fileInfoMeta) Meta() FileMeta {

+	return fi.m

+}

+

+func NewFileMetaInfo(fi os.FileInfo, m FileMeta) FileMetaInfo {

+

+	if fim, ok := fi.(FileMetaInfo); ok {

+		mergeFileMeta(fim.Meta(), m)

+	}

+	return &fileInfoMeta{FileInfo: fi, m: m}

+}

+

+// Merge metadata, last entry wins.

+func mergeFileMeta(from, to FileMeta) {

+	if from == nil {

+		return

+	}

+	for k, v := range from {

+		if _, found := to[k]; !found {

+			to[k] = v

+		}

+	}

+}

+

+type dirNameOnlyFileInfo struct {

+	name string

+}

+

+func (fi *dirNameOnlyFileInfo) Name() string {

+	return fi.name

+}

+

+func (fi *dirNameOnlyFileInfo) Size() int64 {

+	panic("not implemented")

+}

+

+func (fi *dirNameOnlyFileInfo) Mode() os.FileMode {

+	return os.ModeDir

+}

+

+func (fi *dirNameOnlyFileInfo) ModTime() time.Time {

+	return time.Time{}

+}

+

+func (fi *dirNameOnlyFileInfo) IsDir() bool {

+	return true

+}

+

+func (fi *dirNameOnlyFileInfo) Sys() interface{} {

+	return nil

+}

+

+func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {

+	name = normalizeFilename(name)

+	_, base := filepath.Split(name)

+	return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{

+		metaKeyFilename:  name,

+		metaKeyIsOrdered: isOrdered,

+		metaKeyOpener:    fileOpener})

+}

+

+func decorateFileInfo(

+	fi os.FileInfo,

+	fs afero.Fs, opener func() (afero.File, error),

+	filename, filepath string, inMeta FileMeta) FileMetaInfo {

+

+	var meta FileMeta

+	var fim FileMetaInfo

+

+	filepath = strings.TrimPrefix(filepath, filepathSeparator)

+

+	var ok bool

+	if fim, ok = fi.(FileMetaInfo); ok {

+		meta = fim.Meta()

+	} else {

+		meta = make(FileMeta)

+		fim = NewFileMetaInfo(fi, meta)

+	}

+

+	meta.setIfNotZero(metaKeyOpener, opener)

+	meta.setIfNotZero(metaKeyFs, fs)

+	meta.setIfNotZero(metaKeyPath, normalizeFilename(filepath))

+	meta.setIfNotZero(metaKeyFilename, normalizeFilename(filename))

+

+	mergeFileMeta(inMeta, meta)

+

+	return fim

+

+}

+

+func isSymlink(fi os.FileInfo) bool {

+	return fi != nil && fi.Mode()&os.ModeSymlink == os.ModeSymlink

+}

+

+func fileInfosToFileMetaInfos(fis []os.FileInfo) []FileMetaInfo {

+	fims := make([]FileMetaInfo, len(fis))

+	for i, v := range fis {

+		fims[i] = v.(FileMetaInfo)

+	}

+	return fims

+}

+

+func normalizeFilename(filename string) string {

+	if filename == "" {

+		return ""

+	}

+	if runtime.GOOS == "darwin" {

+		// When a file system is HFS+, its filepath is in NFD form.

+		return norm.NFC.String(filename)

+	}

+	return filename

+}

--- /dev/null

+++ b/hugofs/files/classifier.go

@@ -1,0 +1,121 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package files

+

+import (

+	"os"

+	"path/filepath"

+	"sort"

+	"strings"

+)

+

+var (

+	// This should be the only list of valid extensions for content files.

+	contentFileExtensions = []string{

+		"html", "htm",

+		"mdown", "markdown", "md",

+		"asciidoc", "adoc", "ad",

+		"rest", "rst",

+		"mmark",

+		"org",

+		"pandoc", "pdc"}

+

+	contentFileExtensionsSet map[string]bool

+)

+

+func init() {

+	contentFileExtensionsSet = make(map[string]bool)

+	for _, ext := range contentFileExtensions {

+		contentFileExtensionsSet[ext] = true

+	}

+}

+

+func IsContentFile(filename string) bool {

+	return contentFileExtensionsSet[strings.TrimPrefix(filepath.Ext(filename), ".")]

+}

+

+func IsContentExt(ext string) bool {

+	return contentFileExtensionsSet[ext]

+}

+

+const (

+	ContentClassLeaf    = "leaf"

+	ContentClassBranch  = "branch"

+	ContentClassFile    = "zfile" // Sort below

+	ContentClassContent = "zcontent"

+)

+

+func ClassifyContentFile(filename string) string {

+	if !IsContentFile(filename) {

+		return ContentClassFile

+	}

+	if strings.HasPrefix(filename, "_index.") {

+		return ContentClassBranch

+	}

+

+	if strings.HasPrefix(filename, "index.") {

+		return ContentClassLeaf

+	}

+

+	return ContentClassContent

+}

+

+const (

+	ComponentFolderArchetypes = "archetypes"

+	ComponentFolderStatic     = "static"

+	ComponentFolderLayouts    = "layouts"

+	ComponentFolderContent    = "content"

+	ComponentFolderData       = "data"

+	ComponentFolderAssets     = "assets"

+	ComponentFolderI18n       = "i18n"

+

+	FolderResources = "resources"

+)

+

+var (

+	ComponentFolders = []string{

+		ComponentFolderArchetypes,

+		ComponentFolderStatic,

+		ComponentFolderLayouts,

+		ComponentFolderContent,

+		ComponentFolderData,

+		ComponentFolderAssets,

+		ComponentFolderI18n,

+	}

+

+	componentFoldersSet = make(map[string]bool)

+)

+

+func init() {

+	sort.Strings(ComponentFolders)

+	for _, f := range ComponentFolders {

+		componentFoldersSet[f] = true

+	}

+}

+

+// ResolveComponentFolder returns "content" from "content/blog/foo.md" etc.

+func ResolveComponentFolder(filename string) string {

+	filename = strings.TrimPrefix(filename, string(os.PathSeparator))

+	for _, cf := range ComponentFolders {

+		if strings.HasPrefix(filename, cf) {

+			return cf

+		}

+	}

+

+	return ""

+}

+

+func IsComponentFolder(name string) bool {

+	return componentFoldersSet[name]

+}

--- /dev/null

+++ b/hugofs/files/classifier_test.go

@@ -1,0 +1,49 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package files

+

+import (

+	"path/filepath"

+	"testing"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestIsContentFile(t *testing.T) {

+	assert := require.New(t)

+

+	assert.True(IsContentFile(filepath.FromSlash("my/file.md")))

+	assert.True(IsContentFile(filepath.FromSlash("my/file.ad")))

+	assert.False(IsContentFile(filepath.FromSlash("textfile.txt")))

+	assert.True(IsContentExt("md"))

+	assert.False(IsContentExt("json"))

+}

+

+func TestComponentFolders(t *testing.T) {

+	assert := require.New(t)

+

+	// It's important that these are absolutely right and not changed.

+	assert.Equal(len(ComponentFolders), len(componentFoldersSet))

+	assert.True(IsComponentFolder("archetypes"))

+	assert.True(IsComponentFolder("layouts"))

+	assert.True(IsComponentFolder("data"))

+	assert.True(IsComponentFolder("i18n"))

+	assert.True(IsComponentFolder("assets"))

+	assert.False(IsComponentFolder("resources"))

+	assert.True(IsComponentFolder("static"))

+	assert.True(IsComponentFolder("content"))

+	assert.False(IsComponentFolder("foo"))

+	assert.False(IsComponentFolder(""))

+

+}

--- /dev/null

+++ b/hugofs/filter_fs.go

@@ -1,0 +1,341 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"fmt"

+	"io"

+	"os"

+	"path/filepath"

+	"sort"

+	"strings"

+	"syscall"

+	"time"

+

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/spf13/afero"

+)

+

+var (

+	_ afero.Fs      = (*FilterFs)(nil)

+	_ afero.Lstater = (*FilterFs)(nil)

+	_ afero.File    = (*filterDir)(nil)

+)

+

+func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) {

+

+	applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) {

+

+		for i, fi := range fis {

+			if fi.IsDir() {

+				filename := filepath.Join(name, fi.Name())

+				fis[i] = decorateFileInfo(fi, fs, fs.getOpener(filename), "", "", nil)

+				continue

+			}

+

+			meta := fi.(FileMetaInfo).Meta()

+			lang := meta.Lang()

+

+			fileLang, translationBaseName, translationBaseNameWithExt := langInfoFrom(langs, fi.Name())

+			weight := 0

+

+			if fileLang != "" {

+				weight = 1

+				if fileLang == lang {

+					// Give priority to myfile.sv.txt inside the sv filesystem.

+					weight++

+				}

+				lang = fileLang

+			}

+

+			fim := NewFileMetaInfo(fi, FileMeta{

+				metaKeyLang:                       lang,

+				metaKeyWeight:                     weight,

+				metaKeyOrdinal:                    langs[lang],

+				metaKeyTranslationBaseName:        translationBaseName,

+				metaKeyTranslationBaseNameWithExt: translationBaseNameWithExt,

+				metaKeyClassifier:                 files.ClassifyContentFile(fi.Name()),

+			})

+

+			fis[i] = fim

+		}

+	}

+

+	all := func(fis []os.FileInfo) {

+		// Maps translation base name to a list of language codes.

+		translations := make(map[string][]string)

+		trackTranslation := func(meta FileMeta) {

+			name := meta.TranslationBaseNameWithExt()

+			translations[name] = append(translations[name], meta.Lang())

+		}

+		for _, fi := range fis {

+			if fi.IsDir() {

+				continue

+			}

+			meta := fi.(FileMetaInfo).Meta()

+

+			trackTranslation(meta)

+

+		}

+

+		for _, fi := range fis {

+			fim := fi.(FileMetaInfo)

+			langs := translations[fim.Meta().TranslationBaseNameWithExt()]

+			if len(langs) > 0 {

+				fim.Meta()["translations"] = sortAndremoveStringDuplicates(langs)

+			}

+		}

+	}

+

+	return &FilterFs{

+		fs:             fs,

+		applyPerSource: applyMeta,

+		applyAll:       all,

+	}, nil

+

+}

+

+func NewFilterFs(fs afero.Fs) (afero.Fs, error) {

+

+	applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) {

+		for i, fi := range fis {

+			if fi.IsDir() {

+				fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil)

+			}

+		}

+	}

+

+	ffs := &FilterFs{

+		fs:             fs,

+		applyPerSource: applyMeta,

+	}

+

+	return ffs, nil

+

+}

+

+// FilterFs is an ordered composite filesystem.

+type FilterFs struct {

+	fs afero.Fs

+

+	applyPerSource func(fs *FilterFs, name string, fis []os.FileInfo)

+	applyAll       func(fis []os.FileInfo)

+}

+

+func (fs *FilterFs) Chmod(n string, m os.FileMode) error {

+	return syscall.EPERM

+}

+

+func (fs *FilterFs) Chtimes(n string, a, m time.Time) error {

+	return syscall.EPERM

+}

+

+func (fs *FilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

+	fi, b, err := lstatIfPossible(fs.fs, name)

+

+	if err != nil {

+		return nil, false, err

+	}

+

+	if fi.IsDir() {

+		return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil

+	}

+

+	fs.applyFilters(name, -1, fi)

+

+	return fi, b, nil

+

+}

+

+func (fs *FilterFs) Mkdir(n string, p os.FileMode) error {

+	return syscall.EPERM

+}

+

+func (fs *FilterFs) MkdirAll(n string, p os.FileMode) error {

+	return syscall.EPERM

+}

+

+func (fs *FilterFs) Name() string {

+	return "WeightedFileSystem"

+}

+

+func (fs *FilterFs) Open(name string) (afero.File, error) {

+	f, err := fs.fs.Open(name)

+	if err != nil {

+		return nil, err

+	}

+

+	return &filterDir{

+		File: f,

+		ffs:  fs,

+	}, nil

+

+}

+

+func (fs *FilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {

+	panic("not implemented")

+}

+

+func (fs *FilterFs) ReadDir(name string) ([]os.FileInfo, error) {

+	panic("not implemented")

+}

+

+func (fs *FilterFs) Remove(n string) error {

+	return syscall.EPERM

+}

+

+func (fs *FilterFs) RemoveAll(p string) error {

+	return syscall.EPERM

+}

+

+func (fs *FilterFs) Rename(o, n string) error {

+	return syscall.EPERM

+}

+

+func (fs *FilterFs) Stat(name string) (os.FileInfo, error) {

+	fi, _, err := fs.LstatIfPossible(name)

+	return fi, err

+}

+

+func (fs *FilterFs) Create(n string) (afero.File, error) {

+	return nil, syscall.EPERM

+}

+

+func (fs *FilterFs) getOpener(name string) func() (afero.File, error) {

+	return func() (afero.File, error) {

+		return fs.Open(name)

+	}

+}

+

+func (fs *FilterFs) applyFilters(name string, count int, fis ...os.FileInfo) ([]os.FileInfo, error) {

+	if fs.applyPerSource != nil {

+		fs.applyPerSource(fs, name, fis)

+	}

+

+	seen := make(map[string]bool)

+	var duplicates []int

+	for i, dir := range fis {

+		if !dir.IsDir() {

+			continue

+		}

+		if seen[dir.Name()] {

+			duplicates = append(duplicates, i)

+		} else {

+			seen[dir.Name()] = true

+		}

+	}

+

+	// Remove duplicate directories, keep first.

+	if len(duplicates) > 0 {

+		for i := len(duplicates) - 1; i >= 0; i-- {

+			idx := duplicates[i]

+			fis = append(fis[:idx], fis[idx+1:]...)

+		}

+	}

+

+	if fs.applyAll != nil {

+		fs.applyAll(fis)

+	}

+

+	if count > 0 && len(fis) >= count {

+		return fis[:count], nil

+	}

+

+	return fis, nil

+

+}

+

+type filterDir struct {

+	afero.File

+	ffs *FilterFs

+}

+

+func (f *filterDir) Readdir(count int) ([]os.FileInfo, error) {

+	fis, err := f.File.Readdir(-1)

+	if err != nil {

+		return nil, err

+	}

+	return f.ffs.applyFilters(f.Name(), count, fis...)

+}

+

+func (f *filterDir) Readdirnames(count int) ([]string, error) {

+	dirsi, err := f.Readdir(count)

+	if err != nil {

+		return nil, err

+	}

+

+	dirs := make([]string, len(dirsi))

+	for i, d := range dirsi {

+		dirs[i] = d.Name()

+	}

+	return dirs, nil

+}

+

+// Try to extract the language from the given filename.

+// Any valid language identificator in the name will win over the

+// language set on the file system, e.g. "mypost.en.md".

+func langInfoFrom(languages map[string]int, name string) (string, string, string) {

+	var lang string

+

+	baseName := filepath.Base(name)

+	ext := filepath.Ext(baseName)

+	translationBaseName := baseName

+

+	if ext != "" {

+		translationBaseName = strings.TrimSuffix(translationBaseName, ext)

+	}

+

+	fileLangExt := filepath.Ext(translationBaseName)

+	fileLang := strings.TrimPrefix(fileLangExt, ".")

+

+	if _, found := languages[fileLang]; found {

+		lang = fileLang

+		translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt)

+	}

+

+	translationBaseNameWithExt := translationBaseName

+

+	if ext != "" {

+		translationBaseNameWithExt += ext

+	}

+

+	return lang, translationBaseName, translationBaseNameWithExt

+

+}

+

+func printFs(fs afero.Fs, path string, w io.Writer) {

+	if fs == nil {

+		return

+	}

+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {

+		fmt.Println("p:::", path)

+		return nil

+	})

+}

+

+func sortAndremoveStringDuplicates(s []string) []string {

+	ss := sort.StringSlice(s)

+	ss.Sort()

+	i := 0

+	for j := 1; j < len(s); j++ {

+		if !ss.Less(i, j) {

+			continue

+		}

+		i++

+		s[i] = s[j]

+	}

+

+	return s[:i+1]

+}

--- /dev/null

+++ b/hugofs/filter_fs_test.go

@@ -1,0 +1,48 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"path/filepath"

+	"testing"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestLangInfoFrom(t *testing.T) {

+

+	langs := map[string]int{

+		"sv": 10,

+		"en": 20,

+	}

+

+	assert := require.New(t)

+

+	tests := []struct {

+		input    string

+		expected []string

+	}{

+		{"page.sv.md", []string{"sv", "page", "page.md"}},

+		{"page.en.md", []string{"en", "page", "page.md"}},

+		{"page.no.md", []string{"", "page.no", "page.no.md"}},

+		{filepath.FromSlash("tc-lib-color/class-Com.Tecnick.Color.Css"), []string{"", "class-Com.Tecnick.Color", "class-Com.Tecnick.Color.Css"}},

+		{filepath.FromSlash("class-Com.Tecnick.Color.sv.Css"), []string{"sv", "class-Com.Tecnick.Color", "class-Com.Tecnick.Color.Css"}},

+	}

+

+	for _, test := range tests {

+		v1, v2, v3 := langInfoFrom(langs, test.input)

+		assert.Equal(test.expected, []string{v1, v2, v3})

+	}

+

+}

--- a/hugofs/fs.go

+++ b/hugofs/fs.go

@@ -21,8 +21,10 @@

 	"github.com/spf13/afero"

 )

-// Os points to an Os Afero file system.

-var Os = &afero.OsFs{}

+var (

+	// Os points to the (real) Os filesystem.

+	Os = &afero.OsFs{}

+)

 // Fs abstracts the file system to separate source and destination file systems

 // and allows both to be mocked for testing.

--- a/hugofs/language_composite_fs.go

+++ b/hugofs/language_composite_fs.go

@@ -14,6 +14,9 @@

 package hugofs

 import (

+	"os"

+	"path"

+

 	"github.com/spf13/afero"

 )

@@ -30,8 +33,8 @@

 // This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename

 // to the target filesystem. This information is available in Readdir, Stat etc. via the

 // special LanguageFileInfo FileInfo implementation.

-func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs {

-	return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)})

+func NewLanguageCompositeFs(base, overlay afero.Fs) afero.Fs {

+	return &languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}

 }

 // Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged

@@ -48,4 +51,37 @@

 		fu.Merger = LanguageDirsMerger

 	}

 	return f, nil

+}

+

+// LanguageDirsMerger implements the afero.DirsMerger interface, which is used

+// to merge two directories.

+var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {

+	m := make(map[string]FileMetaInfo)

+

+	getKey := func(fim FileMetaInfo) string {

+		return path.Join(fim.Meta().Lang(), fim.Name())

+	}

+

+	for _, fi := range lofi {

+		fim := fi.(FileMetaInfo)

+		m[getKey(fim)] = fim

+	}

+

+	for _, fi := range bofi {

+		fim := fi.(FileMetaInfo)

+		key := getKey(fim)

+		_, found := m[key]

+		if !found {

+			m[key] = fim

+		}

+	}

+

+	merged := make([]os.FileInfo, len(m))

+	i := 0

+	for _, v := range m {

+		merged[i] = v

+		i++

+	}

+

+	return merged, nil

 }

--- a/hugofs/language_composite_fs_test.go

+++ /dev/null

@@ -1,107 +1,0 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugofs

-

-import (

-	"path/filepath"

-

-	"strings"

-

-	"testing"

-

-	"github.com/spf13/afero"

-	"github.com/stretchr/testify/require"

-)

-

-func TestCompositeLanguagFsTest(t *testing.T) {

-	assert := require.New(t)

-

-	languages := map[string]bool{

-		"sv": true,

-		"en": true,

-		"nn": true,

-	}

-	msv := afero.NewMemMapFs()

-	baseSv := "/content/sv"

-	lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv))

-	mnn := afero.NewMemMapFs()

-	baseNn := "/content/nn"

-	lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn))

-	men := afero.NewMemMapFs()

-	baseEn := "/content/en"

-	lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn))

-

-	// The order will be sv, en, nn

-	composite := NewLanguageCompositeFs(lfsnn, lfsen)

-	composite = NewLanguageCompositeFs(composite, lfssv)

-

-	afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755)

-	afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755)

-	afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755)

-

-	// Swedish is the top layer.

-	assertLangFile(t, composite, "f1.txt", "sv")

-

-	afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755)

-	afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755)

-	afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755)

-

-	// English is in the middle, but the most specific language match wins.

-	//assertLangFile(t, composite, "f2.en.txt", "en")

-

-	// Fetch some specific language versions

-	assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn")

-	assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en")

-	assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv")

-

-	// Read the root

-	f, err := composite.Open("/")

-	assert.NoError(err)

-	defer f.Close()

-	files, err := f.Readdir(-1)

-	assert.NoError(err)

-	assert.Equal(4, len(files))

-	expected := map[string]bool{

-		filepath.FromSlash("/content/en/f1.txt"):    true,

-		filepath.FromSlash("/content/nn/f1.txt"):    true,

-		filepath.FromSlash("/content/sv/f1.txt"):    true,

-		filepath.FromSlash("/content/en/f2.en.txt"): true,

-	}

-	got := make(map[string]bool)

-

-	for _, fi := range files {

-		fil, ok := fi.(*LanguageFileInfo)

-		assert.True(ok)

-		got[fil.Filename()] = true

-	}

-	assert.Equal(expected, got)

-}

-

-func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) {

-	f, err := fs.Open(filename)

-	if err != nil {

-		t.Fatal(err)

-	}

-	defer f.Close()

-	b, err := afero.ReadAll(f)

-	if err != nil {

-		t.Fatal(err)

-	}

-

-	s := string(b)

-	if !strings.Contains(s, match) {

-		t.Fatalf("got %q expected it to contain %q", s, match)

-

-	}

-}

--- a/hugofs/language_fs.go

+++ /dev/null

@@ -1,346 +1,0 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugofs

-

-import (

-	"fmt"

-	"os"

-	"path/filepath"

-	"strings"

-

-	"github.com/spf13/afero"

-)

-

-const hugoFsMarker = "__hugofs"

-

-var (

-	_ LanguageAnnouncer = (*LanguageFileInfo)(nil)

-	_ FilePather        = (*LanguageFileInfo)(nil)

-	_ afero.Lstater     = (*LanguageFs)(nil)

-)

-

-// LanguageAnnouncer is aware of its language.

-type LanguageAnnouncer interface {

-	Lang() string

-	TranslationBaseName() string

-}

-

-// FilePather is aware of its file's location.

-type FilePather interface {

-	// Filename gets the full path and filename to the file.

-	Filename() string

-

-	// Path gets the content relative path including file name and extension.

-	// The directory is relative to the content root where "content" is a broad term.

-	Path() string

-

-	// RealName is FileInfo.Name in its original form.

-	RealName() string

-

-	BaseDir() string

-}

-

-// LanguageDirsMerger implements the afero.DirsMerger interface, which is used

-// to merge two directories.

-var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {

-	m := make(map[string]*LanguageFileInfo)

-

-	for _, fi := range lofi {

-		fil, ok := fi.(*LanguageFileInfo)

-		if !ok {

-			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)

-		}

-		m[fil.virtualName] = fil

-	}

-

-	for _, fi := range bofi {

-		fil, ok := fi.(*LanguageFileInfo)

-		if !ok {

-			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)

-		}

-		existing, found := m[fil.virtualName]

-

-		if !found || existing.weight < fil.weight {

-			m[fil.virtualName] = fil

-		}

-	}

-

-	merged := make([]os.FileInfo, len(m))

-	i := 0

-	for _, v := range m {

-		merged[i] = v

-		i++

-	}

-

-	return merged, nil

-}

-

-// LanguageFileInfo is a super-set of os.FileInfo with additional information

-// about the file in relation to its Hugo language.

-type LanguageFileInfo struct {

-	os.FileInfo

-	lang                string

-	baseDir             string

-	realFilename        string

-	relFilename         string

-	name                string

-	realName            string

-	virtualName         string

-	translationBaseName string

-

-	// We add some weight to the files in their own language's content directory.

-	weight int

-}

-

-// Filename returns a file's real filename including the base (ie.

-// "/my/base/sect/page.md").

-func (fi *LanguageFileInfo) Filename() string {

-	return fi.realFilename

-}

-

-// Path returns a file's filename relative to the base (ie. "sect/page.md").

-func (fi *LanguageFileInfo) Path() string {

-	return fi.relFilename

-}

-

-// RealName returns a file's real base name (ie. "page.md").

-func (fi *LanguageFileInfo) RealName() string {

-	return fi.realName

-}

-

-// BaseDir returns a file's base directory (ie. "/my/base").

-func (fi *LanguageFileInfo) BaseDir() string {

-	return fi.baseDir

-}

-

-// Lang returns a file's language (ie. "sv").

-func (fi *LanguageFileInfo) Lang() string {

-	return fi.lang

-}

-

-// TranslationBaseName returns the base filename without any extension or language

-// identifiers (ie. "page").

-func (fi *LanguageFileInfo) TranslationBaseName() string {

-	return fi.translationBaseName

-}

-

-// Name is the name of the file within this filesystem without any path info.

-// It will be marked with language information so we can identify it as ours

-// (ie. "__hugofs_sv_page.md").

-func (fi *LanguageFileInfo) Name() string {

-	return fi.name

-}

-

-type languageFile struct {

-	afero.File

-	fs *LanguageFs

-}

-

-// Readdir creates FileInfo entries by calling Lstat if possible.

-func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) {

-	names, err := l.File.Readdirnames(c)

-	if err != nil {

-		return nil, err

-	}

-

-	fis := make([]os.FileInfo, len(names))

-

-	for i, name := range names {

-		fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name))

-

-		if err != nil {

-			return nil, err

-		}

-		fis[i] = fi

-	}

-

-	return fis, err

-}

-

-// LanguageFs represents a language filesystem.

-type LanguageFs struct {

-	// This Fs is usually created with a BasePathFs

-	basePath   string

-	lang       string

-	nameMarker string

-	languages  map[string]bool

-	afero.Fs

-}

-

-// NewLanguageFs creates a new language filesystem.

-func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs {

-	if lang == "" {

-		panic("no lang set for the language fs")

-	}

-	var basePath string

-

-	if bfs, ok := fs.(*afero.BasePathFs); ok {

-		basePath, _ = bfs.RealPath("")

-	}

-

-	marker := hugoFsMarker + "_" + lang + "_"

-

-	return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker}

-}

-

-// Lang returns a language filesystem's language (ie. "sv").

-func (fs *LanguageFs) Lang() string {

-	return fs.lang

-}

-

-// Stat returns the os.FileInfo of a given file.

-func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) {

-	name, err := fs.realName(name)

-	if err != nil {

-		return nil, err

-	}

-

-	fi, err := fs.Fs.Stat(name)

-	if err != nil {

-		return nil, err

-	}

-

-	return fs.newLanguageFileInfo(name, fi)

-}

-

-// Open opens the named file for reading.

-func (fs *LanguageFs) Open(name string) (afero.File, error) {

-	name, err := fs.realName(name)

-	if err != nil {

-		return nil, err

-	}

-	f, err := fs.Fs.Open(name)

-

-	if err != nil {

-		return nil, err

-	}

-	return &languageFile{File: f, fs: fs}, nil

-}

-

-// LstatIfPossible returns the os.FileInfo structure describing a given file.

-// It attempts to use Lstat if supported or defers to the os.  In addition to

-// the FileInfo, a boolean is returned telling whether Lstat was called.

-func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

-	name, err := fs.realName(name)

-	if err != nil {

-		return nil, false, err

-	}

-

-	var fi os.FileInfo

-	var b bool

-

-	if lif, ok := fs.Fs.(afero.Lstater); ok {

-		fi, b, err = lif.LstatIfPossible(name)

-	} else {

-		fi, err = fs.Fs.Stat(name)

-	}

-

-	if err != nil {

-		return nil, b, err

-	}

-

-	lfi, err := fs.newLanguageFileInfo(name, fi)

-

-	return lfi, b, err

-}

-

-func (fs *LanguageFs) realPath(name string) (string, error) {

-	if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok {

-		return baseFs.RealPath(name)

-	}

-	return name, nil

-}

-

-func (fs *LanguageFs) realName(name string) (string, error) {

-	if strings.Contains(name, hugoFsMarker) {

-		if !strings.Contains(name, fs.nameMarker) {

-			return "", os.ErrNotExist

-		}

-		return strings.Replace(name, fs.nameMarker, "", 1), nil

-	}

-

-	if fs.basePath == "" {

-		return name, nil

-	}

-

-	return strings.TrimPrefix(name, fs.basePath), nil

-}

-

-func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) {

-	filename = filepath.Clean(filename)

-	_, name := filepath.Split(filename)

-

-	realName := name

-	virtualName := name

-

-	realPath, err := fs.realPath(filename)

-	if err != nil {

-		return nil, err

-	}

-

-	lang := fs.Lang()

-

-	baseNameNoExt := ""

-

-	if !fi.IsDir() {

-

-		// Try to extract the language from the file name.

-		// Any valid language identificator in the name will win over the

-		// language set on the file system, e.g. "mypost.en.md".

-		baseName := filepath.Base(name)

-		ext := filepath.Ext(baseName)

-		baseNameNoExt = baseName

-

-		if ext != "" {

-			baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext)

-		}

-

-		fileLangExt := filepath.Ext(baseNameNoExt)

-		fileLang := strings.TrimPrefix(fileLangExt, ".")

-

-		if fs.languages[fileLang] {

-			lang = fileLang

-			baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt)

-		}

-

-		// This connects the filename to the filesystem, not the language.

-		virtualName = baseNameNoExt + "." + lang + ext

-

-		name = fs.nameMarker + name

-	}

-

-	weight := 1

-	// If this file's language belongs in this directory, add some weight to it

-	// to make it more important.

-	if lang == fs.Lang() {

-		weight = 2

-	}

-

-	if fi.IsDir() {

-		// For directories we always want to start from the union view.

-		realPath = strings.TrimPrefix(realPath, fs.basePath)

-	}

-

-	return &LanguageFileInfo{

-		lang:                lang,

-		weight:              weight,

-		realFilename:        realPath,

-		realName:            realName,

-		relFilename:         strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)),

-		name:                name,

-		virtualName:         virtualName,

-		translationBaseName: baseNameNoExt,

-		baseDir:             fs.basePath,

-		FileInfo:            fi}, nil

-}

--- a/hugofs/language_fs_test.go

+++ /dev/null

@@ -1,100 +1,0 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugofs

-

-import (

-	"path/filepath"

-	"testing"

-

-	"github.com/spf13/afero"

-	"github.com/stretchr/testify/require"

-)

-

-func TestLanguagFs(t *testing.T) {

-	languages := map[string]bool{

-		"sv": true,

-	}

-	base := filepath.FromSlash("/my/base")

-	assert := require.New(t)

-	m := afero.NewMemMapFs()

-	bfs := afero.NewBasePathFs(m, base)

-	lfs := NewLanguageFs("sv", languages, bfs)

-	assert.NotNil(lfs)

-	assert.Equal("sv", lfs.Lang())

-	err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777)

-	assert.NoError(err)

-	fi, err := lfs.Stat(filepath.FromSlash("sect/page.md"))

-	assert.NoError(err)

-	assert.Equal("__hugofs_sv_page.md", fi.Name())

-

-	languager, ok := fi.(LanguageAnnouncer)

-	assert.True(ok)

-

-	assert.Equal("sv", languager.Lang())

-

-	lfi, ok := fi.(*LanguageFileInfo)

-	assert.True(ok)

-	assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename())

-	assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path())

-	assert.Equal("page.sv.md", lfi.virtualName)

-	assert.Equal("__hugofs_sv_page.md", lfi.Name())

-	assert.Equal("page.md", lfi.RealName())

-	assert.Equal(filepath.FromSlash("/my/base"), lfi.BaseDir())

-	assert.Equal("sv", lfi.Lang())

-	assert.Equal("page", lfi.TranslationBaseName())

-}

-

-// Issue 4559

-func TestFilenamesHandling(t *testing.T) {

-	languages := map[string]bool{

-		"sv": true,

-	}

-	base := filepath.FromSlash("/my/base")

-	assert := require.New(t)

-	m := afero.NewMemMapFs()

-	bfs := afero.NewBasePathFs(m, base)

-	lfs := NewLanguageFs("sv", languages, bfs)

-	assert.NotNil(lfs)

-	assert.Equal("sv", lfs.Lang())

-

-	for _, test := range []struct {

-		filename string

-		check    func(fi *LanguageFileInfo)

-	}{

-		{"tc-lib-color/class-Com.Tecnick.Color.Css", func(fi *LanguageFileInfo) {

-			assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName())

-			assert.Equal(filepath.FromSlash("/my/base"), fi.BaseDir())

-			assert.Equal(filepath.FromSlash("tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Path())

-			assert.Equal("class-Com.Tecnick.Color.Css", fi.RealName())

-			assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Filename())

-		}},

-		{"tc-lib-color/class-Com.Tecnick.Color.sv.Css", func(fi *LanguageFileInfo) {

-			assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName())

-			assert.Equal("class-Com.Tecnick.Color.sv.Css", fi.RealName())

-			assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.sv.Css"), fi.Filename())

-		}},

-	} {

-		err := afero.WriteFile(lfs, filepath.FromSlash(test.filename), []byte("abc"), 0777)

-		assert.NoError(err)

-		fi, err := lfs.Stat(filepath.FromSlash(test.filename))

-		assert.NoError(err)

-

-		lfi, ok := fi.(*LanguageFileInfo)

-		assert.True(ok)

-		assert.Equal("sv", lfi.Lang())

-		test.check(lfi)

-

-	}

-

-}

--- a/hugofs/nolstat_fs.go

+++ /dev/null

@@ -1,39 +1,0 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugofs

-

-import (

-	"os"

-

-	"github.com/spf13/afero"

-)

-

-var (

-	_ afero.Fs = (*noLstatFs)(nil)

-)

-

-type noLstatFs struct {

-	afero.Fs

-}

-

-// NewNoLstatFs creates a new filesystem with no Lstat support.

-func NewNoLstatFs(fs afero.Fs) afero.Fs {

-	return &noLstatFs{Fs: fs}

-}

-

-// LstatIfPossible always delegates to Stat.

-func (fs *noLstatFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

-	fi, err := fs.Stat(name)

-	return fi, false, err

-}

--- /dev/null

+++ b/hugofs/nosymlink_fs.go

@@ -1,0 +1,85 @@

+// Copyright 2018 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"errors"

+	"os"

+

+	"github.com/spf13/afero"

+)

+

+var (

+	ErrPermissionSymlink = errors.New("symlinks not allowed in this filesystem")

+)

+

+func NewNoSymlinkFs(fs afero.Fs) afero.Fs {

+	return &noSymlinkFs{Fs: fs}

+}

+

+// noSymlinkFs is a filesystem that prevents symlinking.

+type noSymlinkFs struct {

+	afero.Fs

+}

+

+func (fs *noSymlinkFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

+	return fs.stat(name)

+}

+

+func (fs *noSymlinkFs) Stat(name string) (os.FileInfo, error) {

+	fi, _, err := fs.stat(name)

+	return fi, err

+}

+

+func (fs *noSymlinkFs) stat(name string) (os.FileInfo, bool, error) {

+

+	var (

+		fi       os.FileInfo

+		wasLstat bool

+		err      error

+	)

+

+	if lstater, ok := fs.Fs.(afero.Lstater); ok {

+		fi, wasLstat, err = lstater.LstatIfPossible(name)

+	} else {

+

+		fi, err = fs.Fs.Stat(name)

+	}

+

+	var metaIsSymlink bool

+

+	if fim, ok := fi.(FileMetaInfo); ok {

+		metaIsSymlink = fim.Meta().IsSymlink()

+	}

+

+	if metaIsSymlink || isSymlink(fi) {

+		return nil, wasLstat, ErrPermissionSymlink

+	}

+

+	return fi, wasLstat, err

+}

+

+func (fs *noSymlinkFs) Open(name string) (afero.File, error) {

+	if _, _, err := fs.stat(name); err != nil {

+		return nil, err

+	}

+	return fs.Fs.Open(name)

+}

+

+func (fs *noSymlinkFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {

+	if _, _, err := fs.stat(name); err != nil {

+		return nil, err

+	}

+	return fs.Fs.OpenFile(name, flag, perm)

+}

--- /dev/null

+++ b/hugofs/nosymlink_test.go

@@ -1,0 +1,97 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"os"

+	"path/filepath"

+	"testing"

+

+	"github.com/gohugoio/hugo/htesting"

+

+	"github.com/spf13/afero"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestNoSymlinkFs(t *testing.T) {

+	if skipSymlink() {

+		t.Skip("Skip; os.Symlink needs administrator rights on Windows")

+	}

+	assert := require.New(t)

+	workDir, clean, err := htesting.CreateTempDir(Os, "hugo-nosymlink")

+	assert.NoError(err)

+	defer clean()

+	wd, _ := os.Getwd()

+	defer func() {

+		os.Chdir(wd)

+	}()

+

+	blogDir := filepath.Join(workDir, "blog")

+	blogFile := filepath.Join(blogDir, "a.txt")

+	assert.NoError(os.MkdirAll(blogDir, 0777))

+	afero.WriteFile(Os, filepath.Join(blogFile), []byte("content"), 0777)

+	os.Chdir(workDir)

+	assert.NoError(os.Symlink("blog", "symlinkdedir"))

+	os.Chdir(blogDir)

+	assert.NoError(os.Symlink("a.txt", "symlinkdedfile.txt"))

+

+	fs := NewNoSymlinkFs(Os)

+	ls := fs.(afero.Lstater)

+	symlinkedDir := filepath.Join(workDir, "symlinkdedir")

+	symlinkedFile := filepath.Join(blogDir, "symlinkdedfile.txt")

+

+	// Check Stat and Lstat

+	for _, stat := range []func(name string) (os.FileInfo, error){

+		func(name string) (os.FileInfo, error) {

+			return fs.Stat(name)

+		},

+		func(name string) (os.FileInfo, error) {

+			fi, _, err := ls.LstatIfPossible(name)

+			return fi, err

+		},

+	} {

+		_, err = stat(symlinkedDir)

+		assert.Equal(ErrPermissionSymlink, err)

+		_, err = stat(symlinkedFile)

+		assert.Equal(ErrPermissionSymlink, err)

+

+		fi, err := stat(filepath.Join(workDir, "blog"))

+		assert.NoError(err)

+		assert.NotNil(fi)

+

+		fi, err = stat(blogFile)

+		assert.NoError(err)

+		assert.NotNil(fi)

+	}

+

+	// Check Open

+	_, err = fs.Open(symlinkedDir)

+	assert.Equal(ErrPermissionSymlink, err)

+	_, err = fs.OpenFile(symlinkedDir, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)

+	assert.Equal(ErrPermissionSymlink, err)

+	_, err = fs.OpenFile(symlinkedFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)

+	assert.Equal(ErrPermissionSymlink, err)

+	_, err = fs.Open(symlinkedFile)

+	assert.Equal(ErrPermissionSymlink, err)

+	f, err := fs.Open(blogDir)

+	assert.NoError(err)

+	f.Close()

+	f, err = fs.Open(blogFile)

+	assert.NoError(err)

+	f.Close()

+

+	// os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)

+

+}

--- a/hugofs/rootmapping_fs.go

+++ b/hugofs/rootmapping_fs.go

@@ -1,4 +1,4 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

+// Copyright 2019 The Hugo Authors. All rights reserved.

 //

 // Licensed under the Apache License, Version 2.0 (the "License");

 // you may not use this file except in compliance with the License.

@@ -14,11 +14,15 @@

 package hugofs

 import (

+	"fmt"

 	"os"

 	"path/filepath"

 	"strings"

-	"time"

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/pkg/errors"

+

 	radix "github.com/hashicorp/go-immutable-radix"

 	"github.com/spf13/afero"

 )

@@ -25,151 +29,429 @@

 var filepathSeparator = string(filepath.Separator)

-// A RootMappingFs maps several roots into one. Note that the root of this filesystem

-// is directories only, and they will be returned in Readdir and Readdirnames

-// in the order given.

-type RootMappingFs struct {

-	afero.Fs

-	rootMapToReal *radix.Node

-	virtualRoots  []string

+// NewRootMappingFs creates a new RootMappingFs on top of the provided with

+// of root mappings with some optional metadata about the root.

+// Note that From represents a virtual root that maps to the actual filename in To.

+func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {

+	rootMapToReal := radix.New().Txn()

+

+	for _, rm := range rms {

+		(&rm).clean()

+

+		fromBase := files.ResolveComponentFolder(rm.From)

+		if fromBase == "" {

+			panic("unrecognised component folder in" + rm.From)

+		}

+

+		if len(rm.To) < 2 {

+			panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To))

+		}

+

+		_, err := fs.Stat(rm.To)

+		if err != nil {

+			if os.IsNotExist(err) {

+				continue

+			}

+			return nil, err

+		}

+

+		// Extract "blog" from "content/blog"

+		rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)

+

+		key := []byte(rm.rootKey())

+		var mappings []RootMapping

+		v, found := rootMapToReal.Get(key)

+		if found {

+			// There may be more than one language pointing to the same root.

+			mappings = v.([]RootMapping)

+		}

+		mappings = append(mappings, rm)

+		rootMapToReal.Insert(key, mappings)

+	}

+

+	rfs := &RootMappingFs{Fs: fs,

+		virtualRoots:  rms,

+		rootMapToReal: rootMapToReal.Commit().Root()}

+

+	return rfs, nil

 }

-type rootMappingFile struct {

-	afero.File

-	fs   *RootMappingFs

-	name string

+// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking

+// From and To as string pairs.

+func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {

+	rms := make([]RootMapping, len(fromTo)/2)

+	for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 {

+		rms[i] = RootMapping{

+			From: fromTo[j],

+			To:   fromTo[j+1],

+		}

+	}

+

+	return NewRootMappingFs(fs, rms...)

 }

-type rootMappingFileInfo struct {

-	name string

+type RootMapping struct {

+	From string

+	To   string

+

+	path string   // The virtual mount point, e.g. "blog".

+	Meta FileMeta // File metadata (lang etc.)

 }

-func (fi *rootMappingFileInfo) Name() string {

-	return fi.name

+func (rm *RootMapping) clean() {

+	rm.From = filepath.Clean(rm.From)

+	rm.To = filepath.Clean(rm.To)

 }

-func (fi *rootMappingFileInfo) Size() int64 {

-	panic("not implemented")

+func (r RootMapping) filename(name string) string {

+	return filepath.Join(r.To, strings.TrimPrefix(name, r.From))

 }

-func (fi *rootMappingFileInfo) Mode() os.FileMode {

-	return os.ModeDir

+func (r RootMapping) rootKey() string {

+	return r.From

 }

-func (fi *rootMappingFileInfo) ModTime() time.Time {

-	panic("not implemented")

+// A RootMappingFs maps several roots into one. Note that the root of this filesystem

+// is directories only, and they will be returned in Readdir and Readdirnames

+// in the order given.

+type RootMappingFs struct {

+	afero.Fs

+	rootMapToReal *radix.Node

+	virtualRoots  []RootMapping

+	filter        func(r RootMapping) bool

 }

-func (fi *rootMappingFileInfo) IsDir() bool {

-	return true

+func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {

+	roots := fs.getRootsWithPrefix(base)

+

+	if roots == nil {

+		return nil, nil

+	}

+

+	fss := make([]FileMetaInfo, len(roots))

+	for i, r := range roots {

+		bfs := afero.NewBasePathFs(fs.Fs, r.To)

+		bfs = decoratePath(bfs, func(name string) string {

+			p := strings.TrimPrefix(name, r.To)

+			if r.path != "" {

+				// Make sure it's mounted to a any sub path, e.g. blog

+				p = filepath.Join(r.path, p)

+			}

+			p = strings.TrimLeft(p, filepathSeparator)

+			return p

+		})

+		fs := decorateDirs(bfs, r.Meta)

+		fi, err := fs.Stat("")

+		if err != nil {

+			return nil, errors.Wrap(err, "RootMappingFs.Dirs")

+		}

+		fss[i] = fi.(FileMetaInfo)

+	}

+

+	return fss, nil

 }

-func (fi *rootMappingFileInfo) Sys() interface{} {

-	return nil

+// LstatIfPossible returns the os.FileInfo structure describing a given file.

+func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

+	fis, b, err := fs.doLstat(name, false)

+	if err != nil {

+		return nil, b, err

+	}

+	return fis[0], b, nil

+

 }

-func newRootMappingDirFileInfo(name string) *rootMappingFileInfo {

-	return &rootMappingFileInfo{name: name}

+func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) {

+	return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil }

 }

-// NewRootMappingFs creates a new RootMappingFs on top of the provided with

-// a list of from, to string pairs of root mappings.

-// Note that 'from' represents a virtual root that maps to the actual filename in 'to'.

-func NewRootMappingFs(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {

-	rootMapToReal := radix.New().Txn()

-	var virtualRoots []string

+func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, bool, error) {

-	for i := 0; i < len(fromTo); i += 2 {

-		vr := filepath.Clean(fromTo[i])

-		rr := filepath.Clean(fromTo[i+1])

+	if fs.isRoot(name) {

+		return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, false, nil

+	}

-		// We need to preserve the original order for Readdir

-		virtualRoots = append(virtualRoots, vr)

+	roots := fs.getRoots(name)

-		rootMapToReal.Insert([]byte(vr), rr)

+	if len(roots) == 0 {

+		roots := fs.getRootsWithPrefix(name)

+		if len(roots) != 0 {

+			// We have root mappings below name, let's make it look like

+			// a directory.

+			return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, false, nil

+		}

+

+		return nil, false, os.ErrNotExist

 	}

-	return &RootMappingFs{Fs: fs,

-		virtualRoots:  virtualRoots,

-		rootMapToReal: rootMapToReal.Commit().Root()}, nil

+	var (

+		fis  []FileMetaInfo

+		b    bool

+		fi   os.FileInfo

+		root RootMapping

+		err  error

+	)

+

+	for _, root = range roots {

+		fi, b, err = fs.statRoot(root, name)

+		if err != nil {

+			if os.IsNotExist(err) {

+				continue

+			}

+			return nil, false, err

+		}

+		fim := fi.(FileMetaInfo)

+		fis = append(fis, fim)

+	}

+

+	if len(fis) == 0 {

+		return nil, false, os.ErrNotExist

+	}

+

+	if allowMultiple || len(fis) == 1 {

+		return fis, b, nil

+	}

+

+	// Open it in this composite filesystem.

+	opener := func() (afero.File, error) {

+		return fs.Open(name)

+	}

+

+	return []FileMetaInfo{decorateFileInfo(fi, fs, opener, "", "", root.Meta)}, b, nil

+

 }

-// Stat returns the os.FileInfo structure describing a given file.  If there is

-// an error, it will be of type *os.PathError.

-func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) {

+// Open opens the namedrootMappingFile file for reading.

+func (fs *RootMappingFs) Open(name string) (afero.File, error) {

 	if fs.isRoot(name) {

-		return newRootMappingDirFileInfo(name), nil

+		return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil

 	}

-	realName := fs.realName(name)

-	fi, err := fs.Fs.Stat(realName)

-	if rfi, ok := fi.(RealFilenameInfo); ok {

-		return rfi, err

+	fis, _, err := fs.doLstat(name, true)

+	if err != nil {

+		return nil, err

 	}

-	return &realFilenameInfo{FileInfo: fi, realFilename: realName}, err

+	if len(fis) == 1 {

+		fi := fis[0]

+		meta := fi.(FileMetaInfo).Meta()

+		f, err := meta.Open()

+		if err != nil {

+			return nil, err

+		}

+		return &rootMappingFile{File: f, fs: fs, name: name, meta: meta}, nil

+	}

+	return fs.newUnionFile(fis...)

+

 }

+// Stat returns the os.FileInfo structure describing a given file.  If there is

+// an error, it will be of type *os.PathError.

+func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) {

+	fi, _, err := fs.LstatIfPossible(name)

+	return fi, err

+

+}

+

+// Filter creates a copy of this filesystem with the applied filter.

+func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {

+	fs.filter = f

+	return &fs

+}

+

 func (fs *RootMappingFs) isRoot(name string) bool {

 	return name == "" || name == filepathSeparator

 }

-// Open opens the named file for reading.

-func (fs *RootMappingFs) Open(name string) (afero.File, error) {

-	if fs.isRoot(name) {

-		return &rootMappingFile{name: name, fs: fs}, nil

+func (fs *RootMappingFs) getRoots(name string) []RootMapping {

+	nameb := []byte(filepath.Clean(name))

+	_, v, found := fs.rootMapToReal.LongestPrefix(nameb)

+	if !found {

+		return nil

+	}

+

+	rm := v.([]RootMapping)

+

+	if fs.filter != nil {

+		var filtered []RootMapping

+		for _, m := range rm {

+			if fs.filter(m) {

+				filtered = append(filtered, m)

+			}

+		}

+		return filtered

 	}

-	realName := fs.realName(name)

-	f, err := fs.Fs.Open(realName)

+

+	return rm

+}

+

+func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {

+	if fs.isRoot(prefix) {

+		return fs.virtualRoots

+	}

+	prefixb := []byte(filepath.Clean(prefix))

+	var roots []RootMapping

+

+	fs.rootMapToReal.WalkPrefix(prefixb, func(b []byte, v interface{}) bool {

+		roots = append(roots, v.([]RootMapping)...)

+		return false

+	})

+

+	return roots

+}

+

+func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {

+	meta := fis[0].Meta()

+	f, err := meta.Open()

 	if err != nil {

 		return nil, err

 	}

-	return &rootMappingFile{File: f, name: name, fs: fs}, nil

-}

+	rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta}

+	if len(fis) == 1 {

+		return rf, err

+	}

-// LstatIfPossible returns the os.FileInfo structure describing a given file.

-// It attempts to use Lstat if supported or defers to the os.  In addition to

-// the FileInfo, a boolean is returned telling whether Lstat was called.

-func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

+	next, err := fs.newUnionFile(fis[1:]...)

+	if err != nil {

+		return nil, err

+	}

-	if fs.isRoot(name) {

-		return newRootMappingDirFileInfo(name), false, nil

+	uf := &afero.UnionFile{Base: rf, Layer: next}

+

+	uf.Merger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {

+		// Ignore duplicate directory entries

+		seen := make(map[string]bool)

+		var result []os.FileInfo

+

+		for _, fis := range [][]os.FileInfo{bofi, lofi} {

+			for _, fi := range fis {

+

+				if fi.IsDir() && seen[fi.Name()] {

+					continue

+				}

+

+				if fi.IsDir() {

+					seen[fi.Name()] = true

+				}

+

+				result = append(result, fi)

+			}

+		}

+

+		return result, nil

 	}

-	name = fs.realName(name)

+	return uf, nil

+

+}

+

+func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) {

+	filename := root.filename(name)

+

+	var b bool

+	var fi os.FileInfo

+	var err error

+

 	if ls, ok := fs.Fs.(afero.Lstater); ok {

-		fi, b, err := ls.LstatIfPossible(name)

-		return &realFilenameInfo{FileInfo: fi, realFilename: name}, b, err

+		fi, b, err = ls.LstatIfPossible(filename)

+		if err != nil {

+			return nil, b, err

+		}

+

+	} else {

+		fi, err = fs.Fs.Stat(filename)

+		if err != nil {

+			return nil, b, err

+		}

 	}

-	fi, err := fs.Stat(name)

-	return fi, false, err

+

+	// Opens the real directory/file.

+	opener := func() (afero.File, error) {

+		return fs.Fs.Open(filename)

+	}

+

+	if fi.IsDir() {

+		_, name = filepath.Split(name)

+		fi = newDirNameOnlyFileInfo(name, false, opener)

+	}

+

+	return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil

+

 }

-func (fs *RootMappingFs) realName(name string) string {

-	key, val, found := fs.rootMapToReal.LongestPrefix([]byte(filepath.Clean(name)))

-	if !found {

-		return name

+type rootMappingFile struct {

+	afero.File

+	fs     *RootMappingFs

+	name   string

+	meta   FileMeta

+	isRoot bool

+}

+

+func (f *rootMappingFile) Close() error {

+	if f.File == nil {

+		return nil

 	}

-	keystr := string(key)

+	return f.File.Close()

+}

-	return filepath.Join(val.(string), strings.TrimPrefix(name, keystr))

+func (f *rootMappingFile) Name() string {

+	return f.name

 }

 func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {

 	if f.File == nil {

 		dirsn := make([]os.FileInfo, 0)

-		for i := 0; i < len(f.fs.virtualRoots); i++ {

-			if count != -1 && i >= count {

+		roots := f.fs.getRootsWithPrefix(f.name)

+		seen := make(map[string]bool)

+

+		j := 0

+		for _, rm := range roots {

+			if count != -1 && j >= count {

 				break

 			}

-			dirsn = append(dirsn, newRootMappingDirFileInfo(f.fs.virtualRoots[i]))

+

+			opener := func() (afero.File, error) {

+				return f.fs.Open(rm.From)

+			}

+

+			name := rm.From

+			if !f.isRoot {

+				_, name = filepath.Split(rm.From)

+			}

+

+			if seen[name] {

+				continue

+			}

+			seen[name] = true

+

+			j++

+

+			fi := newDirNameOnlyFileInfo(name, false, opener)

+			if rm.Meta != nil {

+				mergeFileMeta(rm.Meta, fi.Meta())

+			}

+

+			dirsn = append(dirsn, fi)

 		}

 		return dirsn, nil

 	}

-	return f.File.Readdir(count)

+	if f.File == nil {

+		panic(fmt.Sprintf("no File for %q", f.name))

+	}

+

+	fis, err := f.File.Readdir(count)

+	if err != nil {

+		return nil, err

+	}

+

+	for i, fi := range fis {

+		fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)

+	}

+

+	return fis, nil

 }

 func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {

@@ -182,15 +464,4 @@

 		dirss[i] = d.Name()

 	}

 	return dirss, nil

-}

-

-func (f *rootMappingFile) Name() string {

-	return f.name

-}

-

-func (f *rootMappingFile) Close() error {

-	if f.File == nil {

-		return nil

-	}

-	return f.File.Close()

 }

--- a/hugofs/rootmapping_fs_test.go

+++ b/hugofs/rootmapping_fs_test.go

@@ -1,4 +1,4 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

+// Copyright 2019 The Hugo Authors. All rights reserved.

 //

 // Licensed under the Apache License, Version 2.0 (the "License");

 // you may not use this file except in compliance with the License.

@@ -19,24 +19,115 @@

 	"path/filepath"

 	"testing"

+	"github.com/spf13/viper"

+

+	"github.com/gohugoio/hugo/htesting"

 	"github.com/spf13/afero"

 	"github.com/stretchr/testify/require"

 )

-func TestRootMappingFsRealName(t *testing.T) {

+func TestLanguageRootMapping(t *testing.T) {

 	assert := require.New(t)

-	fs := afero.NewMemMapFs()

+	v := viper.New()

+	v.Set("contentDir", "content")

-	rfs, err := NewRootMappingFs(fs, "f1", "f1t", "f2", "f2t")

+	fs := NewBaseFileDecorator(afero.NewMemMapFs())

+

+	assert.NoError(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755))

+

+	rfs, err := NewRootMappingFs(fs,

+		RootMapping{

+			From: "content/blog",             // Virtual path, first element is one of content, static, layouts etc.

+			To:   "themes/a/mysvblogcontent", // Real path

+			Meta: FileMeta{"lang": "sv"},

+		},

+		RootMapping{

+			From: "content/blog",

+			To:   "themes/a/myenblogcontent",

+			Meta: FileMeta{"lang": "en"},

+		},

+		RootMapping{

+			From: "content/blog",

+			To:   "content/sv",

+			Meta: FileMeta{"lang": "sv"},

+		},

+		RootMapping{

+			From: "content/blog",

+			To:   "themes/a/myotherenblogcontent",

+			Meta: FileMeta{"lang": "en"},

+		},

+		RootMapping{

+			From: "content/docs",

+			To:   "themes/a/mysvdocs",

+			Meta: FileMeta{"lang": "sv"},

+		},

+	)

+

 	assert.NoError(err)

-	assert.Equal(filepath.FromSlash("f1t/foo/file.txt"), rfs.realName(filepath.Join("f1", "foo", "file.txt")))

+	collected, err := collectFilenames(rfs, "content", "content")

+	assert.NoError(err)

+	assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected)

+	bfs := afero.NewBasePathFs(rfs, "content")

+	collected, err = collectFilenames(bfs, "", "")

+	assert.NoError(err)

+	assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected)

+

+	dirs, err := rfs.Dirs(filepath.FromSlash("content/blog"))

+	assert.NoError(err)

+

+	assert.Equal(4, len(dirs))

+

+	getDirnames := func(name string, rfs *RootMappingFs) []string {

+		filename := filepath.FromSlash(name)

+		f, err := rfs.Open(filename)

+		assert.NoError(err)

+		names, err := f.Readdirnames(-1)

+

+		f.Close()

+		assert.NoError(err)

+

+		info, err := rfs.Stat(filename)

+		assert.NoError(err)

+		f2, err := info.(FileMetaInfo).Meta().Open()

+		assert.NoError(err)

+		names2, err := f2.Readdirnames(-1)

+		assert.NoError(err)

+		assert.Equal(names, names2)

+		f2.Close()

+

+		return names

+	}

+

+	rfsEn := rfs.Filter(func(rm RootMapping) bool {

+		return rm.Meta.Lang() == "en"

+	})

+

+	assert.Equal([]string{"en-f.txt", "en-f2.txt"}, getDirnames("content/blog", rfsEn))

+

+	rfsSv := rfs.Filter(func(rm RootMapping) bool {

+		return rm.Meta.Lang() == "sv"

+	})

+

+	assert.Equal([]string{"sv-f.txt", "svdir"}, getDirnames("content/blog", rfsSv))

+

+	// Make sure we have not messed with the original

+	assert.Equal([]string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"}, getDirnames("content/blog", rfs))

+

+	assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfsSv))

+	assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfs))

+

 }

 func TestRootMappingFsDirnames(t *testing.T) {

 	assert := require.New(t)

-	fs := afero.NewMemMapFs()

+	fs := NewBaseFileDecorator(afero.NewMemMapFs())

 	testfile := "myfile.txt"

 	assert.NoError(fs.Mkdir("f1t", 0755))

@@ -44,13 +135,14 @@

 	assert.NoError(fs.Mkdir("f3t", 0755))

 	assert.NoError(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755))

-	rfs, err := NewRootMappingFs(fs, "bf1", "f1t", "cf2", "f2t", "af3", "f3t")

+	rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")

 	assert.NoError(err)

-	fif, err := rfs.Stat(filepath.Join("cf2", testfile))

+	fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))

 	assert.NoError(err)

 	assert.Equal("myfile.txt", fif.Name())

-	assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fif.(RealFilenameInfo).RealFilename())

+	fifm := fif.(FileMetaInfo).Meta()

+	assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fifm.Filename())

 	root, err := rfs.Open(filepathSeparator)

 	assert.NoError(err)

@@ -61,6 +153,91 @@

 }

+func TestRootMappingFsFilename(t *testing.T) {

+	assert := require.New(t)

+	workDir, clean, err := htesting.CreateTempDir(Os, "hugo-root-filename")

+	assert.NoError(err)

+	defer clean()

+	fs := NewBaseFileDecorator(Os)

+

+	testfilename := filepath.Join(workDir, "f1t/foo/file.txt")

+

+	assert.NoError(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777))

+	assert.NoError(afero.WriteFile(fs, testfilename, []byte("content"), 0666))

+

+	rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))

+	assert.NoError(err)

+

+	fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt"))

+	assert.NoError(err)

+	fim := fi.(FileMetaInfo)

+	assert.Equal(testfilename, fim.Meta().Filename())

+	_, err = rfs.Stat(filepath.FromSlash("static/f1"))

+	assert.NoError(err)

+}

+

+func TestRootMappingFsMount(t *testing.T) {

+	assert := require.New(t)

+	fs := NewBaseFileDecorator(afero.NewMemMapFs())

+

+	testfile := "test.txt"

+

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mynoblogcontent", testfile), []byte("some no content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", testfile), []byte("some en content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", testfile), []byte("some sv content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "other.txt"), []byte("some sv content"), 0755))

+

+	bfs := afero.NewBasePathFs(fs, "themes/a").(*afero.BasePathFs)

+	rm := []RootMapping{

+		RootMapping{From: "content/blog",

+			To:   "mynoblogcontent",

+			Meta: FileMeta{"lang": "no"},

+		},

+		RootMapping{From: "content/blog",

+			To:   "myenblogcontent",

+			Meta: FileMeta{"lang": "en"},

+		},

+		RootMapping{From: "content/blog",

+			To:   "mysvblogcontent",

+			Meta: FileMeta{"lang": "sv"},

+		},

+	}

+

+	rfs, err := NewRootMappingFs(bfs, rm...)

+	assert.NoError(err)

+

+	blog, err := rfs.Stat(filepath.FromSlash("content/blog"))

+	assert.NoError(err)

+	blogm := blog.(FileMetaInfo).Meta()

+	assert.Equal("sv", blogm.Lang()) // Last match

+

+	f, err := blogm.Open()

+	assert.NoError(err)

+	defer f.Close()

+	dirs1, err := f.Readdirnames(-1)

+	assert.NoError(err)

+	// Union with duplicate dir names filtered.

+	assert.Equal([]string{"test.txt", "test.txt", "other.txt", "test.txt"}, dirs1)

+

+	files, err := afero.ReadDir(rfs, filepath.FromSlash("content/blog"))

+	assert.NoError(err)

+	assert.Equal(4, len(files))

+

+	testfilefi := files[1]

+	assert.Equal(testfile, testfilefi.Name())

+

+	testfilem := testfilefi.(FileMetaInfo).Meta()

+	assert.Equal(filepath.FromSlash("themes/a/mynoblogcontent/test.txt"), testfilem.Filename())

+

+	tf, err := testfilem.Open()

+	assert.NoError(err)

+	defer tf.Close()

+	c, err := ioutil.ReadAll(tf)

+	assert.NoError(err)

+	assert.Equal("some no content", string(c))

+

+}

+

 func TestRootMappingFsOs(t *testing.T) {

 	assert := require.New(t)

 	fs := afero.NewOsFs()

@@ -77,10 +254,10 @@

 	assert.NoError(fs.Mkdir(filepath.Join(d, "f3t"), 0755))

 	assert.NoError(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755))

-	rfs, err := NewRootMappingFs(fs, "bf1", filepath.Join(d, "f1t"), "cf2", filepath.Join(d, "f2t"), "af3", filepath.Join(d, "f3t"))

+	rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t"))

 	assert.NoError(err)

-	fif, err := rfs.Stat(filepath.Join("cf2", testfile))

+	fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))

 	assert.NoError(err)

 	assert.Equal("myfile.txt", fif.Name())

--- /dev/null

+++ b/hugofs/slice_fs.go

@@ -1,0 +1,293 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"os"

+	"syscall"

+	"time"

+

+	"github.com/pkg/errors"

+

+	"github.com/spf13/afero"

+)

+

+var (

+	_ afero.Fs      = (*SliceFs)(nil)

+	_ afero.Lstater = (*SliceFs)(nil)

+	_ afero.File    = (*sliceDir)(nil)

+)

+

+func NewSliceFs(dirs ...FileMetaInfo) (afero.Fs, error) {

+	if len(dirs) == 0 {

+		return NoOpFs, nil

+	}

+

+	for _, dir := range dirs {

+		if !dir.IsDir() {

+			return nil, errors.New("this fs supports directories only")

+		}

+	}

+

+	fs := &SliceFs{

+		dirs: dirs,

+	}

+

+	return fs, nil

+

+}

+

+// SliceFs is an ordered composite filesystem.

+type SliceFs struct {

+	dirs []FileMetaInfo

+}

+

+func (fs *SliceFs) Chmod(n string, m os.FileMode) error {

+	return syscall.EPERM

+}

+

+func (fs *SliceFs) Chtimes(n string, a, m time.Time) error {

+	return syscall.EPERM

+}

+

+func (fs *SliceFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {

+	fi, _, err := fs.pickFirst(name)

+

+	if err != nil {

+		return nil, false, err

+	}

+

+	if fi.IsDir() {

+		return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil

+	}

+

+	return nil, false, errors.Errorf("lstat: files not supported: %q", name)

+

+}

+

+func (fs *SliceFs) Mkdir(n string, p os.FileMode) error {

+	return syscall.EPERM

+}

+

+func (fs *SliceFs) MkdirAll(n string, p os.FileMode) error {

+	return syscall.EPERM

+}

+

+func (fs *SliceFs) Name() string {

+	return "SliceFs"

+}

+

+func (fs *SliceFs) Open(name string) (afero.File, error) {

+	fi, idx, err := fs.pickFirst(name)

+	if err != nil {

+		return nil, err

+	}

+

+	if !fi.IsDir() {

+		panic("currently only dirs in here")

+	}

+

+	return &sliceDir{

+		lfs:     fs,

+		idx:     idx,

+		dirname: name,

+	}, nil

+

+}

+

+func (fs *SliceFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {

+	panic("not implemented")

+}

+

+func (fs *SliceFs) ReadDir(name string) ([]os.FileInfo, error) {

+	panic("not implemented")

+}

+

+func (fs *SliceFs) Remove(n string) error {

+	return syscall.EPERM

+}

+

+func (fs *SliceFs) RemoveAll(p string) error {

+	return syscall.EPERM

+}

+

+func (fs *SliceFs) Rename(o, n string) error {

+	return syscall.EPERM

+}

+

+func (fs *SliceFs) Stat(name string) (os.FileInfo, error) {

+	fi, _, err := fs.LstatIfPossible(name)

+	return fi, err

+}

+

+func (fs *SliceFs) Create(n string) (afero.File, error) {

+	return nil, syscall.EPERM

+}

+

+func (fs *SliceFs) getOpener(name string) func() (afero.File, error) {

+	return func() (afero.File, error) {

+		return fs.Open(name)

+	}

+}

+

+func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) {

+	for i, mfs := range fs.dirs {

+		meta := mfs.Meta()

+		fs := meta.Fs()

+		fi, _, err := lstatIfPossible(fs, name)

+		if err == nil {

+			// Gotta match!

+			return fi, i, nil

+		}

+

+		if !os.IsNotExist(err) {

+			// Real error

+			return nil, -1, err

+		}

+	}

+

+	// Not found

+	return nil, -1, os.ErrNotExist

+}

+

+func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, error) {

+	collect := func(lfs FileMeta) ([]os.FileInfo, error) {

+		d, err := lfs.Fs().Open(name)

+		if err != nil {

+			if !os.IsNotExist(err) {

+				return nil, err

+			}

+			return nil, nil

+		} else {

+			defer d.Close()

+			dirs, err := d.Readdir(-1)

+			if err != nil {

+				return nil, err

+			}

+			return dirs, nil

+		}

+	}

+

+	var dirs []os.FileInfo

+

+	for i := startIdx; i < len(fs.dirs); i++ {

+		mfs := fs.dirs[i]

+

+		fis, err := collect(mfs.Meta())

+		if err != nil {

+			return nil, err

+		}

+

+		dirs = append(dirs, fis...)

+

+	}

+

+	seen := make(map[string]bool)

+	var duplicates []int

+	for i, fi := range dirs {

+		if !fi.IsDir() {

+			continue

+		}

+

+		if seen[fi.Name()] {

+			duplicates = append(duplicates, i)

+		} else {

+			// Make sure it's opened by this filesystem.

+			dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil)

+			seen[fi.Name()] = true

+		}

+	}

+

+	// Remove duplicate directories, keep first.

+	if len(duplicates) > 0 {

+		for i := len(duplicates) - 1; i >= 0; i-- {

+			idx := duplicates[i]

+			dirs = append(dirs[:idx], dirs[idx+1:]...)

+		}

+	}

+

+	if count > 0 && len(dirs) >= count {

+		return dirs[:count], nil

+	}

+

+	return dirs, nil

+

+}

+

+type sliceDir struct {

+	lfs     *SliceFs

+	idx     int

+	dirname string

+}

+

+func (f *sliceDir) Close() error {

+	return nil

+}

+

+func (f *sliceDir) Name() string {

+	return f.dirname

+}

+

+func (f *sliceDir) Read(p []byte) (n int, err error) {

+	panic("not implemented")

+}

+

+func (f *sliceDir) ReadAt(p []byte, off int64) (n int, err error) {

+	panic("not implemented")

+}

+

+func (f *sliceDir) Readdir(count int) ([]os.FileInfo, error) {

+	return f.lfs.readDirs(f.dirname, f.idx, count)

+}

+

+func (f *sliceDir) Readdirnames(count int) ([]string, error) {

+	dirsi, err := f.Readdir(count)

+	if err != nil {

+		return nil, err

+	}

+

+	dirs := make([]string, len(dirsi))

+	for i, d := range dirsi {

+		dirs[i] = d.Name()

+	}

+	return dirs, nil

+}

+

+func (f *sliceDir) Seek(offset int64, whence int) (int64, error) {

+	panic("not implemented")

+}

+

+func (f *sliceDir) Stat() (os.FileInfo, error) {

+	panic("not implemented")

+}

+

+func (f *sliceDir) Sync() error {

+	panic("not implemented")

+}

+

+func (f *sliceDir) Truncate(size int64) error {

+	panic("not implemented")

+}

+

+func (f *sliceDir) Write(p []byte) (n int, err error) {

+	panic("not implemented")

+}

+

+func (f *sliceDir) WriteAt(p []byte, off int64) (n int, err error) {

+	panic("not implemented")

+}

+

+func (f *sliceDir) WriteString(s string) (ret int, err error) {

+	panic("not implemented")

+}

--- /dev/null

+++ b/hugofs/walk.go

@@ -1,0 +1,308 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"fmt"

+	"os"

+	"path/filepath"

+	"sort"

+	"strings"

+

+	"github.com/gohugoio/hugo/common/loggers"

+

+	"github.com/pkg/errors"

+

+	"github.com/spf13/afero"

+)

+

+type (

+	WalkFunc func(path string, info FileMetaInfo, err error) error

+	WalkHook func(dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error)

+)

+

+type Walkway struct {

+	fs       afero.Fs

+	root     string

+	basePath string

+

+	logger *loggers.Logger

+

+	// May be pre-set

+	fi         FileMetaInfo

+	dirEntries []FileMetaInfo

+

+	walkFn WalkFunc

+	walked bool

+

+	// We may traverse symbolic links and bite ourself.

+	seen map[string]bool

+

+	// Optional hooks

+	hookPre  WalkHook

+	hookPost WalkHook

+}

+

+type WalkwayConfig struct {

+	Fs       afero.Fs

+	Root     string

+	BasePath string

+

+	Logger *loggers.Logger

+

+	// One or both of these may be pre-set.

+	Info       FileMetaInfo

+	DirEntries []FileMetaInfo

+

+	WalkFn   WalkFunc

+	HookPre  WalkHook

+	HookPost WalkHook

+}

+

+func NewWalkway(cfg WalkwayConfig) *Walkway {

+	var fs afero.Fs

+	if cfg.Info != nil {

+		fs = cfg.Info.Meta().Fs()

+	} else {

+		fs = cfg.Fs

+	}

+

+	basePath := cfg.BasePath

+	if basePath != "" && !strings.HasSuffix(basePath, filepathSeparator) {

+		basePath += filepathSeparator

+	}

+

+	logger := cfg.Logger

+	if logger == nil {

+		logger = loggers.NewWarningLogger()

+	}

+

+	return &Walkway{

+		fs:         fs,

+		root:       cfg.Root,

+		basePath:   basePath,

+		fi:         cfg.Info,

+		dirEntries: cfg.DirEntries,

+		walkFn:     cfg.WalkFn,

+		hookPre:    cfg.HookPre,

+		hookPost:   cfg.HookPost,

+		logger:     logger,

+		seen:       make(map[string]bool)}

+}

+

+func (w *Walkway) Walk() error {

+	if w.walked {

+		panic("this walkway is already walked")

+	}

+	w.walked = true

+

+	if w.fs == NoOpFs {

+		return nil

+	}

+

+	var fi FileMetaInfo

+	if w.fi != nil {

+		fi = w.fi

+	} else {

+		info, _, err := lstatIfPossible(w.fs, w.root)

+		if err != nil {

+			if os.IsNotExist(err) {

+				return nil

+			}

+

+			if err == ErrPermissionSymlink {

+				w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", w.root)

+				return nil

+			}

+

+			return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root))

+		}

+		fi = info.(FileMetaInfo)

+	}

+

+	if !fi.IsDir() {

+		return w.walkFn(w.root, nil, errors.New("file to walk must be a directory"))

+	}

+

+	return w.walk(w.root, fi, w.dirEntries, w.walkFn)

+

+}

+

+// if the filesystem supports it, use Lstat, else use fs.Stat

+func lstatIfPossible(fs afero.Fs, path string) (os.FileInfo, bool, error) {

+	if lfs, ok := fs.(afero.Lstater); ok {

+		fi, b, err := lfs.LstatIfPossible(path)

+		return fi, b, err

+	}

+	fi, err := fs.Stat(path)

+	return fi, false, err

+}

+

+// walk recursively descends path, calling walkFn.

+// It follow symlinks if supported by the filesystem, but only the same path once.

+func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo, walkFn WalkFunc) error {

+	err := walkFn(path, info, nil)

+	if err != nil {

+		if info.IsDir() && err == filepath.SkipDir {

+			return nil

+		}

+		return err

+	}

+	if !info.IsDir() {

+		return nil

+	}

+

+	meta := info.Meta()

+	filename := meta.Filename()

+

+	if dirEntries == nil {

+		f, err := w.fs.Open(path)

+

+		if err != nil {

+			return walkFn(path, info, errors.Wrapf(err, "walk: open %q (%q)", path, w.root))

+		}

+

+		fis, err := f.Readdir(-1)

+		f.Close()

+		if err != nil {

+			if err == ErrPermissionSymlink {

+				w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", filename)

+				return nil

+			}

+			return walkFn(path, info, errors.Wrap(err, "walk: Readdir"))

+		}

+

+		dirEntries = fileInfosToFileMetaInfos(fis)

+

+		if !meta.IsOrdered() {

+			sort.Slice(dirEntries, func(i, j int) bool {

+				fii := dirEntries[i]

+				fij := dirEntries[j]

+

+				fim, fjm := fii.Meta(), fij.Meta()

+

+				// Pull bundle headers to the top.

+				ficlass, fjclass := fim.Classifier(), fjm.Classifier()

+				if ficlass != fjclass {

+					return ficlass < fjclass

+				}

+

+				// With multiple content dirs with different languages,

+				// there can be duplicate files, and a weight will be added

+				// to the closest one.

+				fiw, fjw := fim.Weight(), fjm.Weight()

+				if fiw != fjw {

+					return fiw > fjw

+				}

+

+				// Explicit order set.

+				fio, fjo := fim.Ordinal(), fjm.Ordinal()

+				if fio != fjo {

+					return fio < fjo

+				}

+

+				// When we walk into a symlink, we keep the reference to

+				// the original name.

+				fin, fjn := fim.Name(), fjm.Name()

+				if fin != "" && fjn != "" {

+					return fin < fjn

+				}

+

+				return fii.Name() < fij.Name()

+			})

+		}

+	}

+

+	// First add some metadata to the dir entries

+	for _, fi := range dirEntries {

+		fim := fi.(FileMetaInfo)

+

+		meta := fim.Meta()

+

+		// Note that we use the original Name even if it's a symlink.

+		name := meta.Name()

+		if name == "" {

+			name = fim.Name()

+		}

+

+		if name == "" {

+			panic(fmt.Sprintf("[%s] no name set in %v", path, meta))

+		}

+		pathn := filepath.Join(path, name)

+

+		pathMeta := pathn

+		if w.basePath != "" {

+			pathMeta = strings.TrimPrefix(pathn, w.basePath)

+		}

+

+		meta[metaKeyPath] = normalizeFilename(pathMeta)

+		meta[metaKeyPathWalk] = pathn

+

+		if fim.IsDir() && w.isSeen(meta.Filename()) {

+			// Prevent infinite recursion

+			// Possible cyclic reference

+			meta[metaKeySkipDir] = true

+		}

+	}

+

+	if w.hookPre != nil {

+		dirEntries, err = w.hookPre(info, path, dirEntries)

+		if err != nil {

+			if err == filepath.SkipDir {

+				return nil

+			}

+			return err

+		}

+	}

+

+	for _, fi := range dirEntries {

+		fim := fi.(FileMetaInfo)

+		meta := fim.Meta()

+

+		if meta.SkipDir() {

+			continue

+		}

+

+		err := w.walk(meta.GetString(metaKeyPathWalk), fim, nil, walkFn)

+		if err != nil {

+			if !fi.IsDir() || err != filepath.SkipDir {

+				return err

+			}

+		}

+	}

+

+	if w.hookPost != nil {

+		dirEntries, err = w.hookPost(info, path, dirEntries)

+		if err != nil {

+			if err == filepath.SkipDir {

+				return nil

+			}

+			return err

+		}

+	}

+	return nil

+}

+

+func (w *Walkway) isSeen(filename string) bool {

+	if filename == "" {

+		return false

+	}

+

+	if w.seen[filename] {

+		return true

+	}

+

+	w.seen[filename] = true

+	return false

+}

--- /dev/null

+++ b/hugofs/walk_test.go

@@ -1,0 +1,225 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugofs

+

+import (

+	"fmt"

+	"os"

+	"path/filepath"

+	"runtime"

+	"strings"

+	"testing"

+

+	"github.com/gohugoio/hugo/common/hugo"

+

+	"github.com/pkg/errors"

+

+	"github.com/gohugoio/hugo/htesting"

+

+	"github.com/spf13/afero"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestWalk(t *testing.T) {

+	assert := require.New(t)

+

+	fs := NewBaseFileDecorator(afero.NewMemMapFs())

+

+	afero.WriteFile(fs, "b.txt", []byte("content"), 0777)

+	afero.WriteFile(fs, "c.txt", []byte("content"), 0777)

+	afero.WriteFile(fs, "a.txt", []byte("content"), 0777)

+

+	names, err := collectFilenames(fs, "", "")

+

+	assert.NoError(err)

+	assert.Equal([]string{"a.txt", "b.txt", "c.txt"}, names)

+}

+

+func TestWalkRootMappingFs(t *testing.T) {

+	assert := require.New(t)

+	fs := NewBaseFileDecorator(afero.NewMemMapFs())

+

+	testfile := "test.txt"

+

+	assert.NoError(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755))

+	assert.NoError(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755))

+

+	rm := []RootMapping{

+		RootMapping{

+			From: "static/b",

+			To:   "e/f",

+		},

+		RootMapping{

+			From: "static/a",

+			To:   "c/d",

+		},

+

+		RootMapping{

+			From: "static/c",

+			To:   "a/b",

+		},

+	}

+

+	rfs, err := NewRootMappingFs(fs, rm...)

+	assert.NoError(err)

+	bfs := afero.NewBasePathFs(rfs, "static")

+

+	names, err := collectFilenames(bfs, "", "")

+

+	assert.NoError(err)

+	assert.Equal([]string{"a/test.txt", "b/test.txt", "c/test.txt"}, names)

+

+}

+

+func skipSymlink() bool {

+	return runtime.GOOS == "windows" && os.Getenv("CI") == ""

+}

+

+func TestWalkSymbolicLink(t *testing.T) {

+	if skipSymlink() {

+		t.Skip("Skip; os.Symlink needs administrator rights on Windows")

+	}

+	assert := require.New(t)

+	workDir, clean, err := htesting.CreateTempDir(Os, "hugo-walk-sym")

+	assert.NoError(err)

+	defer clean()

+	wd, _ := os.Getwd()

+	defer func() {

+		os.Chdir(wd)

+	}()

+

+	fs := NewBaseFileDecorator(Os)

+

+	blogDir := filepath.Join(workDir, "blog")

+	docsDir := filepath.Join(workDir, "docs")

+	blogReal := filepath.Join(blogDir, "real")

+	blogRealSub := filepath.Join(blogReal, "sub")

+	assert.NoError(os.MkdirAll(blogRealSub, 0777))

+	assert.NoError(os.MkdirAll(docsDir, 0777))

+	afero.WriteFile(fs, filepath.Join(blogRealSub, "a.txt"), []byte("content"), 0777)

+	afero.WriteFile(fs, filepath.Join(docsDir, "b.txt"), []byte("content"), 0777)

+

+	os.Chdir(blogDir)

+	assert.NoError(os.Symlink("real", "symlinked"))

+	os.Chdir(blogReal)

+	assert.NoError(os.Symlink("../real", "cyclic"))

+	os.Chdir(docsDir)

+	assert.NoError(os.Symlink("../blog/real/cyclic", "docsreal"))

+

+	t.Run("OS Fs", func(t *testing.T) {

+		assert := require.New(t)

+

+		names, err := collectFilenames(fs, workDir, workDir)

+		assert.NoError(err)

+

+		assert.Equal([]string{"blog/real/sub/a.txt", "docs/b.txt"}, names)

+	})

+

+	t.Run("BasePath Fs", func(t *testing.T) {

+		if hugo.GoMinorVersion() < 12 {

+			// https://github.com/golang/go/issues/30520

+			// This is fixed in Go 1.13 and in the latest Go 1.12

+			t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib")

+

+		}

+		assert := require.New(t)

+

+		docsFs := afero.NewBasePathFs(fs, docsDir)

+

+		names, err := collectFilenames(docsFs, "", "")

+		assert.NoError(err)

+

+		// Note: the docsreal folder is considered cyclic when walking from the root, but this works.

+		assert.Equal([]string{"b.txt", "docsreal/sub/a.txt"}, names)

+	})

+

+}

+

+func collectFilenames(fs afero.Fs, base, root string) ([]string, error) {

+	var names []string

+

+	walkFn := func(path string, info FileMetaInfo, err error) error {

+		if err != nil {

+			return err

+		}

+

+		if info.IsDir() {

+			return nil

+		}

+

+		filename := info.Meta().Path()

+		filename = filepath.ToSlash(filename)

+

+		names = append(names, filename)

+

+		return nil

+	}

+

+	w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn})

+

+	err := w.Walk()

+

+	return names, err

+

+}

+

+func BenchmarkWalk(b *testing.B) {

+	assert := require.New(b)

+	fs := NewBaseFileDecorator(afero.NewMemMapFs())

+

+	writeFiles := func(dir string, numfiles int) {

+		for i := 0; i < numfiles; i++ {

+			filename := filepath.Join(dir, fmt.Sprintf("file%d.txt", i))

+			assert.NoError(afero.WriteFile(fs, filename, []byte("content"), 0777))

+		}

+	}

+

+	const numFilesPerDir = 20

+

+	writeFiles("root", numFilesPerDir)

+	writeFiles("root/l1_1", numFilesPerDir)

+	writeFiles("root/l1_1/l2_1", numFilesPerDir)

+	writeFiles("root/l1_1/l2_2", numFilesPerDir)

+	writeFiles("root/l1_2", numFilesPerDir)

+	writeFiles("root/l1_2/l2_1", numFilesPerDir)

+	writeFiles("root/l1_3", numFilesPerDir)

+

+	walkFn := func(path string, info FileMetaInfo, err error) error {

+		if err != nil {

+			return err

+		}

+		if info.IsDir() {

+			return nil

+		}

+

+		filename := info.Meta().Filename()

+		if !strings.HasPrefix(filename, "root") {

+			return errors.New(filename)

+		}

+

+		return nil

+	}

+

+	b.ResetTimer()

+	for i := 0; i < b.N; i++ {

+		w := NewWalkway(WalkwayConfig{Fs: fs, Root: "root", WalkFn: walkFn})

+

+		if err := w.Walk(); err != nil {

+			b.Fatal(err)

+		}

+	}

+

+}

--- a/hugolib/config.go

+++ b/hugolib/config.go

@@ -14,22 +14,25 @@

 package hugolib

 import (

-	"fmt"

-

 	"os"

 	"path/filepath"

 	"strings"

+	"github.com/gohugoio/hugo/common/loggers"

+

+	"github.com/gohugoio/hugo/cache/filecache"

+

+	"github.com/gohugoio/hugo/common/maps"

+

 	"github.com/gohugoio/hugo/parser/metadecoders"

 	"github.com/gohugoio/hugo/common/herrors"

 	"github.com/gohugoio/hugo/common/hugo"

 	"github.com/gohugoio/hugo/hugolib/paths"

+	"github.com/gohugoio/hugo/langs"

+	"github.com/gohugoio/hugo/modules"

 	"github.com/pkg/errors"

-	_errors "github.com/pkg/errors"

-	"github.com/gohugoio/hugo/langs"

-

 	"github.com/gohugoio/hugo/config"

 	"github.com/gohugoio/hugo/config/privacy"

 	"github.com/gohugoio/hugo/config/services"

@@ -67,7 +70,8 @@

 // ConfigSourceDescriptor describes where to find the config (e.g. config.toml etc.).

 type ConfigSourceDescriptor struct {

-	Fs afero.Fs

+	Fs     afero.Fs

+	Logger *loggers.Logger

 	// Path to the config file to use, e.g. /my/project/config.toml

 	Filename string

@@ -84,6 +88,9 @@

 	// production, development

 	Environment string

+

+	// Defaults to os.Environ if not set.

+	Environ []string

 }

 func (d ConfigSourceDescriptor) configFilenames() []string {

@@ -111,26 +118,28 @@

 // LoadConfig loads Hugo configuration into a new Viper and then adds

 // a set of defaults.

 func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (*viper.Viper, []string, error) {

+

 	if d.Environment == "" {

 		d.Environment = hugo.EnvironmentProduction

 	}

+	if len(d.Environ) == 0 {

+		d.Environ = os.Environ()

+	}

+

 	var configFiles []string

 	v := viper.New()

 	l := configLoader{ConfigSourceDescriptor: d}

-	v.AutomaticEnv()

-	v.SetEnvPrefix("hugo")

-

-	var cerr error

-

 	for _, name := range d.configFilenames() {

 		var filename string

-		if filename, cerr = l.loadConfig(name, v); cerr != nil && cerr != ErrNoConfigFile {

-			return nil, nil, cerr

+		filename, err := l.loadConfig(name, v)

+		if err == nil {

+			configFiles = append(configFiles, filename)

+		} else if err != ErrNoConfigFile {

+			return nil, nil, err

 		}

-		configFiles = append(configFiles, filename)

 	}

 	if d.AbsConfigDir != "" {

@@ -137,8 +146,9 @@

 		dirnames, err := l.loadConfigFromConfigDir(v)

 		if err == nil {

 			configFiles = append(configFiles, dirnames...)

+		} else if err != ErrNoConfigFile {

+			return nil, nil, err

 		}

-		cerr = err

 	}

 	if err := loadDefaultSettingsFor(v); err != nil {

@@ -145,17 +155,6 @@

 		return v, configFiles, err

 	}

-	if cerr == nil {

-		themeConfigFiles, err := l.loadThemeConfig(v)

-		if err != nil {

-			return v, configFiles, err

-		}

-

-		if len(themeConfigFiles) > 0 {

-			configFiles = append(configFiles, themeConfigFiles...)

-		}

-	}

-

 	// We create languages based on the settings, so we need to make sure that

 	// all configuration is loaded/set before doing that.

 	for _, d := range doWithConfig {

@@ -164,14 +163,77 @@

 		}

 	}

+	// Apply environment overrides

+	if len(d.Environ) > 0 {

+		// Extract all that start with the HUGO_ prefix

+		const hugoEnvPrefix = "HUGO_"

+		var hugoEnv []string

+		for _, v := range d.Environ {

+			key, val := config.SplitEnvVar(v)

+			if strings.HasPrefix(key, hugoEnvPrefix) {

+				hugoEnv = append(hugoEnv, strings.ToLower(strings.TrimPrefix(key, hugoEnvPrefix)), val)

+			}

+		}

+

+		if len(hugoEnv) > 0 {

+			for i := 0; i < len(hugoEnv); i += 2 {

+				key, valStr := strings.ToLower(hugoEnv[i]), hugoEnv[i+1]

+

+				existing, nestedKey, owner, err := maps.GetNestedParamFn(key, "_", v.Get)

+				if err != nil {

+					return v, configFiles, err

+				}

+

+				if existing != nil {

+					val, err := metadecoders.Default.UnmarshalStringTo(valStr, existing)

+					if err != nil {

+						continue

+					}

+

+					if owner != nil {

+						owner[nestedKey] = val

+					} else {

+						v.Set(key, val)

+					}

+				} else {

+					v.Set(key, valStr)

+				}

+			}

+		}

+	}

+

+	modulesConfig, err := l.loadModulesConfig(v)

+	if err != nil {

+		return v, configFiles, err

+	}

+

+	mods, modulesConfigFiles, err := l.collectModules(modulesConfig, v)

+	if err != nil {

+		return v, configFiles, err

+	}

+

 	if err := loadLanguageSettings(v, nil); err != nil {

 		return v, configFiles, err

 	}

-	return v, configFiles, cerr

+	// Apply default project mounts.

+	if err := modules.ApplyProjectConfigDefaults(v, mods[len(mods)-1]); err != nil {

+		return v, configFiles, err

+	}

+	if len(modulesConfigFiles) > 0 {

+		configFiles = append(configFiles, modulesConfigFiles...)

+	}

+

+	return v, configFiles, nil

+

 }

+func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error {

+	_, err := langs.LoadLanguageSettings(cfg, oldLangs)

+	return err

+}

+

 type configLoader struct {

 	ConfigSourceDescriptor

 }

@@ -334,145 +396,79 @@

 	return dirnames, nil

 }

-func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error {

+func (l configLoader) loadModulesConfig(v1 *viper.Viper) (modules.Config, error) {

-	defaultLang := cfg.GetString("defaultContentLanguage")

-

-	var languages map[string]interface{}

-

-	languagesFromConfig := cfg.GetStringMap("languages")

-	disableLanguages := cfg.GetStringSlice("disableLanguages")

-

-	if len(disableLanguages) == 0 {

-		languages = languagesFromConfig

-	} else {

-		languages = make(map[string]interface{})

-		for k, v := range languagesFromConfig {

-			for _, disabled := range disableLanguages {

-				if disabled == defaultLang {

-					return fmt.Errorf("cannot disable default language %q", defaultLang)

-				}

-

-				if strings.EqualFold(k, disabled) {

-					v.(map[string]interface{})["disabled"] = true

-					break

-				}

-			}

-			languages[k] = v

-		}

+	modConfig, err := modules.DecodeConfig(v1)

+	if err != nil {

+		return modules.Config{}, err

 	}

-	var (

-		languages2 langs.Languages

-		err        error

-	)

+	return modConfig, nil

+}

-	if len(languages) == 0 {

-		languages2 = append(languages2, langs.NewDefaultLanguage(cfg))

-	} else {

-		languages2, err = toSortedLanguages(cfg, languages)

-		if err != nil {

-			return _errors.Wrap(err, "Failed to parse multilingual config")

-		}

+func (l configLoader) collectModules(modConfig modules.Config, v1 *viper.Viper) (modules.Modules, []string, error) {

+	workingDir := l.WorkingDir

+	if workingDir == "" {

+		workingDir = v1.GetString("workingDir")

 	}

-	if oldLangs != nil {

-		// When in multihost mode, the languages are mapped to a server, so

-		// some structural language changes will need a restart of the dev server.

-		// The validation below isn't complete, but should cover the most

-		// important cases.

-		var invalid bool

-		if languages2.IsMultihost() != oldLangs.IsMultihost() {

-			invalid = true

-		} else {

-			if languages2.IsMultihost() && len(languages2) != len(oldLangs) {

-				invalid = true

-			}

-		}

+	themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir"))

-		if invalid {

-			return errors.New("language change needing a server restart detected")

-		}

+	ignoreVendor := v1.GetBool("ignoreVendor")

-		if languages2.IsMultihost() {

-			// We need to transfer any server baseURL to the new language

-			for i, ol := range oldLangs {

-				nl := languages2[i]

-				nl.Set("baseURL", ol.GetString("baseURL"))

-			}

-		}

+	filecacheConfigs, err := filecache.DecodeConfig(l.Fs, v1)

+	if err != nil {

+		return nil, nil, err

 	}

+	v1.Set("filecacheConfigs", filecacheConfigs)

-	// The defaultContentLanguage is something the user has to decide, but it needs

-	// to match a language in the language definition list.

-	langExists := false

-	for _, lang := range languages2 {

-		if lang.Lang == defaultLang {

-			langExists = true

-			break

-		}

-	}

+	modulesClient := modules.NewClient(modules.ClientConfig{

+		Fs:           l.Fs,

+		Logger:       l.Logger,

+		WorkingDir:   workingDir,

+		ThemesDir:    themesDir,

+		CacheDir:     filecacheConfigs.CacheDirModules(),

+		ModuleConfig: modConfig,

+		IgnoreVendor: ignoreVendor,

+	})

-	if !langExists {

-		return fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang)

-	}

+	v1.Set("modulesClient", modulesClient)

-	cfg.Set("languagesSorted", languages2)

-	cfg.Set("multilingual", len(languages2) > 1)

-

-	multihost := languages2.IsMultihost()

-

-	if multihost {

-		cfg.Set("defaultContentLanguageInSubdir", true)

-		cfg.Set("multihost", true)

+	moduleConfig, err := modulesClient.Collect()

+	if err != nil {

+		return nil, nil, err

 	}

-	if multihost {

-		// The baseURL may be provided at the language level. If that is true,

-		// then every language must have a baseURL. In this case we always render

-		// to a language sub folder, which is then stripped from all the Permalink URLs etc.

-		for _, l := range languages2 {

-			burl := l.GetLocal("baseURL")

-			if burl == nil {

-				return errors.New("baseURL must be set on all or none of the languages")

-			}

-		}

+	// Avoid recreating these later.

+	v1.Set("allModules", moduleConfig.ActiveModules)

+	if len(moduleConfig.ActiveModules) == 0 {

+		return nil, nil, nil

 	}

-	return nil

-}

-

-func (l configLoader) loadThemeConfig(v1 *viper.Viper) ([]string, error) {

-	themesDir := paths.AbsPathify(l.WorkingDir, v1.GetString("themesDir"))

-	themes := config.GetStringSlicePreserveString(v1, "theme")

-

-	themeConfigs, err := paths.CollectThemes(l.Fs, themesDir, themes)

-	if err != nil {

-		return nil, err

-	}

-

-	if len(themeConfigs) == 0 {

-		return nil, nil

-	}

-

-	v1.Set("allThemes", themeConfigs)

-

 	var configFilenames []string

-	for _, tc := range themeConfigs {

-		if tc.ConfigFilename != "" {

-			configFilenames = append(configFilenames, tc.ConfigFilename)

+	for _, tc := range moduleConfig.ActiveModules {

+		if tc.ConfigFilename() != "" {

+			if tc.Watch() {

+				configFilenames = append(configFilenames, tc.ConfigFilename())

+			}

 			if err := l.applyThemeConfig(v1, tc); err != nil {

-				return nil, err

+				return nil, nil, err

 			}

 		}

 	}

-	return configFilenames, nil

+	if moduleConfig.GoModulesFilename != "" {

+		// We want to watch this for changes and trigger rebuild on version

+		// changes etc.

+		configFilenames = append(configFilenames, moduleConfig.GoModulesFilename)

+	}

+	return moduleConfig.ActiveModules, configFilenames, nil

+

 }

-func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) error {

+func (l configLoader) applyThemeConfig(v1 *viper.Viper, theme modules.Module) error {

 	const (

 		paramsKey    = "params"

@@ -480,22 +476,12 @@

 		menuKey      = "menus"

 	)

-	v2 := theme.Cfg

+	v2 := theme.Cfg()

 	for _, key := range []string{paramsKey, "outputformats", "mediatypes"} {

 		l.mergeStringMapKeepLeft("", key, v1, v2)

 	}

-	themeLower := strings.ToLower(theme.Name)

-	themeParamsNamespace := paramsKey + "." + themeLower

-

-	// Set namespaced params

-	if v2.IsSet(paramsKey) && !v1.IsSet(themeParamsNamespace) {

-		// Set it in the default store to make sure it gets in the same or

-		// behind the others.

-		v1.SetDefault(themeParamsNamespace, v2.Get(paramsKey))

-	}

-

 	// Only add params and new menu entries, we do not add language definitions.

 	if v1.IsSet(languagesKey) && v2.IsSet(languagesKey) {

 		v1Langs := v1.GetStringMap(languagesKey)

@@ -508,12 +494,6 @@

 			if k == "" {

 				continue

 			}

-			langParamsKey := languagesKey + "." + k + "." + paramsKey

-			langParamsThemeNamespace := langParamsKey + "." + themeLower

-			// Set namespaced params

-			if v2.IsSet(langParamsKey) && !v1.IsSet(langParamsThemeNamespace) {

-				v1.SetDefault(langParamsThemeNamespace, v2.Get(langParamsKey))

-			}

 			langMenuKey := languagesKey + "." + k + "." + menuKey

 			if v2.IsSet(langMenuKey) {

@@ -577,18 +557,23 @@

 	v.RegisterAlias("indexes", "taxonomies")

+	/*

+

+		TODO(bep) from 0.56 these are configured as module mounts.

+			v.SetDefault("contentDir", "content")

+			v.SetDefault("layoutDir", "layouts")

+			v.SetDefault("assetDir", "assets")

+			v.SetDefault("staticDir", "static")

+			v.SetDefault("dataDir", "data")

+			v.SetDefault("i18nDir", "i18n")

+			v.SetDefault("archetypeDir", "archetypes")

+	*/

+

 	v.SetDefault("cleanDestinationDir", false)

 	v.SetDefault("watch", false)

 	v.SetDefault("metaDataFormat", "toml")

-	v.SetDefault("contentDir", "content")

-	v.SetDefault("layoutDir", "layouts")

-	v.SetDefault("assetDir", "assets")

-	v.SetDefault("staticDir", "static")

 	v.SetDefault("resourceDir", "resources")

-	v.SetDefault("archetypeDir", "archetypes")

 	v.SetDefault("publishDir", "public")

-	v.SetDefault("dataDir", "data")

-	v.SetDefault("i18nDir", "i18n")

 	v.SetDefault("themesDir", "themes")

 	v.SetDefault("buildDrafts", false)

 	v.SetDefault("buildFuture", false)

@@ -635,5 +620,6 @@

 	v.SetDefault("disableFastRender", false)

 	v.SetDefault("timeout", 10000) // 10 seconds

 	v.SetDefault("enableInlineShortcodes", false)

+

 	return nil

 }

--- a/hugolib/config_test.go

+++ b/hugolib/config_test.go

@@ -14,6 +14,9 @@

 package hugolib

 import (

+	"bytes"

+	"fmt"

+	"path/filepath"

 	"testing"

 	"github.com/spf13/afero"

@@ -40,10 +43,7 @@

 	require.NoError(t, err)

 	assert.Equal("side", cfg.GetString("paginatePath"))

-	// default

-	assert.Equal("layouts", cfg.GetString("layoutDir"))

-	// no themes

-	assert.False(cfg.IsSet("allThemes"))

+

 }

 func TestLoadMultiConfig(t *testing.T) {

@@ -188,11 +188,6 @@

   "p1": "p1 main",

   "p2": "p2 main",

   "p3": "p3 theme",

-  "test-theme": map[string]interface {}{

-    "p1": "p1 theme",

-    "p2": "p2 theme",

-    "p3": "p3 theme",

-  },

   "top": "top",

 }`, got["params"])

@@ -257,10 +252,6 @@

     "params": map[string]interface {}{

       "pl1": "p1-en-main",

       "pl2": "p2-en-theme",

-      "test-theme": map[string]interface {}{

-        "pl1": "p1-en-theme",

-        "pl2": "p2-en-theme",

-      },

     },

   },

   "nb": map[string]interface {}{

@@ -275,11 +266,6 @@

     "params": map[string]interface {}{

       "pl1": "p1-nb-main",

       "pl2": "p2-nb-theme",

-      "test-theme": map[string]interface {}{

-        "pl1": "p1-nb-theme",

-        "pl2": "p2-nb-theme",

-        "top": "top-nb-theme",

-      },

     },

   },

 }

@@ -395,5 +381,144 @@

 	b.Build(BuildCfg{SkipRender: true})

 	assert.True(b.H.Sites[0].Info.Config().Privacy.YouTube.PrivacyEnhanced)

+

+}

+

+func TestLoadConfigModules(t *testing.T) {

+	t.Parallel()

+

+	assert := require.New(t)

+

+	// https://github.com/gohugoio/hugoThemes#themetoml

+

+	const (

+		// Before Hugo 0.56 each theme/component could have its own theme.toml

+		// with some settings, mostly used on the Hugo themes site.

+		// To preserve combability we read these files into the new "modules"

+		// section in config.toml.

+		o1t = `

+name = "Component o1"

+license = "MIT"

+min_version = 0.38

+`

+		// This is the component's config.toml, using the old theme syntax.

+		o1c = `

+theme = ["n2"]

+`

+

+		n1 = `

+title = "Component n1"

+

+[module]

+description = "Component n1 description"

+[module.hugoVersion]

+min = "0.40.0"

+max = "0.50.0"

+extended = true

+[[module.imports]]

+path="o1"

+[[module.imports]]

+path="n3"

+

+

+`

+

+		n2 = `

+title = "Component n2"

+`

+

+		n3 = `

+title = "Component n3"

+`

+

+		n4 = `

+title = "Component n4"

+`

+	)

+

+	b := newTestSitesBuilder(t)

+

+	writeThemeFiles := func(name, configTOML, themeTOML string) {

+		b.WithSourceFile(filepath.Join("themes", name, "data", "module.toml"), fmt.Sprintf("name=%q", name))

+		if configTOML != "" {

+			b.WithSourceFile(filepath.Join("themes", name, "config.toml"), configTOML)

+		}

+		if themeTOML != "" {

+			b.WithSourceFile(filepath.Join("themes", name, "theme.toml"), themeTOML)

+		}

+	}

+

+	writeThemeFiles("n1", n1, "")

+	writeThemeFiles("n2", n2, "")

+	writeThemeFiles("n3", n3, "")

+	writeThemeFiles("n4", n4, "")

+	writeThemeFiles("o1", o1c, o1t)

+

+	b.WithConfigFile("toml", `

+[module]

+[[module.imports]]

+path="n1"

+[[module.imports]]

+path="n4"

+

+`)

+

+	b.Build(BuildCfg{})

+

+	modulesClient := b.H.Paths.ModulesClient

+	var graphb bytes.Buffer

+	modulesClient.Graph(&graphb)

+

+	assert.Equal(`project n1

+n1 o1

+o1 n2

+n1 n3

+project n4

+`, graphb.String())

+

+}

+

+func TestLoadConfigWithOsEnvOverrides(t *testing.T) {

+

+	assert := require.New(t)

+

+	baseConfig := `

+

+environment = "production"

+enableGitInfo = true

+intSlice = [5,7,9]

+floatSlice = [3.14, 5.19]

+stringSlice = ["a", "b"]

+

+[imaging]

+anchor = "smart"

+quality = 75 

+resamplefilter = "CatmullRom"

+`

+

+	b := newTestSitesBuilder(t).WithConfigFile("toml", baseConfig)

+

+	b.WithEnviron(

+		"HUGO_ENVIRONMENT", "test",

+		"HUGO_NEW", "new", // key not in config.toml

+		"HUGO_ENABLEGITINFO", "false",

+		"HUGO_IMAGING_ANCHOR", "top",

+		"HUGO_STRINGSLICE", `["c", "d"]`,

+		"HUGO_INTSLICE", `[5, 8, 9]`,

+		"HUGO_FLOATSLICE", `[5.32]`,

+	)

+

+	b.Build(BuildCfg{})

+

+	cfg := b.H.Cfg

+

+	assert.Equal("test", cfg.Get("environment"))

+	assert.Equal(false, cfg.GetBool("enablegitinfo"))

+	assert.Equal("new", cfg.Get("new"))

+	assert.Equal("top", cfg.Get("imaging.anchor"))

+	assert.Equal(int64(75), cfg.Get("imaging.quality"))

+	assert.Equal([]interface{}{"c", "d"}, cfg.Get("stringSlice"))

+	assert.Equal([]interface{}{5.32}, cfg.Get("floatSlice"))

+	assert.Equal([]interface{}{5, 8, 9}, cfg.Get("intSlice"))

 }

--- /dev/null

+++ b/hugolib/data/hugo.toml

@@ -1,0 +1,1 @@

+slogan = "Hugo Rocks!"

\ No newline at end of file

--- a/hugolib/disableKinds_test.go

+++ b/hugolib/disableKinds_test.go

@@ -20,11 +20,7 @@

 	"github.com/gohugoio/hugo/resources/page"

-	"github.com/gohugoio/hugo/deps"

-	"github.com/spf13/afero"

-

 	"github.com/gohugoio/hugo/helpers"

-	"github.com/gohugoio/hugo/hugofs"

 	"github.com/stretchr/testify/require"

 )

@@ -80,8 +76,6 @@

 # Doc

 `

-	mf := afero.NewMemMapFs()

-

 	disabledStr := "[]"

 	if len(disabled) > 0 {

@@ -90,40 +84,34 @@

 	}

 	siteConfig := fmt.Sprintf(siteConfigTemplate, disabledStr)

-	writeToFs(t, mf, "config.toml", siteConfig)

-	cfg, err := LoadConfigDefault(mf)

-	require.NoError(t, err)

+	b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)

-	fs := hugofs.NewFrom(mf, cfg)

-	th := testHelper{cfg, fs, t}

+	b.WithTemplates(

+		"index.html", "Home|{{ .Title }}|{{ .Content }}",

+		"_default/single.html", "Single|{{ .Title }}|{{ .Content }}",

+		"_default/list.html", "List|{{ .Title }}|{{ .Content }}",

+		"_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",

+		"layouts/404.html", "Page Not Found",

+	)

-	writeSource(t, fs, "layouts/index.html", "Home|{{ .Title }}|{{ .Content }}")

-	writeSource(t, fs, "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}")

-	writeSource(t, fs, "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}")

-	writeSource(t, fs, "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}")

-	writeSource(t, fs, "layouts/404.html", "Page Not Found")

+	b.WithContent(

+		"sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"),

+		"categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10),

+		"tags/tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10),

+	)

-	writeSource(t, fs, "content/sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"))

+	b.Build(BuildCfg{})

+	h := b.H

-	writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)

-	writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10)

-

-	h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})

-

-	require.NoError(t, err)

 	require.Len(t, h.Sites, 1)

-	err = h.Build(BuildCfg{})

+	assertDisabledKinds(b, h.Sites[0], disabled...)

-	require.NoError(t, err)

-

-	assertDisabledKinds(th, h.Sites[0], disabled...)

-

 }

-func assertDisabledKinds(th testHelper, s *Site, disabled ...string) {

-	assertDisabledKind(th,

+func assertDisabledKinds(b *sitesBuilder, s *Site, disabled ...string) {

+	assertDisabledKind(b,

 		func(isDisabled bool) bool {

 			if isDisabled {

 				return len(s.RegularPages()) == 0

@@ -130,7 +118,7 @@

 			}

 			return len(s.RegularPages()) > 0

 		}, disabled, page.KindPage, "public/sect/p1/index.html", "Single|P1")

-	assertDisabledKind(th,

+	assertDisabledKind(b,

 		func(isDisabled bool) bool {

 			p := s.getPage(page.KindHome)

 			if isDisabled {

@@ -138,7 +126,7 @@

 			}

 			return p != nil

 		}, disabled, page.KindHome, "public/index.html", "Home")

-	assertDisabledKind(th,

+	assertDisabledKind(b,

 		func(isDisabled bool) bool {

 			p := s.getPage(page.KindSection, "sect")

 			if isDisabled {

@@ -146,7 +134,7 @@

 			}

 			return p != nil

 		}, disabled, page.KindSection, "public/sect/index.html", "Sects")

-	assertDisabledKind(th,

+	assertDisabledKind(b,

 		func(isDisabled bool) bool {

 			p := s.getPage(page.KindTaxonomy, "tags", "tag1")

@@ -156,7 +144,7 @@

 			return p != nil

 		}, disabled, page.KindTaxonomy, "public/tags/tag1/index.html", "Tag1")

-	assertDisabledKind(th,

+	assertDisabledKind(b,

 		func(isDisabled bool) bool {

 			p := s.getPage(page.KindTaxonomyTerm, "tags")

 			if isDisabled {

@@ -165,7 +153,7 @@

 			return p != nil

 		}, disabled, page.KindTaxonomyTerm, "public/tags/index.html", "Tags")

-	assertDisabledKind(th,

+	assertDisabledKind(b,

 		func(isDisabled bool) bool {

 			p := s.getPage(page.KindTaxonomyTerm, "categories")

@@ -175,7 +163,7 @@

 			return p != nil

 		}, disabled, page.KindTaxonomyTerm, "public/categories/index.html", "Category Terms")

-	assertDisabledKind(th,

+	assertDisabledKind(b,

 		func(isDisabled bool) bool {

 			p := s.getPage(page.KindTaxonomy, "categories", "hugo")

 			if isDisabled {

@@ -185,15 +173,15 @@

 		}, disabled, page.KindTaxonomy, "public/categories/hugo/index.html", "Hugo")

 	// The below have no page in any collection.

-	assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "<link>")

-	assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap")

-	assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent")

-	assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found")

+	assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "<link>")

+	assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap")

+	assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent")

+	assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found")

 }

-func assertDisabledKind(th testHelper, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) {

+func assertDisabledKind(b *sitesBuilder, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) {

 	isDisabled := stringSliceContains(kind, disabled...)

-	require.True(th.T, kindAssert(isDisabled), fmt.Sprintf("%s: %t", kind, isDisabled))

+	require.True(b.T, kindAssert(isDisabled), fmt.Sprintf("%s: %t", kind, isDisabled))

 	if kind == kindRSS && !isDisabled {

 		// If the home page is also disabled, there is not RSS to look for.

@@ -204,20 +192,11 @@

 	if isDisabled {

 		// Path should not exist

-		fileExists, err := helpers.Exists(path, th.Fs.Destination)

-		require.False(th.T, fileExists)

-		require.NoError(th.T, err)

+		fileExists, err := helpers.Exists(path, b.Fs.Destination)

+		require.False(b.T, fileExists)

+		require.NoError(b.T, err)

 	} else {

-		th.assertFileContent(path, matcher)

+		b.AssertFileContent(path, matcher)

 	}

-}

-

-func stringSliceContains(k string, values ...string) bool {

-	for _, v := range values {

-		if k == v {

-			return true

-		}

-	}

-	return false

 }

--- a/hugolib/fileInfo.go

+++ b/hugolib/fileInfo.go

@@ -16,34 +16,35 @@

 import (

 	"strings"

-	"github.com/gohugoio/hugo/helpers"

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/pkg/errors"

+

+	"github.com/gohugoio/hugo/hugofs"

+

+	"github.com/spf13/afero"

+

 	"github.com/gohugoio/hugo/source"

 )

 // fileInfo implements the File and ReadableFile interface.

 var (

-	_ source.File         = (*fileInfo)(nil)

-	_ source.ReadableFile = (*fileInfo)(nil)

-	_ pathLangFile        = (*fileInfo)(nil)

+	_ source.File = (*fileInfo)(nil)

 )

-// A partial interface to prevent ambigous compiler error.

-type basePather interface {

-	Filename() string

-	RealName() string

-	BaseDir() string

-}

-

 type fileInfo struct {

-	bundleTp bundleDirType

+	source.File

-	source.ReadableFile

-	basePather

-

 	overriddenLang string

+}

-	// Set if the content language for this file is disabled.

-	disabled bool

+func (fi *fileInfo) Open() (afero.File, error) {

+	f, err := fi.FileInfo().Meta().Open()

+	if err != nil {

+		err = errors.Wrap(err, "fileInfo")

+	}

+

+	return f, err

 }

 func (fi *fileInfo) Lang() string {

@@ -50,49 +51,30 @@

 	if fi.overriddenLang != "" {

 		return fi.overriddenLang

 	}

-	return fi.ReadableFile.Lang()

+	return fi.File.Lang()

 }

-func (fi *fileInfo) Filename() string {

-	if fi == nil || fi.basePather == nil {

-		return ""

-	}

-	return fi.basePather.Filename()

-}

-

 func (fi *fileInfo) String() string {

-	if fi == nil || fi.ReadableFile == nil {

+	if fi == nil || fi.File == nil {

 		return ""

 	}

 	return fi.Path()

 }

-func (fi *fileInfo) isOwner() bool {

-	return fi.bundleTp > bundleNot

-}

+// TODO(bep) rename

+func newFileInfo(sp *source.SourceSpec, fi hugofs.FileMetaInfo) (*fileInfo, error) {

-func IsContentFile(filename string) bool {

-	return contentFileExtensionsSet[strings.TrimPrefix(helpers.Ext(filename), ".")]

-}

+	baseFi, err := sp.NewFileInfo(fi)

+	if err != nil {

+		return nil, err

+	}

-func (fi *fileInfo) isContentFile() bool {

-	return contentFileExtensionsSet[fi.Ext()]

-}

-

-func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi pathLangFileFi, tp bundleDirType) *fileInfo {

-

-	baseFi := sp.NewFileInfo(baseDir, filename, tp == bundleLeaf, fi)

 	f := &fileInfo{

-		bundleTp:     tp,

-		ReadableFile: baseFi,

-		basePather:   fi,

+		File: baseFi,

 	}

-	lang := f.Lang()

-	f.disabled = lang != "" && sp.DisabledLanguages[lang]

+	return f, nil

-	return f

-

 }

 type bundleDirType int

@@ -108,7 +90,7 @@

 // Returns the given file's name's bundle type and whether it is a content

 // file or not.

 func classifyBundledFile(name string) (bundleDirType, bool) {

-	if !IsContentFile(name) {

+	if !files.IsContentFile(name) {

 		return bundleNot, false

 	}

 	if strings.HasPrefix(name, "_index.") {

--- a/hugolib/fileInfo_test.go

+++ b/hugolib/fileInfo_test.go

@@ -22,6 +22,7 @@

 func TestFileInfo(t *testing.T) {

 	t.Run("String", func(t *testing.T) {

+		t.Parallel()

 		assert := require.New(t)

 		fi := &fileInfo{}

 		_, err := cast.ToStringE(fi)

--- a/hugolib/filesystems/basefs.go

+++ b/hugolib/filesystems/basefs.go

@@ -16,27 +16,27 @@

 package filesystems

 import (

-	"errors"

+	"io"

 	"os"

+	"path"

 	"path/filepath"

 	"strings"

+	"sync"

-	"github.com/gohugoio/hugo/config"

+	"github.com/gohugoio/hugo/hugofs/files"

+	"github.com/pkg/errors"

+

+	"github.com/gohugoio/hugo/modules"

+

 	"github.com/gohugoio/hugo/hugofs"

 	"fmt"

 	"github.com/gohugoio/hugo/hugolib/paths"

-	"github.com/gohugoio/hugo/langs"

 	"github.com/spf13/afero"

 )

-// When we create a virtual filesystem with data and i18n bundles for the project and the themes,

-// this is the name of the project's virtual root. It got it's funky name to make sure

-// (or very unlikely) that it collides with a theme name.

-const projectVirtualFolder = "__h__project"

-

 var filePathSeparator = string(filepath.Separator)

 // BaseFs contains the core base filesystems used by Hugo. The name "base" is used

@@ -51,16 +51,43 @@

 	// This usually maps to /my-project/public.

 	PublishFs afero.Fs

-	themeFs afero.Fs

+	theBigFs *filesystemsCollector

+}

-	// TODO(bep) improve the "theme interaction"

-	AbsThemeDirs []string

+func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo {

+	var dirs []hugofs.FileMetaInfo

+	for _, dir := range fs.AllDirs() {

+		if dir.Meta().Watch() {

+			dirs = append(dirs, dir)

+		}

+	}

+

+	return dirs

 }

+func (fs *BaseFs) AllDirs() []hugofs.FileMetaInfo {

+	var dirs []hugofs.FileMetaInfo

+	for _, dirSet := range [][]hugofs.FileMetaInfo{

+		fs.Archetypes.Dirs,

+		fs.I18n.Dirs,

+		fs.Data.Dirs,

+		fs.Content.Dirs,

+		fs.Assets.Dirs,

+		fs.Layouts.Dirs,

+		//fs.Resources.Dirs,

+		fs.StaticDirs,

+	} {

+		dirs = append(dirs, dirSet...)

+	}

+

+	return dirs

+}

+

 // RelContentDir tries to create a path relative to the content root from

 // the given filename. The return value is the path and language code.

 func (b *BaseFs) RelContentDir(filename string) string {

-	for _, dirname := range b.SourceFilesystems.Content.Dirnames {

+	for _, dir := range b.SourceFilesystems.Content.Dirs {

+		dirname := dir.Meta().Filename()

 		if strings.HasPrefix(filename, dirname) {

 			rel := strings.TrimPrefix(filename, dirname)

 			return strings.TrimPrefix(rel, filePathSeparator)

@@ -80,16 +107,22 @@

 	Layouts    *SourceFilesystem

 	Archetypes *SourceFilesystem

 	Assets     *SourceFilesystem

-	Resources  *SourceFilesystem

-	// This is a unified read-only view of the project's and themes' workdir.

-	Work *SourceFilesystem

+	// Writable filesystem on top the project's resources directory,

+	// with any sub module's resource fs layered below.

+	ResourcesCache afero.Fs

+	// The project folder.

+	Work afero.Fs

+

 	// When in multihost we have one static filesystem per language. The sync

 	// static files is currently done outside of the Hugo build (where there is

 	// a concept of a site per language).

 	// When in non-multihost mode there will be one entry in this map with a blank key.

 	Static map[string]*SourceFilesystem

+

+	// All the /static dirs (including themes/modules).

+	StaticDirs []hugofs.FileMetaInfo

 }

 // A SourceFilesystem holds the filesystem for a given source type in Hugo (data,

@@ -99,13 +132,10 @@

 	// This is a virtual composite filesystem. It expects path relative to a context.

 	Fs afero.Fs

-	// This is the base source filesystem. In real Hugo, this will be the OS filesystem.

-	// Use this if you need to resolve items in Dirnames below.

-	SourceFs afero.Fs

+	// This filesystem as separate root directories, starting from project and down

+	// to the themes/modules.

+	Dirs []hugofs.FileMetaInfo

-	// Dirnames is absolute filenames to the directories in this filesystem.

-	Dirnames []string

-

 	// When syncing a source folder to the target (e.g. /public), this may

 	// be set to publish into a subfolder. This is used for static syncing

 	// in multihost mode.

@@ -207,7 +237,8 @@

 // MakePathRelative creates a relative path from the given filename.

 // It will return an empty string if the filename is not a member of this filesystem.

 func (d *SourceFilesystem) MakePathRelative(filename string) string {

-	for _, currentPath := range d.Dirnames {

+	for _, dir := range d.Dirs {

+		currentPath := dir.(hugofs.FileMetaInfo).Meta().Filename()

 		if strings.HasPrefix(filename, currentPath) {

 			return strings.TrimPrefix(filename, currentPath)

 		}

@@ -220,8 +251,8 @@

 	if err != nil {

 		return rel

 	}

-	if realfi, ok := fi.(hugofs.RealFilenameInfo); ok {

-		return realfi.RealFilename()

+	if realfi, ok := fi.(hugofs.FileMetaInfo); ok {

+		return realfi.Meta().Filename()

 	}

 	return rel

@@ -229,8 +260,8 @@

 // Contains returns whether the given filename is a member of the current filesystem.

 func (d *SourceFilesystem) Contains(filename string) bool {

-	for _, dir := range d.Dirnames {

-		if strings.HasPrefix(filename, dir) {

+	for _, dir := range d.Dirs {

+		if strings.HasPrefix(filename, dir.Meta().Filename()) {

 			return true

 		}

 	}

@@ -241,9 +272,12 @@

 // path.

 func (d *SourceFilesystem) RealDirs(from string) []string {

 	var dirnames []string

-	for _, dir := range d.Dirnames {

-		dirname := filepath.Join(dir, from)

-		if _, err := d.SourceFs.Stat(dirname); err == nil {

+	for _, dir := range d.Dirs {

+		meta := dir.Meta()

+		dirname := filepath.Join(meta.Filename(), from)

+		_, err := meta.Fs().Stat(from)

+

+		if err == nil {

 			dirnames = append(dirnames, dirname)

 		}

 	}

@@ -254,17 +288,12 @@

 // the same across sites/languages.

 func WithBaseFs(b *BaseFs) func(*BaseFs) error {

 	return func(bb *BaseFs) error {

-		bb.themeFs = b.themeFs

-		bb.AbsThemeDirs = b.AbsThemeDirs

+		bb.theBigFs = b.theBigFs

+		bb.SourceFilesystems = b.SourceFilesystems

 		return nil

 	}

 }

-func newRealBase(base afero.Fs) afero.Fs {

-	return hugofs.NewBasePathRealFilenameFs(base.(*afero.BasePathFs))

-

-}

-

 // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase

 func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) {

 	fs := p.Fs

@@ -271,23 +300,6 @@

 	publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)

-	contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages)

-	if err != nil {

-		return nil, err

-	}

-

-	// Make sure we don't have any overlapping content dirs. That will never work.

-	for i, d1 := range absContentDirs {

-		for j, d2 := range absContentDirs {

-			if i == j {

-				continue

-			}

-			if strings.HasPrefix(d1, d2) || strings.HasPrefix(d2, d1) {

-				return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)

-			}

-		}

-	}

-

 	b := &BaseFs{

 		PublishFs: publishFs,

 	}

@@ -298,463 +310,395 @@

 		}

 	}

+	if b.theBigFs != nil && b.SourceFilesystems != nil {

+		return b, nil

+	}

+

 	builder := newSourceFilesystemsBuilder(p, b)

 	sourceFilesystems, err := builder.Build()

 	if err != nil {

-		return nil, err

+		return nil, errors.Wrap(err, "build filesystems")

 	}

-	sourceFilesystems.Content = &SourceFilesystem{

-		SourceFs: fs.Source,

-		Fs:       contentFs,

-		Dirnames: absContentDirs,

-	}

-

 	b.SourceFilesystems = sourceFilesystems

-	b.themeFs = builder.themeFs

-	b.AbsThemeDirs = builder.absThemeDirs

+	b.theBigFs = builder.theBigFs

 	return b, nil

 }

 type sourceFilesystemsBuilder struct {

-	p            *paths.Paths

-	result       *SourceFilesystems

-	themeFs      afero.Fs

-	hasTheme     bool

-	absThemeDirs []string

+	p        *paths.Paths

+	sourceFs afero.Fs

+	result   *SourceFilesystems

+	theBigFs *filesystemsCollector

 }

 func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder {

-	return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}}

+	sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source)

+	return &sourceFilesystemsBuilder{p: p, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}}

 }

+func (b *sourceFilesystemsBuilder) newSourceFilesystem(fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem {

+	return &SourceFilesystem{

+		Fs:   fs,

+		Dirs: dirs,

+	}

+}

 func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {

-	if b.themeFs == nil && b.p.ThemeSet() {

-		themeFs, absThemeDirs, err := createThemesOverlayFs(b.p)

+

+	if b.theBigFs == nil {

+

+		theBigFs, err := b.createMainOverlayFs(b.p)

 		if err != nil {

-			return nil, err

+			return nil, errors.Wrap(err, "create main fs")

 		}

-		if themeFs == nil {

-			panic("createThemesFs returned nil")

-		}

-		b.themeFs = themeFs

-		b.absThemeDirs = absThemeDirs

+		b.theBigFs = theBigFs

 	}

-	b.hasTheme = len(b.absThemeDirs) > 0

+	createView := func(componentID string) *SourceFilesystem {

+		if b.theBigFs == nil || b.theBigFs.overlayMounts == nil {

+			return b.newSourceFilesystem(hugofs.NoOpFs, nil)

+		}

-	sfs, err := b.createRootMappingFs("dataDir", "data")

-	if err != nil {

-		return nil, err

-	}

-	b.result.Data = sfs

+		dirs := b.theBigFs.overlayDirs[componentID]

-	sfs, err = b.createRootMappingFs("i18nDir", "i18n")

-	if err != nil {

-		return nil, err

-	}

-	b.result.I18n = sfs

+		return b.newSourceFilesystem(afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs)

-	sfs, err = b.createFs(false, true, "layoutDir", "layouts")

-	if err != nil {

-		return nil, err

 	}

-	b.result.Layouts = sfs

-	sfs, err = b.createFs(false, true, "archetypeDir", "archetypes")

-	if err != nil {

-		return nil, err

-	}

-	b.result.Archetypes = sfs

+	b.theBigFs.finalizeDirs()

-	sfs, err = b.createFs(false, true, "assetDir", "assets")

-	if err != nil {

-		return nil, err

-	}

-	b.result.Assets = sfs

+	b.result.Archetypes = createView(files.ComponentFolderArchetypes)

+	b.result.Layouts = createView(files.ComponentFolderLayouts)

+	b.result.Assets = createView(files.ComponentFolderAssets)

+	b.result.ResourcesCache = b.theBigFs.overlayResources

-	sfs, err = b.createFs(true, false, "resourceDir", "resources")

+	// Data, i18n and content cannot use the overlay fs

+	dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData]

+	dataFs, err := hugofs.NewSliceFs(dataDirs...)

 	if err != nil {

 		return nil, err

 	}

-	b.result.Resources = sfs

+	b.result.Data = b.newSourceFilesystem(dataFs, dataDirs)

-	sfs, err = b.createFs(false, true, "", "")

+	i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n]

+	i18nFs, err := hugofs.NewSliceFs(i18nDirs...)

 	if err != nil {

 		return nil, err

 	}

-	b.result.Work = sfs

+	b.result.I18n = b.newSourceFilesystem(i18nFs, i18nDirs)

-	err = b.createStaticFs()

+	contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent]

+	contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent)

+

+	contentFs, err := hugofs.NewLanguageFs(b.p.LanguagesDefaultFirst.AsOrdinalSet(), contentBfs)

 	if err != nil {

-		return nil, err

+		return nil, errors.Wrap(err, "create content filesystem")

 	}

-	return b.result, nil

-}

+	b.result.Content = b.newSourceFilesystem(contentFs, contentDirs)

-func (b *sourceFilesystemsBuilder) createFs(

-	mkdir bool,

-	readOnly bool,

-	dirKey, themeFolder string) (*SourceFilesystem, error) {

-	s := &SourceFilesystem{

-		SourceFs: b.p.Fs.Source,

-	}

+	b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull)

-	if themeFolder == "" {

-		themeFolder = filePathSeparator

-	}

+	// Create static filesystem(s)

+	ms := make(map[string]*SourceFilesystem)

+	b.result.Static = ms

+	b.result.StaticDirs = b.theBigFs.overlayDirs[files.ComponentFolderStatic]

-	var dir string

-	if dirKey != "" {

-		dir = b.p.Cfg.GetString(dirKey)

-		if dir == "" {

-			return s, fmt.Errorf("config %q not set", dirKey)

+	if b.theBigFs.staticPerLanguage != nil {

+		// Multihost mode

+		for k, v := range b.theBigFs.staticPerLanguage {

+			sfs := b.newSourceFilesystem(v, b.result.StaticDirs)

+			sfs.PublishFolder = k

+			ms[k] = sfs

 		}

+	} else {

+		bfs := afero.NewBasePathFs(b.theBigFs.overlayMounts, files.ComponentFolderStatic)

+		ms[""] = b.newSourceFilesystem(bfs, b.result.StaticDirs)

 	}

-	var fs afero.Fs

+	return b.result, nil

-	absDir := b.p.AbsPathify(dir)

-	existsInSource := b.existsInSource(absDir)

-	if !existsInSource && mkdir {

-		// We really need this directory. Make it.

-		if err := b.p.Fs.Source.MkdirAll(absDir, 0777); err == nil {

-			existsInSource = true

-		}

-	}

-	if existsInSource {

-		fs = newRealBase(afero.NewBasePathFs(b.p.Fs.Source, absDir))

-		s.Dirnames = []string{absDir}

-	}

+}

-	if b.hasTheme {

-		if !strings.HasPrefix(themeFolder, filePathSeparator) {

-			themeFolder = filePathSeparator + themeFolder

-		}

-		themeFolderFs := newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder))

-		if fs == nil {

-			fs = themeFolderFs

-		} else {

-			fs = afero.NewCopyOnWriteFs(themeFolderFs, fs)

-		}

+func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) {

-		for _, absThemeDir := range b.absThemeDirs {

-			absThemeFolderDir := filepath.Join(absThemeDir, themeFolder)

-			if b.existsInSource(absThemeFolderDir) {

-				s.Dirnames = append(s.Dirnames, absThemeFolderDir)

-			}

-		}

+	var staticFsMap map[string]afero.Fs

+	if b.p.Cfg.GetBool("multihost") {

+		staticFsMap = make(map[string]afero.Fs)

 	}

-	if fs == nil {

-		s.Fs = hugofs.NoOpFs

-	} else if readOnly {

-		s.Fs = afero.NewReadOnlyFs(fs)

-	} else {

-		s.Fs = fs

+	collector := &filesystemsCollector{

+		sourceProject:     b.sourceFs,

+		sourceModules:     hugofs.NewNoSymlinkFs(b.sourceFs),

+		overlayDirs:       make(map[string][]hugofs.FileMetaInfo),

+		staticPerLanguage: staticFsMap,

 	}

-	return s, nil

-}

+	mods := p.AllModules

-// Used for data, i18n -- we cannot use overlay filsesystems for those, but we need

-// to keep a strict order.

-func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) {

-	s := &SourceFilesystem{

-		SourceFs: b.p.Fs.Source,

+	if len(mods) == 0 {

+		return collector, nil

 	}

-	projectDir := b.p.Cfg.GetString(dirKey)

-	if projectDir == "" {

-		return nil, fmt.Errorf("config %q not set", dirKey)

-	}

+	modsReversed := make([]mountsDescriptor, len(mods))

-	var fromTo []string

-	to := b.p.AbsPathify(projectDir)

+	// The theme components are ordered from left to right.

+	// We need to revert it to get the

+	// overlay logic below working as expected, with the project on top (last).

-	if b.existsInSource(to) {

-		s.Dirnames = []string{to}

-		fromTo = []string{projectVirtualFolder, to}

-	}

+	for i, mod := range mods {

+		dir := mod.Dir()

-	for _, theme := range b.p.AllThemes {

-		to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder))

-		if b.existsInSource(to) {

-			s.Dirnames = append(s.Dirnames, to)

-			from := theme

-			fromTo = append(fromTo, from.Name, to)

+		if i < len(mods)-1 {

+			i = len(mods) - 2 - i

 		}

-	}

-	if len(fromTo) == 0 {

-		s.Fs = hugofs.NoOpFs

-		return s, nil

+		isMainProject := mod.Owner() == nil

+		modsReversed[i] = mountsDescriptor{

+			mounts:        mod.Mounts(),

+			dir:           dir,

+			watch:         mod.Watch(),

+			isMainProject: isMainProject,

+		}

 	}

-	fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...)

-	if err != nil {

-		return nil, err

-	}

+	err := b.createOverlayFs(collector, modsReversed)

-	s.Fs = afero.NewReadOnlyFs(fs)

+	return collector, err

-	return s, nil

 }

-func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool {

-	exists, _ := afero.Exists(b.p.Fs.Source, abspath)

-	return exists

+func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool {

+	return strings.HasPrefix(mnt.Target, files.ComponentFolderContent)

 }

-func (b *sourceFilesystemsBuilder) createStaticFs() error {

-	isMultihost := b.p.Cfg.GetBool("multihost")

-	ms := make(map[string]*SourceFilesystem)

-	b.result.Static = ms

+func (b *sourceFilesystemsBuilder) createModFs(

+	collector *filesystemsCollector,

+	md mountsDescriptor) error {

-	if isMultihost {

-		for _, l := range b.p.Languages {

-			s := &SourceFilesystem{

-				SourceFs:      b.p.Fs.Source,

-				PublishFolder: l.Lang}

-			staticDirs := removeDuplicatesKeepRight(getStaticDirs(l))

-			if len(staticDirs) == 0 {

-				continue

-			}

+	var (

+		fromTo        []hugofs.RootMapping

+		fromToContent []hugofs.RootMapping

+	)

-			for _, dir := range staticDirs {

-				absDir := b.p.AbsPathify(dir)

-				if !b.existsInSource(absDir) {

-					continue

-				}

+	absPathify := func(path string) string {

+		return paths.AbsPathify(md.dir, path)

+	}

-				s.Dirnames = append(s.Dirnames, absDir)

-			}

+	seen := make(map[string]bool)

-			fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames)

-			if err != nil {

-				return err

-			}

+	var mounts []modules.Mount

-			if b.hasTheme {

-				themeFolder := "static"

-				fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs)

-				for _, absThemeDir := range b.absThemeDirs {

-					s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder))

-				}

-			}

+OUTER:

+	for i, mount := range md.mounts {

+		key := path.Join(mount.Lang, mount.Source, mount.Target)

+		if seen[key] {

+			continue

+		}

+		seen[key] = true

-			s.Fs = fs

-			ms[l.Lang] = s

-

+		// Prevent overlapping mounts

+		for j, mount2 := range md.mounts {

+			if j == i || mount2.Target != mount.Target {

+				continue

+			}

+			source := mount.Source

+			if !strings.HasSuffix(source, filePathSeparator) {

+				source += filePathSeparator

+			}

+			if strings.HasPrefix(mount2.Source, source) {

+				continue OUTER

+			}

 		}

-		return nil

+		mounts = append(mounts, mount)

 	}

-	s := &SourceFilesystem{

-		SourceFs: b.p.Fs.Source,

-	}

+	for _, mount := range mounts {

-	var staticDirs []string

+		mountWeight := 1

+		if md.isMainProject {

+			mountWeight++

+		}

-	for _, l := range b.p.Languages {

-		staticDirs = append(staticDirs, getStaticDirs(l)...)

-	}

+		rm := hugofs.RootMapping{

+			From: mount.Target,

+			To:   absPathify(mount.Source),

+			Meta: hugofs.FileMeta{

+				"watch":       md.watch,

+				"mountWeight": mountWeight,

+			},

+		}

-	staticDirs = removeDuplicatesKeepRight(staticDirs)

-	if len(staticDirs) == 0 {

-		return nil

-	}

+		isContentMount := b.isContentMount(mount)

-	for _, dir := range staticDirs {

-		absDir := b.p.AbsPathify(dir)

-		if !b.existsInSource(absDir) {

-			continue

+		lang := mount.Lang

+		if lang == "" && isContentMount {

+			lang = b.p.DefaultContentLanguage

 		}

-		s.Dirnames = append(s.Dirnames, absDir)

-	}

-	fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames)

-	if err != nil {

-		return err

-	}

+		rm.Meta["lang"] = lang

-	if b.hasTheme {

-		themeFolder := "static"

-		fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs)

-		for _, absThemeDir := range b.absThemeDirs {

-			s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder))

+		if isContentMount {

+			fromToContent = append(fromToContent, rm)

+		} else {

+			fromTo = append(fromTo, rm)

 		}

 	}

-	s.Fs = fs

-	ms[""] = s

-

-	return nil

-}

-

-func getStaticDirs(cfg config.Provider) []string {

-	var staticDirs []string

-	for i := -1; i <= 10; i++ {

-		staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)

+	modBase := collector.sourceProject

+	if !md.isMainProject {

+		modBase = collector.sourceModules

 	}

-	return staticDirs

-}

-func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {

-

-	if id >= 0 {

-		key = fmt.Sprintf("%s%d", key, id)

+	rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)

+	if err != nil {

+		return err

 	}

+	rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...)

+	if err != nil {

+		return err

+	}

-	return config.GetStringSlicePreserveString(cfg, key)

+	// We need to keep the ordered list of directories for watching and

+	// some special merge operations (data, i18n).

+	collector.addDirs(rmfs)

+	collector.addDirs(rmfsContent)

-}

+	if collector.staticPerLanguage != nil {

+		for _, l := range b.p.Languages {

+			lang := l.Lang

-func createContentFs(fs afero.Fs,

-	workingDir,

-	defaultContentLanguage string,

-	languages langs.Languages) (afero.Fs, []string, error) {

+			lfs := rmfs.Filter(func(rm hugofs.RootMapping) bool {

+				rlang := rm.Meta.Lang()

+				return rlang == "" || rlang == lang

+			})

-	var contentLanguages langs.Languages

-	var contentDirSeen = make(map[string]bool)

-	languageSet := make(map[string]bool)

+			bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic)

-	// The default content language needs to be first.

-	for _, language := range languages {

-		if language.Lang == defaultContentLanguage {

-			contentLanguages = append(contentLanguages, language)

-			contentDirSeen[language.ContentDir] = true

+			sfs, found := collector.staticPerLanguage[lang]

+			if found {

+				collector.staticPerLanguage[lang] = afero.NewCopyOnWriteFs(sfs, bfs)

+

+			} else {

+				collector.staticPerLanguage[lang] = bfs

+			}

 		}

-		languageSet[language.Lang] = true

 	}

-	for _, language := range languages {

-		if contentDirSeen[language.ContentDir] {

-			continue

+	getResourcesDir := func() string {

+		if md.isMainProject {

+			return b.p.AbsResourcesDir

 		}

-		if language.ContentDir == "" {

-			language.ContentDir = defaultContentLanguage

-		}

-		contentDirSeen[language.ContentDir] = true

-		contentLanguages = append(contentLanguages, language)

-

+		return absPathify(files.FolderResources)

 	}

-	var absContentDirs []string

+	if collector.overlayMounts == nil {

+		collector.overlayMounts = rmfs

+		collector.overlayMountsContent = rmfsContent

+		collector.overlayFull = afero.NewBasePathFs(modBase, md.dir)

+		collector.overlayResources = afero.NewBasePathFs(modBase, getResourcesDir())

+	} else {

-	fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)

-	return fs, absContentDirs, err

-

-}

-

-func createContentOverlayFs(source afero.Fs,

-	workingDir string,

-	languages langs.Languages,

-	languageSet map[string]bool,

-	absContentDirs *[]string) (afero.Fs, error) {

-	if len(languages) == 0 {

-		return source, nil

+		collector.overlayMounts = afero.NewCopyOnWriteFs(collector.overlayMounts, rmfs)

+		collector.overlayMountsContent = hugofs.NewLanguageCompositeFs(collector.overlayMountsContent, rmfsContent)

+		collector.overlayFull = afero.NewCopyOnWriteFs(collector.overlayFull, afero.NewBasePathFs(modBase, md.dir))

+		collector.overlayResources = afero.NewCopyOnWriteFs(collector.overlayResources, afero.NewBasePathFs(modBase, getResourcesDir()))

 	}

-	language := languages[0]

+	return nil

-	contentDir := language.ContentDir

-	if contentDir == "" {

-		panic("missing contentDir")

-	}

+}

-	absContentDir := paths.AbsPathify(workingDir, language.ContentDir)

-	if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) {

-		absContentDir += paths.FilePathSeparator

+func printFs(fs afero.Fs, path string, w io.Writer) {

+	if fs == nil {

+		return

 	}

+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {

+		if err != nil {

+			return err

+		}

+		if info.IsDir() {

+			return nil

+		}

+		var filename string

+		if fim, ok := info.(hugofs.FileMetaInfo); ok {

+			filename = fim.Meta().Filename()

+		}

+		fmt.Fprintf(w, "    %q %q\n", path, filename)

+		return nil

+	})

+}

-	// If root, remove the second '/'

-	if absContentDir == "//" {

-		absContentDir = paths.FilePathSeparator

-	}

+type filesystemsCollector struct {

+	sourceProject afero.Fs // Source for project folders

+	sourceModules afero.Fs // Source for modules/themes

-	if len(absContentDir) < 6 {

-		return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir)

-	}

+	overlayMounts        afero.Fs

+	overlayMountsContent afero.Fs

+	overlayFull          afero.Fs

+	overlayResources     afero.Fs

-	*absContentDirs = append(*absContentDirs, absContentDir)

+	// Maps component type (layouts, static, content etc.) an ordered list of

+	// directories representing the overlay filesystems above.

+	overlayDirs map[string][]hugofs.FileMetaInfo

-	overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))

-	if len(languages) == 1 {

-		return overlay, nil

-	}

+	// Set if in multihost mode

+	staticPerLanguage map[string]afero.Fs

-	base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)

-	if err != nil {

-		return nil, err

-	}

-

-	return hugofs.NewLanguageCompositeFs(base, overlay), nil

-

+	finalizerInit sync.Once

 }

-func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) {

+func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) {

+	for _, componentFolder := range files.ComponentFolders {

+		dirs, err := rfs.Dirs(componentFolder)

-	themes := p.AllThemes

-

-	if len(themes) == 0 {

-		panic("AllThemes not set")

+		if err == nil {

+			c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...)

+		}

 	}

+}

-	themesDir := p.AbsPathify(p.ThemesDir)

-	if themesDir == "" {

-		return nil, nil, errors.New("no themes dir set")

-	}

+func (c *filesystemsCollector) finalizeDirs() {

+	c.finalizerInit.Do(func() {

+		// Order the directories from top to bottom (project, theme a, theme ...).

+		for _, dirs := range c.overlayDirs {

+			c.reverseFis(dirs)

+		}

+	})

-	absPaths := make([]string, len(themes))

+}

-	// The themes are ordered from left to right. We need to revert it to get the

-	// overlay logic below working as expected.

-	for i := 0; i < len(themes); i++ {

-		absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name)

+func (c *filesystemsCollector) reverseFis(fis []hugofs.FileMetaInfo) {

+	for i := len(fis)/2 - 1; i >= 0; i-- {

+		opp := len(fis) - 1 - i

+		fis[i], fis[opp] = fis[opp], fis[i]

 	}

+}

-	fs, err := createOverlayFs(p.Fs.Source, absPaths)

-	fs = hugofs.NewNoLstatFs(fs)

-

-	return fs, absPaths, err

-

+type mountsDescriptor struct {

+	mounts        []modules.Mount

+	dir           string

+	watch         bool // whether this is a candidate for watching in server mode.

+	isMainProject bool

 }

-func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) {

-	if len(absPaths) == 0 {

-		return hugofs.NoOpFs, nil

+func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollector, mounts []mountsDescriptor) error {

+	if len(mounts) == 0 {

+		return nil

 	}

-	if len(absPaths) == 1 {

-		return afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))), nil

-	}

-

-	base := afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0])))

-	overlay, err := createOverlayFs(source, absPaths[1:])

+	err := b.createModFs(collector, mounts[0])

 	if err != nil {

-		return nil, err

+		return err

 	}

-	return afero.NewCopyOnWriteFs(base, overlay), nil

-}

-

-func removeDuplicatesKeepRight(in []string) []string {

-	seen := make(map[string]bool)

-	var out []string

-	for i := len(in) - 1; i >= 0; i-- {

-		v := in[i]

-		if seen[v] {

-			continue

-		}

-		out = append([]string{v}, out...)

-		seen[v] = true

+	if len(mounts) == 1 {

+		return nil

 	}

-	return out

+	return b.createOverlayFs(collector, mounts[1:])

 }

--- a/hugolib/filesystems/basefs_test.go

+++ b/hugolib/filesystems/basefs_test.go

@@ -1,4 +1,4 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

+// Copyright 2019 The Hugo Authors. All rights reserved.

 //

 // Licensed under the Apache License, Version 2.0 (the "License");

 // you may not use this file except in compliance with the License.

@@ -18,8 +18,11 @@

 	"fmt"

 	"os"

 	"path/filepath"

+	"strings"

 	"testing"

+	"github.com/gohugoio/hugo/config"

+

 	"github.com/gohugoio/hugo/langs"

 	"github.com/spf13/afero"

@@ -26,10 +29,48 @@

 	"github.com/gohugoio/hugo/hugofs"

 	"github.com/gohugoio/hugo/hugolib/paths"

+	"github.com/gohugoio/hugo/modules"

 	"github.com/spf13/viper"

 	"github.com/stretchr/testify/require"

 )

+func initConfig(fs afero.Fs, cfg config.Provider) error {

+	if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {

+		return err

+	}

+

+	modConfig, err := modules.DecodeConfig(cfg)

+	if err != nil {

+		return err

+	}

+

+	workingDir := cfg.GetString("workingDir")

+	themesDir := cfg.GetString("themesDir")

+	if !filepath.IsAbs(themesDir) {

+		themesDir = filepath.Join(workingDir, themesDir)

+	}

+	modulesClient := modules.NewClient(modules.ClientConfig{

+		Fs:           fs,

+		WorkingDir:   workingDir,

+		ThemesDir:    themesDir,

+		ModuleConfig: modConfig,

+		IgnoreVendor: true,

+	})

+

+	moduleConfig, err := modulesClient.Collect()

+	if err != nil {

+		return err

+	}

+

+	if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil {

+		return err

+	}

+

+	cfg.Set("allModules", moduleConfig.ActiveModules)

+

+	return nil

+}

+

 func TestNewBaseFs(t *testing.T) {

 	assert := require.New(t)

 	v := viper.New()

@@ -40,7 +81,9 @@

 	workingDir := filepath.FromSlash("/my/work")

 	v.Set("workingDir", workingDir)

+	v.Set("contentDir", "content")

 	v.Set("themesDir", "themes")

+	v.Set("defaultContentLanguage", "en")

 	v.Set("theme", themes[:1])

 	// Write some data to the themes

@@ -47,9 +90,12 @@

 	for _, theme := range themes {

 		for _, dir := range []string{"i18n", "data", "archetypes", "layouts"} {

 			base := filepath.Join(workingDir, "themes", theme, dir)

-			filename := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme))

+			filenameTheme := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme))

+			filenameOverlap := filepath.Join(base, "f3.txt")

 			fs.Source.Mkdir(base, 0755)

-			afero.WriteFile(fs.Source, filename, []byte(fmt.Sprintf("content:%s:%s", theme, dir)), 0755)

+			content := []byte(fmt.Sprintf("content:%s:%s", theme, dir))

+			afero.WriteFile(fs.Source, filenameTheme, content, 0755)

+			afero.WriteFile(fs.Source, filenameOverlap, content, 0755)

 		}

 		// Write some files to the root of the theme

 		base := filepath.Join(workingDir, "themes", theme)

@@ -73,6 +119,7 @@

 	setConfigAndWriteSomeFilesTo(fs.Source, v, "resourceDir", "myrsesource", 10)

 	v.Set("publishDir", "public")

+	assert.NoError(initConfig(fs.Source, v))

 	p, err := paths.New(fs, v)

 	assert.NoError(err)

@@ -85,34 +132,27 @@

 	assert.NoError(err)

 	dirnames, err := root.Readdirnames(-1)

 	assert.NoError(err)

-	assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames)

-	ff, err := bfs.I18n.Fs.Open("myi18n")

-	assert.NoError(err)

-	_, err = ff.Readdirnames(-1)

-	assert.NoError(err)

+	assert.Equal([]string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}, dirnames)

 	root, err = bfs.Data.Fs.Open("")

 	assert.NoError(err)

 	dirnames, err = root.Readdirnames(-1)

 	assert.NoError(err)

-	assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames)

-	ff, err = bfs.I18n.Fs.Open("mydata")

-	assert.NoError(err)

-	_, err = ff.Readdirnames(-1)

-	assert.NoError(err)

+	assert.Equal([]string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt", "f6.txt", "f7.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}, dirnames)

-	checkFileCount(bfs.Content.Fs, "", assert, 3)

-	checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes

+	//printFs(bfs.Work, "", os.Stdout)

+

 	checkFileCount(bfs.Layouts.Fs, "", assert, 7)

+

+	checkFileCount(bfs.Content.Fs, "", assert, 3)

+	checkFileCount(bfs.I18n.Fs, "", assert, 8) // 4 + 4 themes

+

 	checkFileCount(bfs.Static[""].Fs, "", assert, 6)

-	checkFileCount(bfs.Data.Fs, "", assert, 9)        // 7 + 2 themes

+	checkFileCount(bfs.Data.Fs, "", assert, 11)       // 7 + 4 themes

 	checkFileCount(bfs.Archetypes.Fs, "", assert, 10) // 8 + 2 themes

 	checkFileCount(bfs.Assets.Fs, "", assert, 9)

-	checkFileCount(bfs.Resources.Fs, "", assert, 10)

-	checkFileCount(bfs.Work.Fs, "", assert, 78)

+	checkFileCount(bfs.Work, "", assert, 82)

-	assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames)

-

 	assert.True(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt")))

 	assert.True(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt")))

 	assert.True(bfs.IsLayout(filepath.Join(workingDir, "mylayouts", "file1.txt")))

@@ -125,13 +165,13 @@

 	assert.Equal("file1.txt", rel)

 	// Check Work fs vs theme

-	checkFileContent(bfs.Work.Fs, "file-root.txt", assert, "content-project")

-	checkFileContent(bfs.Work.Fs, "theme-root-atheme.txt", assert, "content:atheme")

+	checkFileContent(bfs.Work, "file-root.txt", assert, "content-project")

+	checkFileContent(bfs.Work, "theme-root-atheme.txt", assert, "content:atheme")

 	// https://github.com/gohugoio/hugo/issues/5318

 	// Check both project and theme.

 	for _, fs := range []afero.Fs{bfs.Archetypes.Fs, bfs.Layouts.Fs} {

-		for _, filename := range []string{"/file1.txt", "/theme-file-atheme.txt"} {

+		for _, filename := range []string{"/f1.txt", "/theme-file-atheme.txt"} {

 			filename = filepath.FromSlash(filename)

 			f, err := fs.Open(filename)

 			assert.NoError(err)

@@ -153,6 +193,7 @@

 	v.Set("assetDir", "myassets")

 	v.Set("resourceDir", "resources")

 	v.Set("publishDir", "public")

+	v.Set("defaultContentLanguage", "en")

 	return v

 }

@@ -161,17 +202,18 @@

 	assert := require.New(t)

 	v := createConfig()

 	fs := hugofs.NewMem(v)

+	assert.NoError(initConfig(fs.Source, v))

+

 	p, err := paths.New(fs, v)

 	assert.NoError(err)

 	bfs, err := NewBase(p)

 	assert.NoError(err)

 	assert.NotNil(bfs)

-	assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs)

-	assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs)

-	assert.Equal(hugofs.NoOpFs, bfs.Data.Fs)

-	assert.Equal(hugofs.NoOpFs, bfs.Assets.Fs)

-	assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs)

-	assert.NotNil(bfs.Work.Fs)

+	assert.NotNil(bfs.Archetypes.Fs)

+	assert.NotNil(bfs.Layouts.Fs)

+	assert.NotNil(bfs.Data.Fs)

+	assert.NotNil(bfs.I18n.Fs)

+	assert.NotNil(bfs.Work)

 	assert.NotNil(bfs.Content.Fs)

 	assert.NotNil(bfs.Static)

 }

@@ -217,11 +259,14 @@

 	afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0755)

 	afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0755)

+	assert.NoError(initConfig(fs.Source, v))

+

 	p, err := paths.New(fs, v)

 	assert.NoError(err)

 	bfs, err := NewBase(p)

 	assert.NoError(err)

 	assert.NotNil(bfs)

+

 	checkFileCount(bfs.Assets.Fs, "", assert, 6)

 	realDirs := bfs.Assets.RealDirs("scss")

@@ -229,14 +274,8 @@

 	assert.Equal(filepath.Join(root, "myassets/scss"), realDirs[0])

 	assert.Equal(filepath.Join(themesDir, "mytheme/assets/scss"), realDirs[len(realDirs)-1])

-	checkFileCount(bfs.Resources.Fs, "", assert, 3)

+	assert.NotNil(bfs.theBigFs)

-	assert.NotNil(bfs.themeFs)

-	fi, b, err := bfs.themeFs.(afero.Lstater).LstatIfPossible(filepath.Join("resources", "t1.txt"))

-	assert.NoError(err)

-	assert.False(b)

-	assert.Equal("t1.txt", fi.Name())

-

 }

 func TestStaticFs(t *testing.T) {

@@ -245,20 +284,25 @@

 	workDir := "mywork"

 	v.Set("workingDir", workDir)

 	v.Set("themesDir", "themes")

-	v.Set("theme", "t1")

+	v.Set("theme", []string{"t1", "t2"})

 	fs := hugofs.NewMem(v)

 	themeStaticDir := filepath.Join(workDir, "themes", "t1", "static")

+	themeStaticDir2 := filepath.Join(workDir, "themes", "t2", "static")

 	afero.WriteFile(fs.Source, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0755)

 	afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755)

 	afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755)

+	afero.WriteFile(fs.Source, filepath.Join(themeStaticDir2, "f2.txt"), []byte("Hugo Themes Rocks in t2!"), 0755)

+	assert.NoError(initConfig(fs.Source, v))

+

 	p, err := paths.New(fs, v)

 	assert.NoError(err)

 	bfs, err := NewBase(p)

 	assert.NoError(err)

+

 	sfs := bfs.StaticFs("en")

 	checkFileContent(sfs, "f1.txt", assert, "Hugo Rocks!")

 	checkFileContent(sfs, "f2.txt", assert, "Hugo Themes Still Rocks!")

@@ -272,21 +316,19 @@

 	v.Set("workingDir", workDir)

 	v.Set("themesDir", "themes")

 	v.Set("theme", "t1")

-	v.Set("multihost", true)

+	v.Set("defaultContentLanguage", "en")

-	vn := viper.New()

-	vn.Set("staticDir", "nn_static")

-

-	en := langs.NewLanguage("en", v)

-	no := langs.NewLanguage("no", v)

-	no.Set("staticDir", "static_no")

-

-	languages := langs.Languages{

-		en,

-		no,

+	langConfig := map[string]interface{}{

+		"no": map[string]interface{}{

+			"staticDir": "static_no",

+			"baseURL":   "https://example.org/no/",

+		},

+		"en": map[string]interface{}{

+			"baseURL": "https://example.org/en/",

+		},

 	}

-	v.Set("languagesSorted", languages)

+	v.Set("languages", langConfig)

 	fs := hugofs.NewMem(v)

@@ -298,6 +340,8 @@

 	afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755)

 	afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755)

+	assert.NoError(initConfig(fs.Source, v))

+

 	p, err := paths.New(fs, v)

 	assert.NoError(err)

 	bfs, err := NewBase(p)

@@ -312,9 +356,9 @@

 }

 func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) {

-	count, _, err := countFileaAndGetDirs(fs, dirname)

-	assert.NoError(err)

-	assert.Equal(expected, count)

+	count, fnames, err := countFileaAndGetFilenames(fs, dirname)

+	assert.NoError(err, fnames)

+	assert.Equal(expected, count, fnames)

 }

 func checkFileContent(fs afero.Fs, filename string, assert *require.Assertions, expected ...string) {

@@ -329,27 +373,38 @@

 	}

 }

-func countFileaAndGetDirs(fs afero.Fs, dirname string) (int, []string, error) {

+func countFileaAndGetFilenames(fs afero.Fs, dirname string) (int, []string, error) {

 	if fs == nil {

 		return 0, nil, errors.New("no fs")

 	}

 	counter := 0

-	var dirs []string

+	var filenames []string

-	afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error {

-		if info != nil {

-			if !info.IsDir() {

-				counter++

-			} else if info.Name() != "." {

-				dirs = append(dirs, filepath.Join(path, info.Name()))

-			}

+	wf := func(path string, info hugofs.FileMetaInfo, err error) error {

+		if err != nil {

+			return err

 		}

+		if !info.IsDir() {

+			counter++

+		}

+		if info.Name() != "." {

+			name := info.Name()

+			name = strings.Replace(name, filepath.FromSlash("/my/work"), "WORK_DIR", 1)

+			filenames = append(filenames, name)

+		}

+

 		return nil

-	})

+	}

-	return counter, dirs, nil

+	w := hugofs.NewWalkway(hugofs.WalkwayConfig{Fs: fs, Root: dirname, WalkFn: wf})

+

+	if err := w.Walk(); err != nil {

+		return -1, nil, err

+	}

+

+	return counter, filenames, nil

 }

 func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) {

@@ -357,7 +412,7 @@

 	v.Set(key, val)

 	fs.Mkdir(val, 0755)

 	for i := 0; i < num; i++ {

-		filename := filepath.Join(workingDir, val, fmt.Sprintf("file%d.txt", i+1))

+		filename := filepath.Join(workingDir, val, fmt.Sprintf("f%d.txt", i+1))

 		afero.WriteFile(fs, filename, []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755)

 	}

 }

--- /dev/null

+++ b/hugolib/hugo_modules_test.go

@@ -1,0 +1,492 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugolib

+

+import (

+	"fmt"

+	"math/rand"

+	"os"

+	"path/filepath"

+	"strings"

+	"testing"

+	"time"

+

+	"github.com/gohugoio/hugo/common/loggers"

+

+	"github.com/spf13/afero"

+

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/gohugoio/hugo/common/hugo"

+

+	"github.com/gohugoio/hugo/htesting"

+	"github.com/gohugoio/hugo/hugofs"

+

+	"github.com/gohugoio/testmodBuilder/mods"

+	"github.com/spf13/viper"

+	"github.com/stretchr/testify/require"

+)

+

+// TODO(bep) this fails when testmodBuilder is also building ...

+func TestHugoModules(t *testing.T) {

+	t.Parallel()

+

+	if hugo.GoMinorVersion() < 12 {

+		// https://github.com/golang/go/issues/26794

+		// There were some concurrent issues with Go modules in < Go 12.

+		t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib")

+	}

+

+	if testing.Short() {

+		t.Skip()

+	}

+

+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))

+	gooss := []string{"linux", "darwin", "windows"}

+	goos := gooss[rnd.Intn(len(gooss))]

+	ignoreVendor := rnd.Intn(2) == 0

+	testmods := mods.CreateModules(goos).Collect()

+	rnd.Shuffle(len(testmods), func(i, j int) { testmods[i], testmods[j] = testmods[j], testmods[i] })

+

+	for _, m := range testmods[:2] {

+		assert := require.New(t)

+

+		v := viper.New()

+

+		workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-test")

+		assert.NoError(err)

+		defer clean()

+

+		configTemplate := `

+baseURL = "https://example.com"

+title = "My Modular Site"

+workingDir = %q

+theme = %q

+ignoreVendor = %t

+

+`

+

+		config := fmt.Sprintf(configTemplate, workingDir, m.Path(), ignoreVendor)

+

+		b := newTestSitesBuilder(t)

+

+		// Need to use OS fs for this.

+		b.Fs = hugofs.NewDefault(v)

+

+		b.WithWorkingDir(workingDir).WithConfigFile("toml", config)

+		b.WithContent("page.md", `

+---

+title: "Foo"

+---

+`)

+		b.WithTemplates("home.html", `

+

+{{ $mod := .Site.Data.modinfo.module }}

+Mod Name: {{ $mod.name }}

+Mod Version: {{ $mod.version }}

+----

+{{ range $k, $v := .Site.Data.modinfo }}

+- {{ $k }}: {{ range $kk, $vv := $v }}{{ $kk }}: {{ $vv }}|{{ end -}}

+{{ end }}

+

+

+`)

+		b.WithSourceFile("go.mod", `

+module github.com/gohugoio/tests/testHugoModules

+

+

+`)

+

+		b.Build(BuildCfg{})

+

+		// Verify that go.mod is autopopulated with all the modules in config.toml.

+		b.AssertFileContent("go.mod", m.Path())

+

+		b.AssertFileContent("public/index.html",

+			"Mod Name: "+m.Name(),

+			"Mod Version: v1.4.0")

+

+		b.AssertFileContent("public/index.html", createChildModMatchers(m, ignoreVendor, m.Vendor)...)

+

+	}

+}

+

+func createChildModMatchers(m *mods.Md, ignoreVendor, vendored bool) []string {

+	// Child depdendencies are one behind.

+	expectMinorVersion := 3

+

+	if !ignoreVendor && vendored {

+		// Vendored modules are stuck at v1.1.0.

+		expectMinorVersion = 1

+	}

+

+	expectVersion := fmt.Sprintf("v1.%d.0", expectMinorVersion)

+

+	var matchers []string

+

+	for _, mm := range m.Children {

+		matchers = append(

+			matchers,

+			fmt.Sprintf("%s: name: %s|version: %s", mm.Name(), mm.Name(), expectVersion))

+		matchers = append(matchers, createChildModMatchers(mm, ignoreVendor, vendored || mm.Vendor)...)

+	}

+	return matchers

+}

+

+func TestModulesWithContent(t *testing.T) {

+	t.Parallel()

+

+	b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", `

+baseURL="https://example.org"

+

+workingDir="/site"

+

+defaultContentLanguage = "en"

+

+[module]

+[[module.imports]]

+path="a"

+[[module.imports.mounts]]

+source="myacontent"

+target="content/blog"

+lang="en"

+[[module.imports]]

+path="b"

+[[module.imports.mounts]]

+source="mybcontent"

+target="content/blog"

+lang="nn"

+[[module.imports]]

+path="c"

+[[module.imports]]

+path="d"

+

+[languages]

+

+[languages.en]

+title = "Title in English"

+languageName = "English"

+weight = 1

+[languages.nn]

+languageName = "Nynorsk"

+weight = 2

+title = "Tittel på nynorsk"

+[languages.nb]

+languageName = "Bokmål"

+weight = 3

+title = "Tittel på bokmål"

+[languages.fr]

+languageName = "French"

+weight = 4

+title = "French Title"

+

+

+`)

+

+	b.WithTemplatesAdded("index.html", `

+{{ range .Site.RegularPages }}

+|{{ .Title }}|{{ .RelPermalink }}|{{ .Plain }}

+{{ end }}

+{{ $data := .Site.Data }}

+Data Common: {{ $data.common.value }}

+Data C: {{ $data.c.value }}

+Data D: {{ $data.d.value }}

+All Data: {{ $data }}

+

+i18n hello: {{ i18n "hello" . }}

+i18n theme: {{ i18n "theme" . }}

+i18n theme2: {{ i18n "theme2" . }}

+`)

+

+	content := func(id string) string {

+		return fmt.Sprintf(`---

+title: Title %s

+---

+Content %s

+

+`, id, id)

+	}

+

+	i18nContent := func(id, value string) string {

+		return fmt.Sprintf(`

+[%s]

+other = %q

+`, id, value)

+	}

+

+	// Content files

+	b.WithSourceFile("themes/a/myacontent/page.md", content("theme-a-en"))

+	b.WithSourceFile("themes/b/mybcontent/page.md", content("theme-b-nn"))

+	b.WithSourceFile("themes/c/content/blog/c.md", content("theme-c-nn"))

+

+	// Data files

+	b.WithSourceFile("data/common.toml", `value="Project"`)

+	b.WithSourceFile("themes/c/data/common.toml", `value="Theme C"`)

+	b.WithSourceFile("themes/c/data/c.toml", `value="Hugo Rocks!"`)

+	b.WithSourceFile("themes/d/data/c.toml", `value="Hugo Rodcks!"`)

+	b.WithSourceFile("themes/d/data/d.toml", `value="Hugo Rodks!"`)

+

+	// i18n files

+	b.WithSourceFile("i18n/en.toml", i18nContent("hello", "Project"))

+	b.WithSourceFile("themes/c/en.toml", i18nContent("hello", "Theme C"))

+	b.WithSourceFile("themes/c/i18n/en.toml", i18nContent("theme", "Theme C"))

+	b.WithSourceFile("themes/d/i18n/en.toml", i18nContent("theme", "Theme D"))

+	b.WithSourceFile("themes/d/i18n/en.toml", i18nContent("theme2", "Theme2 D"))

+

+	// Static files

+	b.WithSourceFile("themes/c/static/hello.txt", `Hugo Rocks!"`)

+

+	b.Build(BuildCfg{})

+

+	b.AssertFileContent("public/index.html", "|Title theme-a-en|/blog/page/|Content theme-a-en")

+	b.AssertFileContent("public/nn/index.html", "|Title theme-b-nn|/nn/blog/page/|Content theme-b-nn")

+

+	// Data

+	b.AssertFileContent("public/index.html",

+		"Data Common: Project",

+		"Data C: Hugo Rocks!",

+		"Data D: Hugo Rodks!",

+	)

+

+	// i18n

+	b.AssertFileContent("public/index.html",

+		"i18n hello: Project",

+		"i18n theme: Theme C",

+		"i18n theme2: Theme2 D",

+	)

+

+}

+

+func TestModulesIgnoreConfig(t *testing.T) {

+	b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", `

+baseURL="https://example.org"

+

+workingDir="/site"

+

+[module]

+[[module.imports]]

+path="a"

+ignoreConfig=true

+

+`)

+

+	b.WithSourceFile("themes/a/config.toml", `

+[params]

+a = "Should Be Ignored!"

+`)

+

+	b.WithTemplatesAdded("index.html", `Params: {{ .Site.Params }}`)

+

+	b.Build(BuildCfg{})

+

+	b.AssertFileContentFn("public/index.html", func(s string) bool {

+		return !strings.Contains(s, "Ignored")

+	})

+

+}

+

+func TestModulesDisabled(t *testing.T) {

+	b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", `

+baseURL="https://example.org"

+

+workingDir="/site"

+

+[module]

+[[module.imports]]

+path="a"

+[[module.imports]]

+path="b"

+disabled=true

+

+

+`)

+

+	b.WithSourceFile("themes/a/config.toml", `

+[params]

+a = "A param"

+`)

+

+	b.WithSourceFile("themes/b/config.toml", `

+[params]

+b = "B param"

+`)

+

+	b.WithTemplatesAdded("index.html", `Params: {{ .Site.Params }}`)

+

+	b.Build(BuildCfg{})

+

+	b.AssertFileContentFn("public/index.html", func(s string) bool {

+		return strings.Contains(s, "A param") && !strings.Contains(s, "B param")

+	})

+

+}

+

+func TestModulesIncompatible(t *testing.T) {

+	b := newTestSitesBuilder(t).WithWorkingDir("/site").WithConfigFile("toml", `

+baseURL="https://example.org"

+

+workingDir="/site"

+

+[module]

+[[module.imports]]

+path="ok"

+[[module.imports]]

+path="incompat1"

+[[module.imports]]

+path="incompat2"

+

+

+`)

+

+	b.WithSourceFile("themes/ok/data/ok.toml", `title = "OK"`)

+

+	b.WithSourceFile("themes/incompat1/config.toml", `

+

+[module]

+[module.hugoVersion]

+min = "0.33.2"

+max = "0.45.0"

+

+`)

+

+	// Old setup.

+	b.WithSourceFile("themes/incompat2/theme.toml", `

+min_version = "5.0.0"

+

+`)

+

+	logger := loggers.NewWarningLogger()

+	b.WithLogger(logger)

+

+	b.Build(BuildCfg{})

+

+	assert := require.New(t)

+

+	assert.Equal(uint64(2), logger.WarnCounter.Count())

+

+}

+

+func TestModulesSymlinks(t *testing.T) {

+	skipSymlink(t)

+

+	wd, _ := os.Getwd()

+	defer func() {

+		os.Chdir(wd)

+	}()

+

+	assert := require.New(t)

+	// We need to use the OS fs for this.

+	cfg := viper.New()

+	fs := hugofs.NewFrom(hugofs.Os, cfg)

+

+	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mod-sym")

+	assert.NoError(err)

+

+	defer clean()

+

+	const homeTemplate = `

+Data: {{ .Site.Data }}

+`

+

+	createDirsAndFiles := func(baseDir string) {

+		for _, dir := range files.ComponentFolders {

+			realDir := filepath.Join(baseDir, dir, "real")

+			assert.NoError(os.MkdirAll(realDir, 0777))

+			assert.NoError(afero.WriteFile(fs.Source, filepath.Join(realDir, "data.toml"), []byte("[hello]\nother = \"hello\""), 0777))

+		}

+

+		assert.NoError(afero.WriteFile(fs.Source, filepath.Join(baseDir, "layouts", "index.html"), []byte(homeTemplate), 0777))

+	}

+

+	// Create project dirs and files.

+	createDirsAndFiles(workDir)

+	// Create one module inside the default themes folder.

+	themeDir := filepath.Join(workDir, "themes", "mymod")

+	createDirsAndFiles(themeDir)

+

+	createSymlinks := func(baseDir, id string) {

+		for _, dir := range files.ComponentFolders {

+			assert.NoError(os.Chdir(filepath.Join(baseDir, dir)))

+			assert.NoError(os.Symlink("real", fmt.Sprintf("realsym%s", id)))

+			assert.NoError(os.Chdir(filepath.Join(baseDir, dir, "real")))

+			assert.NoError(os.Symlink("data.toml", fmt.Sprintf(filepath.FromSlash("datasym%s.toml"), id)))

+		}

+	}

+

+	createSymlinks(workDir, "project")

+	createSymlinks(themeDir, "mod")

+

+	config := `

+baseURL = "https://example.com"

+theme="mymod"

+defaultContentLanguage="nn"

+defaultContentLanguageInSubDir=true

+

+[languages]

+[languages.nn]

+weight = 1

+[languages.en]

+weight = 2

+

+

+`

+

+	b := newTestSitesBuilder(t).WithNothingAdded().WithWorkingDir(workDir)

+	b.Fs = fs

+

+	b.WithConfigFile("toml", config)

+	assert.NoError(os.Chdir(workDir))

+

+	b.Build(BuildCfg{})

+

+	b.AssertFileContentFn(filepath.Join("public", "en", "index.html"), func(s string) bool {

+		// Symbolic links only followed in project. There should be WARNING logs.

+		return !strings.Contains(s, "symmod") && strings.Contains(s, "symproject")

+	})

+

+	bfs := b.H.BaseFs

+

+	for _, componentFs := range []afero.Fs{

+		bfs.Archetypes.Fs,

+		bfs.Content.Fs,

+		bfs.Data.Fs,

+		bfs.Assets.Fs,

+		bfs.Static[""].Fs,

+		bfs.I18n.Fs} {

+

+		for i, id := range []string{"mod", "project"} {

+

+			statCheck := func(fs afero.Fs, filename string) {

+				shouldFail := i == 0

+				_, err := fs.Stat(filepath.FromSlash(filename))

+				if err != nil {

+					if strings.HasSuffix(filename, "toml") && strings.Contains(err.Error(), "files not supported") {

+						// OK

+						return

+					}

+				}

+				if shouldFail {

+					assert.Error(err)

+					assert.Equal(hugofs.ErrPermissionSymlink, err)

+				} else {

+					assert.NoError(err)

+				}

+			}

+

+			statCheck(componentFs, fmt.Sprintf("realsym%s", id))

+			statCheck(componentFs, fmt.Sprintf("real/datasym%s.toml", id))

+

+		}

+	}

+}

--- a/hugolib/hugo_sites.go

+++ b/hugolib/hugo_sites.go

@@ -22,6 +22,8 @@

 	"strings"

 	"sync"

+	radix "github.com/hashicorp/go-immutable-radix"

+

 	"github.com/gohugoio/hugo/output"

 	"github.com/gohugoio/hugo/parser/metadecoders"

@@ -33,7 +35,6 @@

 	"github.com/bep/gitmap"

 	"github.com/gohugoio/hugo/config"

-	"github.com/spf13/afero"

 	"github.com/gohugoio/hugo/publisher"

@@ -199,10 +200,11 @@

 	return h != nil && h.multihost

 }

-func (h *HugoSites) LanguageSet() map[string]bool {

-	set := make(map[string]bool)

-	for _, s := range h.Sites {

-		set[s.language.Lang] = true

+// TODO(bep) consolidate

+func (h *HugoSites) LanguageSet() map[string]int {

+	set := make(map[string]int)

+	for i, s := range h.Sites {

+		set[s.language.Lang] = i

 	}

 	return set

 }

@@ -222,14 +224,6 @@

 	helpers.ProcessingStatsTable(w, stats...)

 }

-func (h *HugoSites) langSite() map[string]*Site {

-	m := make(map[string]*Site)

-	for _, s := range h.Sites {

-		m[s.language.Lang] = s

-	}

-	return m

-}

-

 // GetContentPage finds a Page with content given the absolute filename.

 // Returns nil if none found.

 func (h *HugoSites) GetContentPage(filename string) page.Page {

@@ -265,7 +259,7 @@

 	langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...)

 	if err != nil {

-		return nil, err

+		return nil, errors.Wrap(err, "failed to create language config")

 	}

 	var contentChangeTracker *contentChangeMap

@@ -288,8 +282,11 @@

 	}

 	h.init.data.Add(func() (interface{}, error) {

-		err := h.loadData(h.PathSpec.BaseFs.Data.Fs)

-		return err, nil

+		err := h.loadData(h.PathSpec.BaseFs.Data.Dirs)

+		if err != nil {

+			return nil, errors.Wrap(err, "failed to load data")

+		}

+		return nil, nil

 	})

 	h.init.translations.Add(func() (interface{}, error) {

@@ -303,7 +300,10 @@

 	h.init.gitInfo.Add(func() (interface{}, error) {

 		err := h.loadGitInfo()

-		return nil, err

+		if err != nil {

+			return nil, errors.Wrap(err, "failed to load Git info")

+		}

+		return nil, nil

 	})

 	for _, s := range sites {

@@ -311,7 +311,7 @@

 	}

 	if err := applyDeps(cfg, sites...); err != nil {

-		return nil, err

+		return nil, errors.Wrap(err, "add site dependencies")

 	}

 	h.Deps = sites[0].Deps

@@ -319,7 +319,12 @@

 	// Only needed in server mode.

 	// TODO(bep) clean up the running vs watching terms

 	if cfg.Running {

-		contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)}

+		contentChangeTracker = &contentChangeMap{

+			pathSpec:      h.PathSpec,

+			symContent:    make(map[string]map[string]bool),

+			leafBundles:   radix.New(),

+			branchBundles: make(map[string]bool),

+		}

 		h.ContentChanges = contentChangeTracker

 	}

@@ -371,7 +376,7 @@

 			siteConfig, err := loadSiteConfig(s.language)

 			if err != nil {

-				return err

+				return errors.Wrap(err, "load site config")

 			}

 			s.siteConfigConfig = siteConfig

 			s.siteRefLinker, err = newSiteRefLinker(s.language, s)

@@ -388,17 +393,17 @@

 			var err error

 			d, err = deps.New(cfg)

 			if err != nil {

-				return err

+				return errors.Wrap(err, "create deps")

 			}

 			d.OutputFormatsConfig = s.outputFormatsConfig

 			if err := onCreated(d); err != nil {

-				return err

+				return errors.Wrap(err, "on created")

 			}

 			if err = d.LoadResources(); err != nil {

-				return err

+				return errors.Wrap(err, "load resources")

 			}

 		} else {

@@ -418,7 +423,7 @@

 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {

 	sites, err := createSitesFromConfig(cfg)

 	if err != nil {

-		return nil, err

+		return nil, errors.Wrap(err, "from config")

 	}

 	return newHugoSites(cfg, sites...)

 }

@@ -800,25 +805,32 @@

 	return h.Sites[0].AllPages()

 }

-func (h *HugoSites) loadData(fs afero.Fs) (err error) {

-	spec := source.NewSourceSpec(h.PathSpec, fs)

-	fileSystem := spec.NewFilesystem("")

+func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) {

+	spec := source.NewSourceSpec(h.PathSpec, nil)

+

 	h.data = make(map[string]interface{})

-	for _, r := range fileSystem.Files() {

-		if err := h.handleDataFile(r); err != nil {

+	for _, fi := range fis {

+		fileSystem := spec.NewFilesystemFromFileMetaInfo(fi)

+		files, err := fileSystem.Files()

+		if err != nil {

 			return err

 		}

+		for _, r := range files {

+			if err := h.handleDataFile(r); err != nil {

+				return err

+			}

+		}

 	}

 	return

 }

-func (h *HugoSites) handleDataFile(r source.ReadableFile) error {

+func (h *HugoSites) handleDataFile(r source.File) error {

 	var current map[string]interface{}

-	f, err := r.Open()

+	f, err := r.FileInfo().Meta().Open()

 	if err != nil {

-		return errors.Wrapf(err, "Failed to open data file %q:", r.LogicalName())

+		return errors.Wrapf(err, "data: failed to open %q:", r.LogicalName())

 	}

 	defer f.Close()

@@ -825,16 +837,13 @@

 	// Crawl in data tree to insert data

 	current = h.data

 	keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator)

-	// The first path element is the virtual folder (typically theme name), which is

-	// not part of the key.

-	if len(keyParts) > 1 {

-		for _, key := range keyParts[1:] {

-			if key != "" {

-				if _, ok := current[key]; !ok {

-					current[key] = make(map[string]interface{})

-				}

-				current = current[key].(map[string]interface{})

+

+	for _, key := range keyParts {

+		if key != "" {

+			if _, ok := current[key]; !ok {

+				current[key] = make(map[string]interface{})

 			}

+			current = current[key].(map[string]interface{})

 		}

 	}

@@ -848,15 +857,10 @@

 	}

 	// filepath.Walk walks the files in lexical order, '/' comes before '.'

-	// this warning could happen if

-	// 1. A theme uses the same key; the main data folder wins

-	// 2. A sub folder uses the same key: the sub folder wins

 	higherPrecedentData := current[r.BaseFileName()]

 	switch data.(type) {

 	case nil:

-		// hear the crickets?

-

 	case map[string]interface{}:

 		switch higherPrecedentData.(type) {

@@ -868,7 +872,11 @@

 			higherPrecedentMap := higherPrecedentData.(map[string]interface{})

 			for key, value := range data.(map[string]interface{}) {

 				if _, exists := higherPrecedentMap[key]; exists {

-					h.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path())

+					// this warning could happen if

+					// 1. A theme uses the same key; the main data folder wins

+					// 2. A sub folder uses the same key: the sub folder wins

+					// TODO(bep) figure out a way to detect 2) above and make that a WARN

+					h.Log.INFO.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path())

 				} else {

 					higherPrecedentMap[key] = value

 				}

@@ -896,12 +904,12 @@

 }

 func (h *HugoSites) errWithFileContext(err error, f source.File) error {

-	rfi, ok := f.FileInfo().(hugofs.RealFilenameInfo)

+	fim, ok := f.FileInfo().(hugofs.FileMetaInfo)

 	if !ok {

 		return err

 	}

-	realFilename := rfi.RealFilename()

+	realFilename := fim.Meta().Filename()

 	err, _ = herrors.WithFileContextForFile(

 		err,

@@ -913,8 +921,8 @@

 	return err

 }

-func (h *HugoSites) readData(f source.ReadableFile) (interface{}, error) {

-	file, err := f.Open()

+func (h *HugoSites) readData(f source.File) (interface{}, error) {

+	file, err := f.FileInfo().Meta().Open()

 	if err != nil {

 		return nil, errors.Wrap(err, "readData: failed to open data file")

 	}

@@ -939,10 +947,15 @@

 // Used in partial reloading to determine if the change is in a bundle.

 type contentChangeMap struct {

-	mu       sync.RWMutex

-	branches []string

-	leafs    []string

+	mu sync.RWMutex

+	// Holds directories with leaf bundles.

+	leafBundles    *radix.Tree

+	leafBundlesTxn *radix.Txn

+

+	// Holds directories with branch bundles.

+	branchBundles map[string]bool

+

 	pathSpec *helpers.PathSpec

 	// Hugo supports symlinked content (both directories and files). This

@@ -950,11 +963,24 @@

 	// locations in /content -- which is really cool, but also means we have to

 	// go an extra mile to handle changes.

 	// This map is only used in watch mode.

-	// It maps either file to files or the real dir to a set of content directories where it is in use.

-	symContent   map[string]map[string]bool

+	// It maps either file to files or the real dir to a set of content directories

+	// where it is in use.

 	symContentMu sync.Mutex

+	symContent   map[string]map[string]bool

 }

+func (m *contentChangeMap) start() {

+	m.mu.Lock()

+	m.leafBundlesTxn = m.leafBundles.Txn()

+	m.mu.Unlock()

+}

+

+func (m *contentChangeMap) stop() {

+	m.mu.Lock()

+	m.leafBundles = m.leafBundlesTxn.Commit()

+	m.mu.Unlock()

+}

+

 func (m *contentChangeMap) add(filename string, tp bundleDirType) {

 	m.mu.Lock()

 	dir := filepath.Dir(filename) + helpers.FilePathSeparator

@@ -961,9 +987,9 @@

 	dir = strings.TrimPrefix(dir, ".")

 	switch tp {

 	case bundleBranch:

-		m.branches = append(m.branches, dir)

+		m.branchBundles[dir] = true

 	case bundleLeaf:

-		m.leafs = append(m.leafs, dir)

+		m.leafBundlesTxn.Insert([]byte(dir), true)

 	default:

 		panic("invalid bundle type")

 	}

@@ -970,59 +996,54 @@

 	m.mu.Unlock()

 }

-// Track the addition of bundle dirs.

-func (m *contentChangeMap) handleBundles(b *bundleDirs) {

-	for _, bd := range b.bundles {

-		m.add(bd.fi.Path(), bd.tp)

-	}

-}

-

-// resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant.

-// It also removes the entry from the map. It will be re-added again by the partial

-// build if it still is a bundle.

 func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) {

 	m.mu.RLock()

 	defer m.mu.RUnlock()

 	// Bundles share resources, so we need to start from the virtual root.

-	relPath := m.pathSpec.RelContentDir(filename)

-	dir, name := filepath.Split(relPath)

+	relFilename := m.pathSpec.RelContentDir(filename)

+	dir, name := filepath.Split(relFilename)

 	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {

 		dir += helpers.FilePathSeparator

 	}

-	fileTp, isContent := classifyBundledFile(name)

-

-	// This may be a member of a bundle. Start with branch bundles, the most specific.

-	if fileTp == bundleBranch || (fileTp == bundleNot && !isContent) {

-		for i, b := range m.branches {

-			if b == dir {

-				m.branches = append(m.branches[:i], m.branches[i+1:]...)

-				return dir, b, bundleBranch

-			}

-		}

+	if _, found := m.branchBundles[dir]; found {

+		delete(m.branchBundles, dir)

+		return dir, dir, bundleBranch

 	}

-	// And finally the leaf bundles, which can contain anything.

-	for i, l := range m.leafs {

-		if strings.HasPrefix(dir, l) {

-			m.leafs = append(m.leafs[:i], m.leafs[i+1:]...)

-			return dir, l, bundleLeaf

-		}

+	if key, _, found := m.leafBundles.Root().LongestPrefix([]byte(dir)); found {

+		m.leafBundlesTxn.Delete(key)

+		dir = string(key)

+		return dir, dir, bundleLeaf

 	}

+	fileTp, isContent := classifyBundledFile(name)

 	if isContent && fileTp != bundleNot {

 		// A new bundle.

 		return dir, dir, fileTp

 	}

-	// Not part of any bundle

 	return dir, filename, bundleNot

+

 }

-func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) {

+func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) {

+	meta := fim.Meta()

+	if !meta.IsSymlink() {

+		return

+	}

 	m.symContentMu.Lock()

+

+	from, to := meta.Filename(), meta.OriginalFilename()

+	if fim.IsDir() {

+		if !strings.HasSuffix(from, helpers.FilePathSeparator) {

+			from += helpers.FilePathSeparator

+		}

+	}

+

 	mm, found := m.symContent[from]

+

 	if !found {

 		mm = make(map[string]bool)

 		m.symContent[from] = mm

@@ -1044,5 +1065,6 @@

 	}

 	sort.Strings(dirs)

+

 	return dirs

 }

--- a/hugolib/hugo_sites_build.go

+++ b/hugolib/hugo_sites_build.go

@@ -22,7 +22,7 @@

 	"github.com/gohugoio/hugo/output"

-	"errors"

+	"github.com/pkg/errors"

 	"github.com/fsnotify/fsnotify"

 	"github.com/gohugoio/hugo/helpers"

@@ -82,11 +82,11 @@

 			if len(events) > 0 {

 				// Rebuild

 				if err := h.initRebuild(conf); err != nil {

-					return err

+					return errors.Wrap(err, "initRebuild")

 				}

 			} else {

 				if err := h.initSites(conf); err != nil {

-					return err

+					return errors.Wrap(err, "initSites")

 				}

 			}

@@ -97,7 +97,7 @@

 			}

 			trace.WithRegion(ctx, "process", f)

 			if err != nil {

-				return err

+				return errors.Wrap(err, "process")

 			}

 			f = func() {

--- a/hugolib/hugo_sites_build_errors_test.go

+++ b/hugolib/hugo_sites_build_errors_test.go

@@ -40,7 +40,6 @@

 }

 func TestSiteBuildErrors(t *testing.T) {

-	t.Parallel()

 	const (

 		yamlcontent = "yamlcontent"

@@ -226,7 +225,9 @@

 	}

 	for _, test := range tests {

+		test := test

 		t.Run(test.name, func(t *testing.T) {

+			t.Parallel()

 			assert := require.New(t)

 			errorAsserter := testSiteBuildErrorAsserter{

 				assert: assert,

--- a/hugolib/hugo_sites_build_test.go

+++ b/hugolib/hugo_sites_build_test.go

@@ -2,10 +2,10 @@

 import (

 	"fmt"

+	"os"

 	"strings"

 	"testing"

-	"os"

 	"path/filepath"

 	"time"

@@ -246,6 +246,8 @@

 	require.Equal(t, "en", enSite.language.Lang)

+	//dumpPages(enSite.RegularPages()...)

+

 	assert.Equal(5, len(enSite.RegularPages()))

 	assert.Equal(32, len(enSite.AllPages()))

@@ -447,7 +449,7 @@

 	require.NotNil(t, homeEn)

 	assert.Len(homeEn.Translations(), 3)

-	contentFs := b.H.BaseFs.Content.Fs

+	contentFs := b.H.Fs.Source

 	for i, this := range []struct {

 		preFunc    func(t *testing.T)

@@ -480,9 +482,9 @@

 		},

 		{

 			func(t *testing.T) {

-				writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "new1.en.md", -5)

-				writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "new2.en.md", -10)

-				writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "new1.fr.md", 10)

+				writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "content/new1.en.md", -5)

+				writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "content/new2.en.md", -10)

+				writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "content/new1.fr.md", 10)

 			},

 			[]fsnotify.Event{

 				{Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Create},

@@ -503,7 +505,7 @@

 		},

 		{

 			func(t *testing.T) {

-				p := "sect/doc1.en.md"

+				p := "content/sect/doc1.en.md"

 				doc1 := readFileFromFs(t, contentFs, p)

 				doc1 += "CHANGED"

 				writeToFs(t, contentFs, p, doc1)

@@ -519,7 +521,7 @@

 		// Rename a file

 		{

 			func(t *testing.T) {

-				if err := contentFs.Rename("new1.en.md", "new1renamed.en.md"); err != nil {

+				if err := contentFs.Rename("content/new1.en.md", "content/new1renamed.en.md"); err != nil {

 					t.Fatalf("Rename failed: %s", err)

 				}

 			},

@@ -672,38 +674,6 @@

 }

-func TestChangeDefaultLanguage(t *testing.T) {

-	t.Parallel()

-

-	assert := require.New(t)

-

-	b := newMultiSiteTestBuilder(t, "", "", map[string]interface{}{

-		"DefaultContentLanguage":         "fr",

-		"DefaultContentLanguageInSubdir": false,

-	})

-	b.CreateSites().Build(BuildCfg{})

-

-	b.AssertFileContent("public/sect/doc1/index.html", "Single", "Bonjour")

-	b.AssertFileContent("public/en/sect/doc2/index.html", "Single", "Hello")

-

-	// Switch language

-	b.WithNewConfigData(map[string]interface{}{

-		"DefaultContentLanguage":         "en",

-		"DefaultContentLanguageInSubdir": false,

-	})

-

-	assert.NoError(b.LoadConfig())

-	err := b.H.Build(BuildCfg{NewConfig: b.Cfg})

-

-	if err != nil {

-		t.Fatalf("Failed to rebuild sites: %s", err)

-	}

-

-	// Default language is now en, so that should now be the "root" language

-	b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Bonjour")

-	b.AssertFileContent("public/sect/doc2/index.html", "Single", "Hello")

-}

-

 // https://github.com/gohugoio/hugo/issues/4706

 func TestContentStressTest(t *testing.T) {

 	b := newTestSitesBuilder(t)

@@ -1261,10 +1231,12 @@

 `

 func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) {

+	t.Helper()

 	writeToFs(t, fs.Source, filename, content)

 }

 func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {

+	t.Helper()

 	if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {

 		t.Fatalf("Failed to write file: %s", err)

 	}

@@ -1271,6 +1243,7 @@

 }

 func readDestination(t testing.TB, fs *hugofs.Fs, filename string) string {

+	t.Helper()

 	return readFileFromFs(t, fs.Destination, filename)

 }

@@ -1287,6 +1260,7 @@

 }

 func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {

+	t.Helper()

 	filename = filepath.Clean(filename)

 	b, err := afero.ReadFile(fs, filename)

 	if err != nil {

@@ -1309,7 +1283,7 @@

 		}

 		helpers.PrintFs(fs, root, os.Stdout)

-		Fatalf(t, "Failed to read file: %s", err)

+		t.Fatalf("Failed to read file: %s", err)

 	}

 	return string(b)

 }

--- a/hugolib/hugo_themes_test.go

+++ /dev/null

@@ -1,268 +1,0 @@

-// Copyright 2018 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugolib

-

-import (

-	"fmt"

-	"os"

-	"path/filepath"

-	"testing"

-

-	"github.com/gohugoio/hugo/common/loggers"

-)

-

-func TestThemesGraph(t *testing.T) {

-	t.Parallel()

-

-	const (

-		themeStandalone = `

-title = "Theme Standalone"

-[params]

-v1 = "v1s"

-v2 = "v2s"

-`

-		themeCyclic = `

-title = "Theme Cyclic"

-theme = "theme3"

-[params]

-v1 = "v1c"

-v2 = "v2c"

-`

-		theme1 = `

-title = "Theme #1"

-theme = "themeStandalone"

-[params]

-v2 = "v21"

-`

-

-		theme2 = `

-title = "Theme #2"

-theme = "theme1"

-[params]

-v1 = "v12"

-`

-

-		theme3 = `

-title = "Theme #3"

-theme = ["theme2", "themeStandalone", "themeCyclic"]

-[params]

-v1 = "v13"

-v2 = "v24"

-`

-

-		theme4 = `

-title = "Theme #4"

-theme = "theme3"

-[params]

-v1 = "v14"

-v2 = "v24"

-`

-

-		site1 = `

-			theme = "theme4"

-			

-			[params]

-			v1 = "site"

-`

-		site2 = `

-			theme = ["theme2", "themeStandalone"]

-`

-	)

-

-	var (

-		testConfigs = []struct {

-			siteConfig string

-

-			// The name of theme somewhere in the middle to write custom key/files.

-			offset string

-

-			check func(b *sitesBuilder)

-		}{

-			{site1, "theme3", func(b *sitesBuilder) {

-

-				// site1: theme4 theme3 theme2 theme1 themeStandalone themeCyclic

-

-				// Check data

-				// theme3 should win the offset competition

-				b.AssertFileContent("public/index.html", "theme1o::[offset][v]theme3", "theme4o::[offset][v]theme3", "themeStandaloneo::[offset][v]theme3")

-				b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme4|[theme1][other]theme1")

-				b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme4|[theme][other]theme4|[theme1][other]theme1|[theme2][other]theme2|[theme3][other]theme3")

-				b.AssertFileContent("public/index.html", "theme1::[inner][other]project|[project][other]project|[theme][other]theme1|[theme1][other]theme1|")

-				b.AssertFileContent("public/index.html", "theme4::[inner][other]project|[project][other]project|[theme][other]theme4|[theme4][other]theme4|")

-

-				// Check layouts

-				b.AssertFileContent("public/index.html", "partial ntheme: theme4", "partial theme2o: theme3")

-

-				// Check i18n

-				b.AssertFileContent("public/index.html", "i18n: project theme4")

-

-				// Check static files

-				// TODO(bep) static files not currently part of the build b.AssertFileContent("public/nproject.txt", "TODO")

-

-				// Check site params

-				b.AssertFileContent("public/index.html", "v1::site", "v2::v24")

-			}},

-			{site2, "", func(b *sitesBuilder) {

-

-				// site2: theme2 theme1 themeStandalone

-				b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|")

-				b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme2|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|")

-				b.AssertFileContent("public/index.html", "i18n: project theme2")

-				b.AssertFileContent("public/index.html", "partial ntheme: theme2")

-

-				// Params only set in themes

-				b.AssertFileContent("public/index.html", "v1::v12", "v2::v21")

-

-			}},

-		}

-

-		themeConfigs = []struct {

-			name   string

-			config string

-		}{

-			{"themeStandalone", themeStandalone},

-			{"themeCyclic", themeCyclic},

-			{"theme1", theme1},

-			{"theme2", theme2},

-			{"theme3", theme3},

-			{"theme4", theme4},

-		}

-	)

-

-	for i, testConfig := range testConfigs {

-		t.Log(fmt.Sprintf("Test %d", i))

-		b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger())

-		b.WithConfigFile("toml", testConfig.siteConfig)

-

-		for _, tc := range themeConfigs {

-			var variationsNameBase = []string{"nproject", "ntheme", tc.name}

-

-			themeRoot := filepath.Join("themes", tc.name)

-			b.WithSourceFile(filepath.Join(themeRoot, "config.toml"), tc.config)

-

-			b.WithSourceFile(filepath.Join("layouts", "partials", "m.html"), `{{- range $k, $v := . }}{{ $k }}::{{ template "printv" $v }}

-{{ end }}	

-{{ define "printv" }}

-{{- $tp := printf "%T" . -}}

-{{- if (strings.HasSuffix $tp "map[string]interface {}") -}}

-{{- range $k, $v := . }}[{{ $k }}]{{ template "printv" $v }}{{ end -}}

-{{- else -}}

-{{- . }}|

-{{- end -}}

-{{ end }}

-`)

-

-			for _, nameVariaton := range variationsNameBase {

-				roots := []string{"", themeRoot}

-

-				for _, root := range roots {

-					name := tc.name

-					if root == "" {

-						name = "project"

-					}

-

-					if nameVariaton == "ntheme" && name == "project" {

-						continue

-					}

-

-					// static

-					b.WithSourceFile(filepath.Join(root, "static", nameVariaton+".txt"), name)

-

-					// layouts

-					if i == 1 {

-						b.WithSourceFile(filepath.Join(root, "layouts", "partials", "theme2o.html"), "Not Set")

-					}

-					b.WithSourceFile(filepath.Join(root, "layouts", "partials", nameVariaton+".html"), name)

-					if root != "" && testConfig.offset == tc.name {

-						for _, tc2 := range themeConfigs {

-							b.WithSourceFile(filepath.Join(root, "layouts", "partials", tc2.name+"o.html"), name)

-						}

-					}

-

-					// i18n + data

-

-					var dataContent string

-					if root == "" {

-						dataContent = fmt.Sprintf(`

-[%s]

-other = %q

-

-[inner]

-other = %q

-

-`, name, name, name)

-					} else {

-						dataContent = fmt.Sprintf(`

-[%s]

-other = %q

-

-[inner]

-other = %q

-

-[theme]

-other = %q

-

-`, name, name, name, name)

-					}

-

-					b.WithSourceFile(filepath.Join(root, "data", nameVariaton+".toml"), dataContent)

-					b.WithSourceFile(filepath.Join(root, "i18n", "en.toml"), dataContent)

-

-					// If an offset is set, duplicate a data key with a winner in the middle.

-					if root != "" && testConfig.offset == tc.name {

-						for _, tc2 := range themeConfigs {

-							dataContent := fmt.Sprintf(`

-[offset]

-v = %q

-`, tc.name)

-							b.WithSourceFile(filepath.Join(root, "data", tc2.name+"o.toml"), dataContent)

-						}

-					}

-				}

-

-			}

-

-		}

-

-		for _, themeConfig := range themeConfigs {

-			b.WithSourceFile(filepath.Join("themes", "config.toml"), themeConfig.config)

-		}

-

-		b.WithContent(filepath.Join("content", "page.md"), `---

-title: "Page"

----

-

-`)

-

-		homeTpl := `

-data: {{ partial "m" .Site.Data }}

-i18n: {{ i18n "inner" }} {{ i18n "theme" }}

-partial ntheme: {{ partial "ntheme" . }}

-partial theme2o: {{ partial "theme2o" . }}

-params: {{ partial "m" .Site.Params }} 

-		

-`

-

-		b.WithTemplates(filepath.Join("layouts", "home.html"), homeTpl)

-

-		b.Build(BuildCfg{})

-

-		var _ = os.Stdout

-

-		//	printFs(b.H.Deps.BaseFs.LayoutsFs, "", os.Stdout)

-		testConfig.check(b)

-

-	}

-

-}

--- a/hugolib/language_content_dir_test.go

+++ b/hugolib/language_content_dir_test.go

@@ -211,8 +211,12 @@

 	_ = os.Stdout

-	b.Build(BuildCfg{})

+	err := b.BuildE(BuildCfg{})

+	//dumpPages(b.H.Sites[1].RegularPages()...)

+

+	assert.NoError(err)

+

 	assert.Equal(3, len(b.H.Sites))

 	enSite := b.H.Sites[0]

@@ -222,7 +226,8 @@

 	b.AssertFileContent("/my/project/public/en/mystatic/file1.yaml", "en")

 	b.AssertFileContent("/my/project/public/nn/mystatic/file1.yaml", "nn")

-	//dumpPages(nnSite.RegularPages...)

+	//dumpPages(nnSite.RegularPages()...)

+

 	assert.Equal(12, len(nnSite.RegularPages()))

 	assert.Equal(13, len(enSite.RegularPages()))

@@ -281,9 +286,9 @@

 	assert.Equal("/en/sect/mybundle/", bundleEn.RelPermalink())

 	assert.Equal("/sv/sect/mybundle/", bundleSv.RelPermalink())

-	assert.Equal(4, len(bundleEn.Resources()))

 	assert.Equal(4, len(bundleNn.Resources()))

 	assert.Equal(4, len(bundleSv.Resources()))

+	assert.Equal(4, len(bundleEn.Resources()))

 	b.AssertFileContent("/my/project/public/en/sect/mybundle/index.html", "image/png: /en/sect/mybundle/logo.png")

 	b.AssertFileContent("/my/project/public/nn/sect/mybundle/index.html", "image/png: /nn/sect/mybundle/logo.png")

--- a/hugolib/menu_test.go

+++ b/hugolib/menu_test.go

@@ -18,8 +18,6 @@

 	"fmt"

-	"github.com/spf13/afero"

-

 	"github.com/stretchr/testify/require"

 )

@@ -45,11 +43,10 @@

 sectionPagesMenu = "sect"

 `

-	th, h := newTestSitesFromConfig(

-		t,

-		afero.NewMemMapFs(),

-		siteConfig,

-		"layouts/partials/menu.html",

+	b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)

+

+	b.WithTemplates(

+		"partials/menu.html",

 		`{{- $p := .page -}}

 {{- $m := .menu -}}

 {{ range (index $p.Site.Menus $m) -}}

@@ -58,29 +55,26 @@

 {{- if $p.HasMenuCurrent $m . }}HasMenuCurrent{{ else }}-{{ end -}}|

 {{- end -}}

 `,

-		"layouts/_default/single.html",

+		"_default/single.html",

 		`Single|{{ .Title }}

 Menu Sect:  {{ partial "menu.html" (dict "page" . "menu" "sect") }}

 Menu Main:  {{ partial "menu.html" (dict "page" . "menu" "main") }}`,

-		"layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}",

+		"_default/list.html", "List|{{ .Title }}|{{ .Content }}",

 	)

-	require.Len(t, h.Sites, 1)

-	fs := th.Fs

+	b.WithContent(

+		"sect1/p1.md", fmt.Sprintf(menuPageTemplate, "p1", 1, "main", "atitle1", 40),

+		"sect1/p2.md", fmt.Sprintf(menuPageTemplate, "p2", 2, "main", "atitle2", 30),

+		"sect2/p3.md", fmt.Sprintf(menuPageTemplate, "p3", 3, "main", "atitle3", 20),

+		"sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10),

+		"sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5),

+		"sect1/_index.md", newTestPage("Section One", "2017-01-01", 100),

+		"sect5/_index.md", newTestPage("Section Five", "2017-01-01", 10),

+	)

-	writeSource(t, fs, "content/sect1/p1.md", fmt.Sprintf(menuPageTemplate, "p1", 1, "main", "atitle1", 40))

-	writeSource(t, fs, "content/sect1/p2.md", fmt.Sprintf(menuPageTemplate, "p2", 2, "main", "atitle2", 30))

-	writeSource(t, fs, "content/sect2/p3.md", fmt.Sprintf(menuPageTemplate, "p3", 3, "main", "atitle3", 20))

-	writeSource(t, fs, "content/sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10))

-	writeSource(t, fs, "content/sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5))

+	b.Build(BuildCfg{})

+	h := b.H

-	writeNewContentFile(t, fs.Source, "Section One", "2017-01-01", "content/sect1/_index.md", 100)

-	writeNewContentFile(t, fs.Source, "Section Five", "2017-01-01", "content/sect5/_index.md", 10)

-

-	err := h.Build(BuildCfg{})

-

-	require.NoError(t, err)

-

 	s := h.Sites[0]

 	require.Len(t, s.Menus(), 2)

@@ -90,7 +84,7 @@

 	// There is only one menu in the page, but it is "member of" 2

 	require.Len(t, p1, 1)

-	th.assertFileContent("public/sect1/p1/index.html", "Single",

+	b.AssertFileContent("public/sect1/p1/index.html", "Single",

 		"Menu Sect:  "+

 			"/sect5/|Section Five|Section Five|10|-|-|"+

 			"/sect1/|Section One|Section One|100|-|HasMenuCurrent|"+

@@ -104,7 +98,7 @@

 			"/sect1/p1/|p1|atitle1|40|IsMenuCurrent|-|",

 	)

-	th.assertFileContent("public/sect2/p3/index.html", "Single",

+	b.AssertFileContent("public/sect2/p3/index.html", "Single",

 		"Menu Sect:  "+

 			"/sect5/|Section Five|Section Five|10|-|-|"+

 			"/sect1/|Section One|Section One|100|-|-|"+

--- a/hugolib/multilingual.go

+++ b/hugolib/multilingual.go

@@ -16,17 +16,11 @@

 import (

 	"sync"

-	"github.com/gohugoio/hugo/common/maps"

-

-	"sort"

-

 	"errors"

-	"fmt"

 	"github.com/gohugoio/hugo/langs"

 	"github.com/gohugoio/hugo/config"

-	"github.com/spf13/cast"

 )

 // Multilingual manages the all languages used in a multilingual site.

@@ -87,54 +81,4 @@

 		return false

 	}

 	return s.h.multilingual != nil && s.h.multilingual.enabled()

-}

-

-func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (langs.Languages, error) {

-	languages := make(langs.Languages, len(l))

-	i := 0

-

-	for lang, langConf := range l {

-		langsMap, err := cast.ToStringMapE(langConf)

-

-		if err != nil {

-			return nil, fmt.Errorf("Language config is not a map: %T", langConf)

-		}

-

-		language := langs.NewLanguage(lang, cfg)

-

-		for loki, v := range langsMap {

-			switch loki {

-			case "title":

-				language.Title = cast.ToString(v)

-			case "languagename":

-				language.LanguageName = cast.ToString(v)

-			case "weight":

-				language.Weight = cast.ToInt(v)

-			case "contentdir":

-				language.ContentDir = cast.ToString(v)

-			case "disabled":

-				language.Disabled = cast.ToBool(v)

-			case "params":

-				m := cast.ToStringMap(v)

-				// Needed for case insensitive fetching of params values

-				maps.ToLower(m)

-				for k, vv := range m {

-					language.SetParam(k, vv)

-				}

-			}

-

-			// Put all into the Params map

-			language.SetParam(loki, v)

-

-			// Also set it in the configuration map (for baseURL etc.)

-			language.Set(loki, v)

-		}

-

-		languages[i] = language

-		i++

-	}

-

-	sort.Sort(languages)

-

-	return languages, nil

 }

--- a/hugolib/page.go

+++ b/hugolib/page.go

@@ -23,6 +23,8 @@

 	"sort"

 	"strings"

+	"github.com/gohugoio/hugo/hugofs/files"

+

 	"github.com/bep/gitmap"

 	"github.com/gohugoio/hugo/helpers"

@@ -290,7 +292,9 @@

 		switch p.Kind() {

 		case page.KindSection:

-			section = sections[0]

+			if len(sections) > 0 {

+				section = sections[0]

+			}

 		case page.KindTaxonomyTerm:

 			section = p.getTaxonomyNodeInfo().singular

 		case page.KindTaxonomy:

@@ -365,6 +369,7 @@

 		var toBeDeleted []int

 		for i, r := range p.Resources() {

+

 			if _, ok := r.(page.Page); ok {

 				// Pages gets rendered with the owning page but we count them here.

 				p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages)

@@ -491,14 +496,6 @@

 	p.parent.subSections = append(p.parent.subSections, p)

 }

-func (p *pageState) contentMarkupType() string {

-	if p.m.markup != "" {

-		return p.m.markup

-

-	}

-	return p.File().Ext()

-}

-

 func (p *pageState) mapContent(meta *pageMeta) error {

 	s := p.shortcodeState

@@ -843,6 +840,7 @@

 func (s *Site) sectionsFromFile(fi source.File) []string {

 	dirname := fi.Dir()

+

 	dirname = strings.Trim(dirname, helpers.FilePathSeparator)

 	if dirname == "" {

 		return nil

@@ -850,7 +848,7 @@

 	parts := strings.Split(dirname, helpers.FilePathSeparator)

 	if fii, ok := fi.(*fileInfo); ok {

-		if fii.bundleTp == bundleLeaf && len(parts) > 0 {

+		if len(parts) > 0 && fii.FileInfo().Meta().Classifier() == files.ContentClassLeaf {

 			// my-section/mybundle/index.md => my-section

 			return parts[:len(parts)-1]

 		}

--- a/hugolib/page__meta.go

+++ b/hugolib/page__meta.go

@@ -21,6 +21,8 @@

 	"strings"

 	"time"

+	"github.com/gohugoio/hugo/hugofs/files"

+

 	"github.com/gohugoio/hugo/common/hugo"

 	"github.com/gohugoio/hugo/related"

@@ -591,15 +593,14 @@

 	}

 	if p.IsNode() {

-		p.bundleType = "branch"

+		p.bundleType = files.ContentClassBranch

 	} else {

 		source := p.File()

 		if fi, ok := source.(*fileInfo); ok {

-			switch fi.bundleTp {

-			case bundleBranch:

-				p.bundleType = "branch"

-			case bundleLeaf:

-				p.bundleType = "leaf"

+			class := fi.FileInfo().Meta().Classifier()

+			switch class {

+			case files.ContentClassBranch, files.ContentClassLeaf:

+				p.bundleType = class

 			}

 		}

 	}

--- a/hugolib/page_permalink_test.go

+++ b/hugolib/page_permalink_test.go

@@ -63,8 +63,9 @@

 	}

 	for i, test := range tests {

+		test := test

 		t.Run(fmt.Sprintf("%s-%d", test.file, i), func(t *testing.T) {

-

+			t.Parallel()

 			cfg, fs := newTestCfg()

 			cfg.Set("uglyURLs", test.uglyURLs)

--- a/hugolib/page_test.go

+++ b/hugolib/page_test.go

@@ -18,6 +18,8 @@

 	"html/template"

 	"os"

+	"github.com/gohugoio/hugo/config"

+

 	"github.com/gohugoio/hugo/common/loggers"

 	"path/filepath"

@@ -29,7 +31,6 @@

 	"github.com/gohugoio/hugo/resources/page"

 	"github.com/gohugoio/hugo/resources/resource"

-

 	"github.com/spf13/afero"

 	"github.com/spf13/viper"

@@ -301,6 +302,7 @@

 }

 func checkPageContent(t *testing.T, page page.Page, expected string, msg ...interface{}) {

+	t.Helper()

 	a := normalizeContent(expected)

 	b := normalizeContent(content(page))

 	if a != b {

@@ -387,11 +389,13 @@

 			continue

 		}

-		cfg, fs := newTestCfg()

+		cfg, fs := newTestCfg(func(cfg config.Provider) error {

+			for k, v := range settings {

+				cfg.Set(k, v)

+			}

+			return nil

-		for k, v := range settings {

-			cfg.Set(k, v)

-		}

+		})

 		contentDir := "content"

@@ -413,8 +417,11 @@

 		homePath := fmt.Sprintf("_index.%s", e.ext)

 		writeSource(t, fs, filepath.Join(contentDir, homePath), homePage)

-		s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true})

+		b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded()

+		b.Build(BuildCfg{SkipRender: true})

+		s := b.H.Sites[0]

+

 		require.Len(t, s.RegularPages(), len(pageSources))

 		assertFunc(t, e.ext, s.RegularPages())

@@ -770,6 +777,9 @@

 	fs := hugofs.NewFrom(hugofs.Os, cfg)

 	fs.Destination = &afero.MemMapFs{}

+	wd, err := os.Getwd()

+	assrt.NoError(err)

+

 	cfg.Set("frontmatter", map[string]interface{}{

 		"lastmod": []string{":git", "lastmod"},

 	})

@@ -791,20 +801,15 @@

 	cfg.Set("languages", langConfig)

 	cfg.Set("enableGitInfo", true)

-	assrt.NoError(loadDefaultSettingsFor(cfg))

-	assrt.NoError(loadLanguageSettings(cfg, nil))

-

-	wd, err := os.Getwd()

-	assrt.NoError(err)

 	cfg.Set("workingDir", filepath.Join(wd, "testsite"))

-	h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})

+	b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded()

-	assrt.NoError(err)

+	b.Build(BuildCfg{SkipRender: true})

+	h := b.H

+

 	assrt.Len(h.Sites, 2)

-	require.NoError(t, h.Build(BuildCfg{SkipRender: true}))

-

 	enSite := h.Sites[0]

 	assrt.Len(enSite.RegularPages(), 1)

@@ -820,10 +825,10 @@

 }

 func TestPageWithFrontMatterConfig(t *testing.T) {

-	t.Parallel()

-

 	for _, dateHandler := range []string{":filename", ":fileModTime"} {

+		dateHandler := dateHandler

 		t.Run(fmt.Sprintf("dateHandler=%q", dateHandler), func(t *testing.T) {

+			t.Parallel()

 			assrt := require.New(t)

 			cfg, fs := newTestCfg()

@@ -852,8 +857,10 @@

 			c2fi, err := fs.Source.Stat(c2)

 			assrt.NoError(err)

-			s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true})

+			b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded()

+			b.Build(BuildCfg{SkipRender: true})

+			s := b.H.Sites[0]

 			assrt.Len(s.RegularPages(), 2)

 			noSlug := s.RegularPages()[0]

@@ -1051,10 +1058,8 @@

 	for _, enableEmoji := range []bool{true, false} {

 		v := viper.New()

 		v.Set("enableEmoji", enableEmoji)

-		b := newTestSitesBuilder(t)

-		b.WithViper(v)

-		b.WithSimpleConfigFile()

+		b := newTestSitesBuilder(t).WithViper(v)

 		b.WithContent("page-emoji.md", `---

 title: "Hugo Smile"

@@ -1329,11 +1334,12 @@

 // "dot" in path: #1885 and #2110

 // disablePathToLower regression: #3374

 func TestPathIssues(t *testing.T) {

-	t.Parallel()

 	for _, disablePathToLower := range []bool{false, true} {

 		for _, uglyURLs := range []bool{false, true} {

+			disablePathToLower := disablePathToLower

+			uglyURLs := uglyURLs

 			t.Run(fmt.Sprintf("disablePathToLower=%t,uglyURLs=%t", disablePathToLower, uglyURLs), func(t *testing.T) {

-

+				t.Parallel()

 				cfg, fs := newTestCfg()

 				th := testHelper{cfg, fs, t}

--- a/hugolib/pagebundler.go

+++ /dev/null

@@ -1,206 +1,0 @@

-// Copyright 2019 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugolib

-

-import (

-	"context"

-	"fmt"

-	"math"

-	"path/filepath"

-

-	"github.com/gohugoio/hugo/config"

-

-	_errors "github.com/pkg/errors"

-

-	"golang.org/x/sync/errgroup"

-)

-

-type siteContentProcessor struct {

-	site *Site

-

-	handleContent contentHandler

-

-	ctx context.Context

-

-	// The input file bundles.

-	fileBundlesChan chan *bundleDir

-

-	// The input file singles.

-	fileSinglesChan chan *fileInfo

-

-	// These assets should be just copied to destination.

-	fileAssetsChan chan pathLangFile

-

-	numWorkers int

-

-	// The output Pages

-	pagesChan chan *pageState

-

-	// Used for partial rebuilds (aka. live reload)

-	// Will signal replacement of pages in the site collection.

-	partialBuild bool

-}

-

-func (s *siteContentProcessor) processBundle(b *bundleDir) {

-	select {

-	case s.fileBundlesChan <- b:

-	case <-s.ctx.Done():

-	}

-}

-

-func (s *siteContentProcessor) processSingle(fi *fileInfo) {

-	select {

-	case s.fileSinglesChan <- fi:

-	case <-s.ctx.Done():

-	}

-}

-

-func (s *siteContentProcessor) processAsset(asset pathLangFile) {

-	select {

-	case s.fileAssetsChan <- asset:

-	case <-s.ctx.Done():

-	}

-}

-

-func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor {

-	numWorkers := config.GetNumWorkerMultiplier() * 3

-

-	numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites))))

-

-	return &siteContentProcessor{

-		ctx:             ctx,

-		partialBuild:    partialBuild,

-		site:            s,

-		handleContent:   newHandlerChain(s),

-		fileBundlesChan: make(chan *bundleDir, numWorkers),

-		fileSinglesChan: make(chan *fileInfo, numWorkers),

-		fileAssetsChan:  make(chan pathLangFile, numWorkers),

-		numWorkers:      numWorkers,

-		pagesChan:       make(chan *pageState, numWorkers),

-	}

-}

-

-func (s *siteContentProcessor) closeInput() {

-	close(s.fileSinglesChan)

-	close(s.fileBundlesChan)

-	close(s.fileAssetsChan)

-}

-

-func (s *siteContentProcessor) process(ctx context.Context) error {

-	g1, ctx := errgroup.WithContext(ctx)

-	g2, ctx := errgroup.WithContext(ctx)

-

-	// There can be only one of these per site.

-	g1.Go(func() error {

-		for p := range s.pagesChan {

-			if p.s != s.site {

-				panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s))

-			}

-

-			p.forceRender = s.partialBuild

-

-			if p.forceRender {

-				s.site.replacePage(p)

-			} else {

-				s.site.addPage(p)

-			}

-		}

-		return nil

-	})

-

-	for i := 0; i < s.numWorkers; i++ {

-		g2.Go(func() error {

-			for {

-				select {

-				case f, ok := <-s.fileSinglesChan:

-					if !ok {

-						return nil

-					}

-

-					err := s.readAndConvertContentFile(f)

-					if err != nil {

-						return err

-					}

-				case <-ctx.Done():

-					return ctx.Err()

-				}

-			}

-		})

-

-		g2.Go(func() error {

-			for {

-				select {

-				case file, ok := <-s.fileAssetsChan:

-					if !ok {

-						return nil

-					}

-					f, err := s.site.BaseFs.Content.Fs.Open(file.Filename())

-					if err != nil {

-						return _errors.Wrap(err, "failed to open assets file")

-					}

-					filename := filepath.Join(s.site.GetTargetLanguageBasePath(), file.Path())

-					err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, filename, f)

-					f.Close()

-					if err != nil {

-						return err

-					}

-				case <-ctx.Done():

-					return ctx.Err()

-				}

-			}

-		})

-

-		g2.Go(func() error {

-			for {

-				select {

-				case bundle, ok := <-s.fileBundlesChan:

-					if !ok {

-						return nil

-					}

-					err := s.readAndConvertContentBundle(bundle)

-					if err != nil {

-						return err

-					}

-				case <-ctx.Done():

-					return ctx.Err()

-				}

-			}

-		})

-	}

-

-	err := g2.Wait()

-

-	close(s.pagesChan)

-

-	if err != nil {

-		return err

-	}

-

-	if err := g1.Wait(); err != nil {

-		return err

-	}

-

-	return nil

-

-}

-

-func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error {

-	ctx := &handlerContext{source: file, pages: s.pagesChan}

-	return s.handleContent(ctx).err

-}

-

-func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error {

-	ctx := &handlerContext{bundle: bundle, pages: s.pagesChan}

-	return s.handleContent(ctx).err

-}

--- a/hugolib/pagebundler_capture.go

+++ /dev/null

@@ -1,773 +1,0 @@

-// Copyright 2017-present The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugolib

-

-import (

-	"errors"

-	"fmt"

-	"os"

-	"path"

-	"path/filepath"

-

-	"github.com/gohugoio/hugo/config"

-

-	"github.com/gohugoio/hugo/common/loggers"

-	_errors "github.com/pkg/errors"

-

-	"sort"

-	"strings"

-	"sync"

-

-	"github.com/spf13/afero"

-

-	"github.com/gohugoio/hugo/hugofs"

-

-	"github.com/gohugoio/hugo/helpers"

-

-	"golang.org/x/sync/errgroup"

-

-	"github.com/gohugoio/hugo/source"

-)

-

-var errSkipCyclicDir = errors.New("skip potential cyclic dir")

-

-type capturer struct {

-	// To prevent symbolic link cycles: Visit same folder only once.

-	seen   map[string]bool

-	seenMu sync.Mutex

-

-	handler captureResultHandler

-

-	sourceSpec *source.SourceSpec

-	fs         afero.Fs

-	logger     *loggers.Logger

-

-	// Filenames limits the content to process to a list of filenames/directories.

-	// This is used for partial building in server mode.

-	filenames []string

-

-	// Used to determine how to handle content changes in server mode.

-	contentChanges *contentChangeMap

-

-	// Semaphore used to throttle the concurrent sub directory handling.

-	sem chan bool

-}

-

-func newCapturer(

-	logger *loggers.Logger,

-	sourceSpec *source.SourceSpec,

-	handler captureResultHandler,

-	contentChanges *contentChangeMap,

-	filenames ...string) *capturer {

-

-	numWorkers := config.GetNumWorkerMultiplier()

-

-	// TODO(bep) the "index" vs "_index" check/strings should be moved in one place.

-	isBundleHeader := func(filename string) bool {

-		base := filepath.Base(filename)

-		name := helpers.Filename(base)

-		return IsContentFile(base) && (name == "index" || name == "_index")

-	}

-

-	// Make sure that any bundle header files are processed before the others. This makes

-	// sure that any bundle head is processed before its resources.

-	sort.Slice(filenames, func(i, j int) bool {

-		a, b := filenames[i], filenames[j]

-		ac, bc := isBundleHeader(a), isBundleHeader(b)

-

-		if ac {

-			return true

-		}

-

-		if bc {

-			return false

-		}

-

-		return a < b

-	})

-

-	c := &capturer{

-		sem:            make(chan bool, numWorkers),

-		handler:        handler,

-		sourceSpec:     sourceSpec,

-		fs:             sourceSpec.SourceFs,

-		logger:         logger,

-		contentChanges: contentChanges,

-		seen:           make(map[string]bool),

-		filenames:      filenames}

-

-	return c

-}

-

-// Captured files and bundles ready to be processed will be passed on to

-// these channels.

-type captureResultHandler interface {

-	handleSingles(fis ...*fileInfo)

-	handleCopyFile(fi pathLangFile)

-	captureBundlesHandler

-}

-

-type captureBundlesHandler interface {

-	handleBundles(b *bundleDirs)

-}

-

-type captureResultHandlerChain struct {

-	handlers []captureBundlesHandler

-}

-

-func (c *captureResultHandlerChain) handleSingles(fis ...*fileInfo) {

-	for _, h := range c.handlers {

-		if hh, ok := h.(captureResultHandler); ok {

-			hh.handleSingles(fis...)

-		}

-	}

-}

-func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) {

-	for _, h := range c.handlers {

-		h.handleBundles(b)

-	}

-}

-

-func (c *captureResultHandlerChain) handleCopyFile(file pathLangFile) {

-	for _, h := range c.handlers {

-		if hh, ok := h.(captureResultHandler); ok {

-			hh.handleCopyFile(file)

-		}

-	}

-}

-

-func (c *capturer) capturePartial(filenames ...string) error {

-	handled := make(map[string]bool)

-

-	for _, filename := range filenames {

-		dir, resolvedFilename, tp := c.contentChanges.resolveAndRemove(filename)

-		if handled[resolvedFilename] {

-			continue

-		}

-

-		handled[resolvedFilename] = true

-

-		switch tp {

-		case bundleLeaf:

-			if err := c.handleDir(resolvedFilename); err != nil {

-				// Directory may have been deleted.

-				if !os.IsNotExist(err) {

-					return err

-				}

-			}

-		case bundleBranch:

-			if err := c.handleBranchDir(resolvedFilename); err != nil {

-				// Directory may have been deleted.

-				if !os.IsNotExist(err) {

-					return err

-				}

-			}

-		default:

-			fi, err := c.resolveRealPath(resolvedFilename)

-			if os.IsNotExist(err) {

-				// File has been deleted.

-				continue

-			}

-

-			// Just in case the owning dir is a new symlink -- this will

-			// create the proper mapping for it.

-			c.resolveRealPath(dir)

-

-			f, active := c.newFileInfo(fi, tp)

-			if active {

-				c.copyOrHandleSingle(f)

-			}

-		}

-	}

-

-	return nil

-}

-

-func (c *capturer) capture() error {

-	if len(c.filenames) > 0 {

-		return c.capturePartial(c.filenames...)

-	}

-

-	err := c.handleDir(helpers.FilePathSeparator)

-	if err != nil {

-		return err

-	}

-

-	return nil

-}

-

-func (c *capturer) handleNestedDir(dirname string) error {

-	select {

-	case c.sem <- true:

-		var g errgroup.Group

-

-		g.Go(func() error {

-			defer func() {

-				<-c.sem

-			}()

-			return c.handleDir(dirname)

-		})

-		return g.Wait()

-	default:

-		// For deeply nested file trees, waiting for a semaphore wil deadlock.

-		return c.handleDir(dirname)

-	}

-}

-

-// This handles a bundle branch and its resources only. This is used

-// in server mode on changes. If this dir does not (anymore) represent a bundle

-// branch, the handling is upgraded to the full handleDir method.

-func (c *capturer) handleBranchDir(dirname string) error {

-	files, err := c.readDir(dirname)

-	if err != nil {

-

-		return err

-	}

-

-	var (

-		dirType bundleDirType

-	)

-

-	for _, fi := range files {

-		if !fi.IsDir() {

-			tp, _ := classifyBundledFile(fi.RealName())

-			if dirType == bundleNot {

-				dirType = tp

-			}

-

-			if dirType == bundleLeaf {

-				return c.handleDir(dirname)

-			}

-		}

-	}

-

-	if dirType != bundleBranch {

-		return c.handleDir(dirname)

-	}

-

-	dirs := newBundleDirs(bundleBranch, c)

-

-	var secondPass []*fileInfo

-

-	// Handle potential bundle headers first.

-	for _, fi := range files {

-		if fi.IsDir() {

-			continue

-		}

-

-		tp, isContent := classifyBundledFile(fi.RealName())

-

-		f, active := c.newFileInfo(fi, tp)

-

-		if !active {

-			continue

-		}

-

-		if !f.isOwner() {

-			if !isContent {

-				// This is a partial update -- we only care about the files that

-				// is in this bundle.

-				secondPass = append(secondPass, f)

-			}

-			continue

-		}

-		dirs.addBundleHeader(f)

-	}

-

-	for _, f := range secondPass {

-		dirs.addBundleFiles(f)

-	}

-

-	c.handler.handleBundles(dirs)

-

-	return nil

-

-}

-

-func (c *capturer) handleDir(dirname string) error {

-

-	files, err := c.readDir(dirname)

-	if err != nil {

-		return err

-	}

-

-	type dirState int

-

-	const (

-		dirStateDefault dirState = iota

-

-		dirStateAssetsOnly

-		dirStateSinglesOnly

-	)

-

-	var (

-		fileBundleTypes = make([]bundleDirType, len(files))

-

-		// Start with the assumption that this dir contains only non-content assets (images etc.)

-		// If that is still true after we had a first look at the list of files, we

-		// can just copy the files to destination. We will still have to look at the

-		// sub-folders for potential bundles.

-		state = dirStateAssetsOnly

-

-		// Start with the assumption that this dir is not a bundle.

-		// A directory is a bundle if it contains a index content file,

-		// e.g. index.md (a leaf bundle) or a _index.md (a branch bundle).

-		bundleType = bundleNot

-	)

-

-	/* First check for any content files.

-	- If there are none, then this is a assets folder only (images etc.)

-	and we can just plainly copy them to

-	destination.

-	- If this is a section with no image etc. or similar, we can just handle it

-	as it was a single content file.

-	*/

-	var hasNonContent, isBranch bool

-

-	for i, fi := range files {

-		if !fi.IsDir() {

-			tp, isContent := classifyBundledFile(fi.RealName())

-

-			fileBundleTypes[i] = tp

-			if !isBranch {

-				isBranch = tp == bundleBranch

-			}

-

-			if isContent {

-				// This is not a assets-only folder.

-				state = dirStateDefault

-			} else {

-				hasNonContent = true

-			}

-		}

-	}

-

-	if isBranch && !hasNonContent {

-		// This is a section or similar with no need for any bundle handling.

-		state = dirStateSinglesOnly

-	}

-

-	if state > dirStateDefault {

-		return c.handleNonBundle(dirname, files, state == dirStateSinglesOnly)

-	}

-

-	var fileInfos = make([]*fileInfo, 0, len(files))

-

-	for i, fi := range files {

-

-		currentType := bundleNot

-

-		if !fi.IsDir() {

-			currentType = fileBundleTypes[i]

-			if bundleType == bundleNot && currentType != bundleNot {

-				bundleType = currentType

-			}

-		}

-

-		if bundleType == bundleNot && currentType != bundleNot {

-			bundleType = currentType

-		}

-

-		f, active := c.newFileInfo(fi, currentType)

-

-		if !active {

-			continue

-		}

-

-		fileInfos = append(fileInfos, f)

-	}

-

-	var todo []*fileInfo

-

-	if bundleType != bundleLeaf {

-		for _, fi := range fileInfos {

-			if fi.FileInfo().IsDir() {

-				// Handle potential nested bundles.

-				if err := c.handleNestedDir(fi.Path()); err != nil {

-					return err

-				}

-			} else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) {

-				// Not in a bundle.

-				c.copyOrHandleSingle(fi)

-			} else {

-				// This is a section folder or similar with non-content files in it.

-				todo = append(todo, fi)

-			}

-		}

-	} else {

-		todo = fileInfos

-	}

-

-	if len(todo) == 0 {

-		return nil

-	}

-

-	dirs, err := c.createBundleDirs(todo, bundleType)

-	if err != nil {

-		return err

-	}

-

-	// Send the bundle to the next step in the processor chain.

-	c.handler.handleBundles(dirs)

-

-	return nil

-}

-

-func (c *capturer) handleNonBundle(

-	dirname string,

-	fileInfos pathLangFileFis,

-	singlesOnly bool) error {

-

-	for _, fi := range fileInfos {

-		if fi.IsDir() {

-			if err := c.handleNestedDir(fi.Filename()); err != nil {

-				return err

-			}

-		} else {

-			if singlesOnly {

-				f, active := c.newFileInfo(fi, bundleNot)

-				if !active {

-					continue

-				}

-				c.handler.handleSingles(f)

-			} else {

-				c.handler.handleCopyFile(fi)

-			}

-		}

-	}

-

-	return nil

-}

-

-func (c *capturer) copyOrHandleSingle(fi *fileInfo) {

-	if fi.isContentFile() {

-		c.handler.handleSingles(fi)

-	} else {

-		// These do not currently need any further processing.

-		c.handler.handleCopyFile(fi)

-	}

-}

-

-func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirType) (*bundleDirs, error) {

-	dirs := newBundleDirs(bundleType, c)

-

-	for _, fi := range fileInfos {

-		if fi.FileInfo().IsDir() {

-			var collector func(fis ...*fileInfo)

-

-			if bundleType == bundleBranch {

-				// All files in the current directory are part of this bundle.

-				// Trying to include sub folders in these bundles are filled with ambiguity.

-				collector = func(fis ...*fileInfo) {

-					for _, fi := range fis {

-						c.copyOrHandleSingle(fi)

-					}

-				}

-			} else {

-				// All nested files and directories are part of this bundle.

-				collector = func(fis ...*fileInfo) {

-					fileInfos = append(fileInfos, fis...)

-				}

-			}

-			err := c.collectFiles(fi.Path(), collector)

-			if err != nil {

-				return nil, err

-			}

-

-		} else if fi.isOwner() {

-			// There can be more than one language, so:

-			// 1. Content files must be attached to its language's bundle.

-			// 2. Other files must be attached to all languages.

-			// 3. Every content file needs a bundle header.

-			dirs.addBundleHeader(fi)

-		}

-	}

-

-	for _, fi := range fileInfos {

-		if fi.FileInfo().IsDir() || fi.isOwner() {

-			continue

-		}

-

-		if fi.isContentFile() {

-			if bundleType != bundleBranch {

-				dirs.addBundleContentFile(fi)

-			}

-		} else {

-			dirs.addBundleFiles(fi)

-		}

-	}

-

-	return dirs, nil

-}

-

-func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error {

-

-	filesInDir, err := c.readDir(dirname)

-	if err != nil {

-		return err

-	}

-

-	for _, fi := range filesInDir {

-		if fi.IsDir() {

-			err := c.collectFiles(fi.Filename(), handleFiles)

-			if err != nil {

-				return err

-			}

-		} else {

-			f, active := c.newFileInfo(fi, bundleNot)

-			if active {

-				handleFiles(f)

-			}

-		}

-	}

-

-	return nil

-}

-

-func (c *capturer) readDir(dirname string) (pathLangFileFis, error) {

-	if c.sourceSpec.IgnoreFile(dirname) {

-		return nil, nil

-	}

-

-	dir, err := c.fs.Open(dirname)

-	if err != nil {

-		return nil, err

-	}

-	defer dir.Close()

-	fis, err := dir.Readdir(-1)

-	if err != nil {

-		return nil, err

-	}

-

-	pfis := make(pathLangFileFis, 0, len(fis))

-

-	for _, fi := range fis {

-		fip := fi.(pathLangFileFi)

-

-		if !c.sourceSpec.IgnoreFile(fip.Filename()) {

-

-			err := c.resolveRealPathIn(fip)

-

-			if err != nil {

-				// It may have been deleted in the meantime.

-				if err == errSkipCyclicDir || os.IsNotExist(err) {

-					continue

-				}

-				return nil, err

-			}

-

-			pfis = append(pfis, fip)

-		}

-	}

-

-	return pfis, nil

-}

-

-func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) {

-	f := newFileInfo(c.sourceSpec, "", "", fi, tp)

-	return f, !f.disabled

-}

-

-type pathLangFile interface {

-	hugofs.LanguageAnnouncer

-	hugofs.FilePather

-}

-

-type pathLangFileFi interface {

-	os.FileInfo

-	pathLangFile

-}

-

-type pathLangFileFis []pathLangFileFi

-

-type bundleDirs struct {

-	tp bundleDirType

-	// Maps languages to bundles.

-	bundles map[string]*bundleDir

-

-	// Keeps track of language overrides for non-content files, e.g. logo.en.png.

-	langOverrides map[string]bool

-

-	c *capturer

-}

-

-func newBundleDirs(tp bundleDirType, c *capturer) *bundleDirs {

-	return &bundleDirs{tp: tp, bundles: make(map[string]*bundleDir), langOverrides: make(map[string]bool), c: c}

-}

-

-type bundleDir struct {

-	tp bundleDirType

-	fi *fileInfo

-

-	resources map[string]*fileInfo

-}

-

-func (b bundleDir) clone() *bundleDir {

-	b.resources = make(map[string]*fileInfo)

-	fic := *b.fi

-	b.fi = &fic

-	return &b

-}

-

-func newBundleDir(fi *fileInfo, bundleType bundleDirType) *bundleDir {

-	return &bundleDir{fi: fi, tp: bundleType, resources: make(map[string]*fileInfo)}

-}

-

-func (b *bundleDirs) addBundleContentFile(fi *fileInfo) {

-	dir, found := b.bundles[fi.Lang()]

-	if !found {

-		// Every bundled content file needs a bundle header.

-		// If one does not exist in its language, we pick the default

-		// language version, or a random one if that doesn't exist, either.

-		tl := b.c.sourceSpec.DefaultContentLanguage

-		ldir, found := b.bundles[tl]

-		if !found {

-			// Just pick one.

-			for _, v := range b.bundles {

-				ldir = v

-				break

-			}

-		}

-

-		if ldir == nil {

-			panic(fmt.Sprintf("bundle not found for file %q", fi.Filename()))

-		}

-

-		dir = ldir.clone()

-		dir.fi.overriddenLang = fi.Lang()

-		b.bundles[fi.Lang()] = dir

-	}

-

-	dir.resources[fi.Path()] = fi

-}

-

-func (b *bundleDirs) addBundleFiles(fi *fileInfo) {

-	dir := filepath.ToSlash(fi.Dir())

-	p := dir + fi.TranslationBaseName() + "." + fi.Ext()

-	for lang, bdir := range b.bundles {

-		key := path.Join(lang, p)

-

-		// Given mypage.de.md (German translation) and mypage.md we pick the most

-		// specific for that language.

-		if fi.Lang() == lang || !b.langOverrides[key] {

-			bdir.resources[key] = fi

-		}

-		b.langOverrides[key] = true

-	}

-}

-

-func (b *bundleDirs) addBundleHeader(fi *fileInfo) {

-	b.bundles[fi.Lang()] = newBundleDir(fi, b.tp)

-}

-

-func (c *capturer) isSeen(dirname string) bool {

-	c.seenMu.Lock()

-	defer c.seenMu.Unlock()

-	seen := c.seen[dirname]

-	c.seen[dirname] = true

-	if seen {

-		c.logger.INFO.Printf("Content dir %q already processed; skipped to avoid infinite recursion.", dirname)

-		return true

-

-	}

-	return false

-}

-

-func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) {

-	fileInfo, err := c.lstatIfPossible(path)

-	if err != nil {

-		return nil, err

-	}

-	return fileInfo, c.resolveRealPathIn(fileInfo)

-}

-

-func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error {

-

-	basePath := fileInfo.BaseDir()

-	path := fileInfo.Filename()

-

-	realPath := path

-

-	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {

-		link, err := filepath.EvalSymlinks(path)

-		if err != nil {

-			return _errors.Wrapf(err, "Cannot read symbolic link %q, error was:", path)

-		}

-

-		// This is a file on the outside of any base fs, so we have to use the os package.

-		sfi, err := os.Stat(link)

-		if err != nil {

-			return _errors.Wrapf(err, "Cannot stat  %q, error was:", link)

-		}

-

-		// TODO(bep) improve all of this.

-		if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok {

-			a.FileInfo = sfi

-		}

-

-		realPath = link

-

-		if realPath != path && sfi.IsDir() && c.isSeen(realPath) {

-			// Avoid cyclic symlinks.

-			// Note that this may prevent some uses that isn't cyclic and also

-			// potential useful, but this implementation is both robust and simple:

-			// We stop at the first directory that we have seen before, e.g.

-			// /content/blog will only be processed once.

-			return errSkipCyclicDir

-		}

-

-		if c.contentChanges != nil {

-			// Keep track of symbolic links in watch mode.

-			var from, to string

-			if sfi.IsDir() {

-				from = realPath

-				to = path

-

-				if !strings.HasSuffix(to, helpers.FilePathSeparator) {

-					to = to + helpers.FilePathSeparator

-				}

-				if !strings.HasSuffix(from, helpers.FilePathSeparator) {

-					from = from + helpers.FilePathSeparator

-				}

-

-				if !strings.HasSuffix(basePath, helpers.FilePathSeparator) {

-					basePath = basePath + helpers.FilePathSeparator

-				}

-

-				if strings.HasPrefix(from, basePath) {

-					// With symbolic links inside /content we need to keep

-					// a reference to both. This may be confusing with --navigateToChanged

-					// but the user has chosen this him or herself.

-					c.contentChanges.addSymbolicLinkMapping(from, from)

-				}

-

-			} else {

-				from = realPath

-				to = path

-			}

-

-			c.contentChanges.addSymbolicLinkMapping(from, to)

-		}

-	}

-

-	return nil

-}

-

-func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) {

-	fi, err := helpers.LstatIfPossible(c.fs, path)

-	if err != nil {

-		return nil, err

-	}

-	return fi.(pathLangFileFi), nil

-}

--- a/hugolib/pagebundler_capture_test.go

+++ /dev/null

@@ -1,272 +1,0 @@

-// Copyright 2017-present The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugolib

-

-import (

-	"fmt"

-	"os"

-	"path"

-	"path/filepath"

-	"sort"

-

-	"github.com/gohugoio/hugo/common/loggers"

-

-	"runtime"

-	"strings"

-	"sync"

-	"testing"

-

-	"github.com/gohugoio/hugo/helpers"

-	"github.com/gohugoio/hugo/source"

-	"github.com/stretchr/testify/require"

-)

-

-type storeFilenames struct {

-	sync.Mutex

-	filenames []string

-	copyNames []string

-	dirKeys   []string

-}

-

-func (s *storeFilenames) handleSingles(fis ...*fileInfo) {

-	s.Lock()

-	defer s.Unlock()

-	for _, fi := range fis {

-		s.filenames = append(s.filenames, filepath.ToSlash(fi.Filename()))

-	}

-}

-

-func (s *storeFilenames) handleBundles(d *bundleDirs) {

-	s.Lock()

-	defer s.Unlock()

-	var keys []string

-	for _, b := range d.bundles {

-		res := make([]string, len(b.resources))

-		i := 0

-		for _, r := range b.resources {

-			res[i] = path.Join(r.Lang(), filepath.ToSlash(r.Filename()))

-			i++

-		}

-		sort.Strings(res)

-		keys = append(keys, path.Join("__bundle", b.fi.Lang(), filepath.ToSlash(b.fi.Filename()), "resources", strings.Join(res, "|")))

-	}

-	s.dirKeys = append(s.dirKeys, keys...)

-}

-

-func (s *storeFilenames) handleCopyFile(file pathLangFile) {

-	s.Lock()

-	defer s.Unlock()

-	s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename()))

-}

-

-func (s *storeFilenames) sortedStr() string {

-	s.Lock()

-	defer s.Unlock()

-	sort.Strings(s.filenames)

-	sort.Strings(s.dirKeys)

-	sort.Strings(s.copyNames)

-	return "\nF:\n" + strings.Join(s.filenames, "\n") + "\nD:\n" + strings.Join(s.dirKeys, "\n") +

-		"\nC:\n" + strings.Join(s.copyNames, "\n") + "\n"

-}

-

-func TestPageBundlerCaptureSymlinks(t *testing.T) {

-	if runtime.GOOS == "windows" && os.Getenv("CI") == "" {

-		t.Skip("Skip TestPageBundlerCaptureSymlinks as os.Symlink needs administrator rights on Windows")

-	}

-

-	assert := require.New(t)

-	ps, clean, workDir := newTestBundleSymbolicSources(t)

-	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs)

-	defer clean()

-

-	fileStore := &storeFilenames{}

-	logger := loggers.NewErrorLogger()

-	c := newCapturer(logger, sourceSpec, fileStore, nil)

-

-	assert.NoError(c.capture())

-

-	expected := `

-F:

-/base/a/page_s.md

-/base/a/regular.md

-/base/symbolic1/s1.md

-/base/symbolic1/s2.md

-/base/symbolic3/circus/a/page_s.md

-/base/symbolic3/circus/a/regular.md

-D:

-__bundle/en/base/symbolic2/a1/index.md/resources/en/base/symbolic2/a1/logo.png|en/base/symbolic2/a1/page.md

-C:

-/base/symbolic3/s1.png

-/base/symbolic3/s2.png

-`

-

-	got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1)

-	got = strings.Replace(got, "//", "/", -1)

-

-	if expected != got {

-		diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got))

-		t.Log(got)

-		t.Fatalf("Failed:\n%s", diff)

-	}

-}

-

-func TestPageBundlerCaptureBasic(t *testing.T) {

-	t.Parallel()

-

-	assert := require.New(t)

-	fs, cfg := newTestBundleSources(t)

-	assert.NoError(loadDefaultSettingsFor(cfg))

-	assert.NoError(loadLanguageSettings(cfg, nil))

-	ps, err := helpers.NewPathSpec(fs, cfg)

-	assert.NoError(err)

-

-	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs)

-

-	fileStore := &storeFilenames{}

-

-	c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil)

-

-	assert.NoError(c.capture())

-

-	expected := `

-F:

-/work/base/_1.md

-/work/base/a/1.md

-/work/base/a/2.md

-/work/base/assets/pages/mypage.md

-D:

-__bundle/en/work/base/_index.md/resources/en/work/base/_1.png

-__bundle/en/work/base/a/b/index.md/resources/en/work/base/a/b/ab1.md

-__bundle/en/work/base/b/my-bundle/index.md/resources/en/work/base/b/my-bundle/1.md|en/work/base/b/my-bundle/2.md|en/work/base/b/my-bundle/c/logo.png|en/work/base/b/my-bundle/custom-mime.bep|en/work/base/b/my-bundle/sunset1.jpg|en/work/base/b/my-bundle/sunset2.jpg

-__bundle/en/work/base/c/bundle/index.md/resources/en/work/base/c/bundle/logo-은행.png

-__bundle/en/work/base/root/index.md/resources/en/work/base/root/1.md|en/work/base/root/c/logo.png

-C:

-/work/base/assets/pic1.png

-/work/base/assets/pic2.png

-/work/base/images/hugo-logo.png

-`

-

-	got := fileStore.sortedStr()

-

-	if expected != got {

-		diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got))

-		t.Log(got)

-		t.Fatalf("Failed:\n%s", diff)

-	}

-}

-

-func TestPageBundlerCaptureMultilingual(t *testing.T) {

-	t.Parallel()

-

-	assert := require.New(t)

-	fs, cfg := newTestBundleSourcesMultilingual(t)

-	assert.NoError(loadDefaultSettingsFor(cfg))

-	assert.NoError(loadLanguageSettings(cfg, nil))

-

-	ps, err := helpers.NewPathSpec(fs, cfg)

-	assert.NoError(err)

-

-	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs)

-	fileStore := &storeFilenames{}

-	c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil)

-

-	assert.NoError(c.capture())

-

-	expected := `

-F:

-/work/base/1s/mypage.md

-/work/base/1s/mypage.nn.md

-/work/base/bb/_1.md

-/work/base/bb/_1.nn.md

-/work/base/bb/en.md

-/work/base/bc/page.md

-/work/base/bc/page.nn.md

-/work/base/be/_index.md

-/work/base/be/page.md

-/work/base/be/page.nn.md

-D:

-__bundle/en/work/base/bb/_index.md/resources/en/work/base/bb/a.png|en/work/base/bb/b.png|nn/work/base/bb/c.nn.png

-__bundle/en/work/base/bc/_index.md/resources/en/work/base/bc/logo-bc.png

-__bundle/en/work/base/bd/index.md/resources/en/work/base/bd/page.md

-__bundle/en/work/base/bf/my-bf-bundle/index.md/resources/en/work/base/bf/my-bf-bundle/page.md

-__bundle/en/work/base/lb/index.md/resources/en/work/base/lb/1.md|en/work/base/lb/2.md|en/work/base/lb/c/d/deep.png|en/work/base/lb/c/logo.png|en/work/base/lb/c/one.png|en/work/base/lb/c/page.md

-__bundle/nn/work/base/bb/_index.nn.md/resources/en/work/base/bb/a.png|nn/work/base/bb/b.nn.png|nn/work/base/bb/c.nn.png

-__bundle/nn/work/base/bd/index.md/resources/nn/work/base/bd/page.nn.md

-__bundle/nn/work/base/bf/my-bf-bundle/index.nn.md/resources

-__bundle/nn/work/base/lb/index.nn.md/resources/en/work/base/lb/c/d/deep.png|en/work/base/lb/c/one.png|nn/work/base/lb/2.nn.md|nn/work/base/lb/c/logo.nn.png

-C:

-/work/base/1s/mylogo.png

-/work/base/bb/b/d.nn.png

-`

-

-	got := fileStore.sortedStr()

-

-	if expected != got {

-		diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got))

-		t.Log(got)

-		t.Fatalf("Failed:\n%s", strings.Join(diff, "\n"))

-	}

-

-}

-

-type noOpFileStore int

-

-func (noOpFileStore) handleSingles(fis ...*fileInfo)   {}

-func (noOpFileStore) handleBundles(b *bundleDirs)      {}

-func (noOpFileStore) handleCopyFile(file pathLangFile) {}

-

-func BenchmarkPageBundlerCapture(b *testing.B) {

-	capturers := make([]*capturer, b.N)

-

-	for i := 0; i < b.N; i++ {

-		cfg, fs := newTestCfg()

-		ps, _ := helpers.NewPathSpec(fs, cfg)

-		sourceSpec := source.NewSourceSpec(ps, fs.Source)

-

-		base := fmt.Sprintf("base%d", i)

-		for j := 1; j <= 5; j++ {

-			js := fmt.Sprintf("j%d", j)

-			writeSource(b, fs, filepath.Join(base, js, "index.md"), "content")

-			writeSource(b, fs, filepath.Join(base, js, "logo1.png"), "content")

-			writeSource(b, fs, filepath.Join(base, js, "sub", "logo2.png"), "content")

-			writeSource(b, fs, filepath.Join(base, js, "section", "_index.md"), "content")

-			writeSource(b, fs, filepath.Join(base, js, "section", "logo.png"), "content")

-			writeSource(b, fs, filepath.Join(base, js, "section", "sub", "logo.png"), "content")

-

-			for k := 1; k <= 5; k++ {

-				ks := fmt.Sprintf("k%d", k)

-				writeSource(b, fs, filepath.Join(base, js, ks, "logo1.png"), "content")

-				writeSource(b, fs, filepath.Join(base, js, "section", ks, "logo.png"), "content")

-			}

-		}

-

-		for i := 1; i <= 5; i++ {

-			writeSource(b, fs, filepath.Join(base, "assetsonly", fmt.Sprintf("image%d.png", i)), "image")

-		}

-

-		for i := 1; i <= 5; i++ {

-			writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content")

-		}

-

-		capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base)

-	}

-

-	b.ResetTimer()

-	for i := 0; i < b.N; i++ {

-		err := capturers[i].capture()

-		if err != nil {

-			b.Fatal(err)

-		}

-	}

-}

--- a/hugolib/pagebundler_handlers.go

+++ /dev/null

@@ -1,305 +1,0 @@

-// Copyright 2019 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package hugolib

-

-import (

-	"errors"

-	"fmt"

-	"path/filepath"

-

-	"github.com/gohugoio/hugo/common/hugio"

-

-	"strings"

-

-	"github.com/gohugoio/hugo/resources"

-	"github.com/gohugoio/hugo/resources/resource"

-)

-

-var (

-	// This should be the only list of valid extensions for content files.

-	contentFileExtensions = []string{

-		"html", "htm",

-		"mdown", "markdown", "md",

-		"asciidoc", "adoc", "ad",

-		"rest", "rst",

-		"mmark",

-		"org",

-		"pandoc", "pdc"}

-

-	contentFileExtensionsSet map[string]bool

-)

-

-func init() {

-	contentFileExtensionsSet = make(map[string]bool)

-	for _, ext := range contentFileExtensions {

-		contentFileExtensionsSet[ext] = true

-	}

-}

-

-func newHandlerChain(s *Site) contentHandler {

-	c := &contentHandlers{s: s}

-

-	contentFlow := c.parsePage(

-		c.handlePageContent(),

-	)

-

-	c.rootHandler = c.processFirstMatch(

-		contentFlow,

-

-		// Creates a file resource (image, CSS etc.) if there is a parent

-		// page set on the current context.

-		c.createResource(),

-

-		// Everything that isn't handled above, will just be copied

-		// to destination.

-		c.copyFile(),

-	)

-

-	return c.rootHandler

-

-}

-

-type contentHandlers struct {

-	s           *Site

-	rootHandler contentHandler

-}

-

-func (c *contentHandlers) processFirstMatch(handlers ...contentHandler) func(ctx *handlerContext) handlerResult {

-	return func(ctx *handlerContext) handlerResult {

-		for _, h := range handlers {

-			res := h(ctx)

-			if res.handled || res.err != nil {

-				return res

-			}

-		}

-		return handlerResult{err: errors.New("no matching handler found")}

-	}

-}

-

-type handlerContext struct {

-	// These are the pages stored in Site.

-	pages chan<- *pageState

-

-	doNotAddToSiteCollections bool

-

-	currentPage *pageState

-	parentPage  *pageState

-

-	bundle *bundleDir

-

-	source *fileInfo

-

-	// Relative path to the target.

-	target string

-}

-

-func (c *handlerContext) ext() string {

-	if c.currentPage != nil {

-		return c.currentPage.contentMarkupType()

-	}

-

-	if c.bundle != nil {

-		return c.bundle.fi.Ext()

-	} else {

-		return c.source.Ext()

-	}

-}

-

-func (c *handlerContext) targetPath() string {

-	if c.target != "" {

-		return c.target

-	}

-

-	return c.source.Filename()

-}

-

-func (c *handlerContext) file() *fileInfo {

-	if c.bundle != nil {

-		return c.bundle.fi

-	}

-

-	return c.source

-}

-

-// Create a copy with the current context as its parent.

-func (c handlerContext) childCtx(fi *fileInfo) *handlerContext {

-	if c.currentPage == nil {

-		panic("Need a Page to create a child context")

-	}

-

-	c.target = strings.TrimPrefix(fi.Path(), c.bundle.fi.Dir())

-	c.source = fi

-

-	c.doNotAddToSiteCollections = c.bundle != nil && c.bundle.tp != bundleBranch

-

-	c.bundle = nil

-

-	c.parentPage = c.currentPage

-	c.currentPage = nil

-

-	return &c

-}

-

-func (c *handlerContext) supports(exts ...string) bool {

-	ext := c.ext()

-	for _, s := range exts {

-		if s == ext {

-			return true

-		}

-	}

-

-	return false

-}

-

-func (c *handlerContext) isContentFile() bool {

-	return contentFileExtensionsSet[c.ext()]

-}

-

-type (

-	handlerResult struct {

-		err     error

-		handled bool

-		result  interface{}

-	}

-

-	contentHandler func(ctx *handlerContext) handlerResult

-)

-

-var (

-	notHandled handlerResult

-)

-

-func (c *contentHandlers) parsePage(h contentHandler) contentHandler {

-	return func(ctx *handlerContext) handlerResult {

-		if !ctx.isContentFile() {

-			return notHandled

-		}

-

-		result := handlerResult{handled: true}

-		fi := ctx.file()

-

-		content := func() (hugio.ReadSeekCloser, error) {

-			f, err := fi.Open()

-			if err != nil {

-				return nil, fmt.Errorf("failed to open content file %q: %s", fi.Filename(), err)

-			}

-			return f, nil

-		}

-

-		ps, err := newPageWithContent(fi, c.s, ctx.parentPage != nil, content)

-		if err != nil {

-			return handlerResult{err: err}

-		}

-

-		if !c.s.shouldBuild(ps) {

-			if !ctx.doNotAddToSiteCollections {

-				ctx.pages <- ps

-			}

-			return result

-		}

-

-		ctx.currentPage = ps

-

-		if ctx.bundle != nil {

-			// Add the bundled files

-			for _, fi := range ctx.bundle.resources {

-				childCtx := ctx.childCtx(fi)

-				res := c.rootHandler(childCtx)

-				if res.err != nil {

-					return res

-				}

-				if res.result != nil {

-					switch resv := res.result.(type) {

-					case *pageState:

-						resv.m.resourcePath = filepath.ToSlash(childCtx.target)

-						resv.parent = ps

-						ps.addResources(resv)

-					case resource.Resource:

-						ps.addResources(resv)

-

-					default:

-						panic("Unknown type")

-					}

-				}

-			}

-		}

-

-		return h(ctx)

-	}

-}

-

-func (c *contentHandlers) handlePageContent() contentHandler {

-	return func(ctx *handlerContext) handlerResult {

-		p := ctx.currentPage

-

-		if !ctx.doNotAddToSiteCollections {

-			ctx.pages <- p

-		}

-

-		return handlerResult{handled: true, result: p}

-	}

-}

-

-func (c *contentHandlers) createResource() contentHandler {

-	return func(ctx *handlerContext) handlerResult {

-		if ctx.parentPage == nil {

-			return notHandled

-		}

-

-		// TODO(bep) consolidate with multihost logic + clean up

-		outputFormats := ctx.parentPage.m.outputFormats()

-		seen := make(map[string]bool)

-		var targetBasePaths []string

-		// Make sure bundled resources are published to all of the ouptput formats'

-		// sub paths.

-		for _, f := range outputFormats {

-			p := f.Path

-			if seen[p] {

-				continue

-			}

-			seen[p] = true

-			targetBasePaths = append(targetBasePaths, p)

-

-		}

-

-		resource, err := c.s.ResourceSpec.New(

-			resources.ResourceSourceDescriptor{

-				TargetPaths:       ctx.parentPage.getTargetPaths,

-				SourceFile:        ctx.source,

-				RelTargetFilename: ctx.target,

-				TargetBasePaths:   targetBasePaths,

-			})

-

-		return handlerResult{err: err, handled: true, result: resource}

-	}

-}

-

-func (c *contentHandlers) copyFile() contentHandler {

-	return func(ctx *handlerContext) handlerResult {

-		f, err := c.s.BaseFs.Content.Fs.Open(ctx.source.Filename())

-		if err != nil {

-			err := fmt.Errorf("failed to open file in copyFile: %s", err)

-			return handlerResult{err: err}

-		}

-

-		target := ctx.targetPath()

-

-		defer f.Close()

-		if err := c.s.publish(&c.s.PathSpec.ProcessingStats.Files, target, f); err != nil {

-			return handlerResult{err: err}

-		}

-

-		return handlerResult{handled: true}

-	}

-}

--- a/hugolib/pagebundler_test.go

+++ b/hugolib/pagebundler_test.go

@@ -16,18 +16,18 @@

 import (

 	"os"

 	"path"

-	"runtime"

+	"regexp"

 	"strings"

 	"testing"

+	"github.com/gohugoio/hugo/hugofs"

+

 	"github.com/gohugoio/hugo/common/loggers"

 	"github.com/gohugoio/hugo/resources/page"

-	"github.com/gohugoio/hugo/helpers"

-

 	"io"

-	"github.com/spf13/afero"

+	"github.com/gohugoio/hugo/htesting"

 	"github.com/gohugoio/hugo/media"

@@ -36,7 +36,6 @@

 	"fmt"

 	"github.com/gohugoio/hugo/deps"

-	"github.com/gohugoio/hugo/hugofs"

 	"github.com/spf13/viper"

 	"github.com/stretchr/testify/require"

@@ -54,8 +53,11 @@

 				if baseURLPathId == "" {

 					baseURLPathId = "NONE"

 				}

+				ugly := ugly

+				canonify := canonify

 				t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId),

 					func(t *testing.T) {

+						t.Parallel()

 						baseURL := baseBaseURL + baseURLPath

 						relURLBase := baseURLPath

 						if canonify {

@@ -65,14 +67,12 @@

 						fs, cfg := newTestBundleSources(t)

 						cfg.Set("baseURL", baseURL)

 						cfg.Set("canonifyURLs", canonify)

-						assert.NoError(loadDefaultSettingsFor(cfg))

-						assert.NoError(loadLanguageSettings(cfg, nil))

 						cfg.Set("permalinks", map[string]string{

 							"a": ":sections/:filename",

 							"b": ":year/:slug/",

 							"c": ":sections/:slug",

-							"":  ":filename/",

+							"/": ":filename/",

 						})

 						cfg.Set("outputFormats", map[string]interface{}{

@@ -92,10 +92,12 @@

 						cfg.Set("uglyURLs", ugly)

-						s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewErrorLogger(), Fs: fs, Cfg: cfg}, BuildCfg{})

+						b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Logger: loggers.NewErrorLogger(), Fs: fs, Cfg: cfg}).WithNothingAdded()

-						th := testHelper{s.Cfg, s.Fs, t}

+						b.Build(BuildCfg{})

+						s := b.H.Sites[0]

+

 						assert.Len(s.RegularPages(), 8)

 						singlePage := s.getPage(page.KindPage, "a/1.md")

@@ -127,7 +129,7 @@

 						// Check both output formats

 						rel, filename := relFilename("/a/1/", "index.html")

-						th.assertFileContent(filepath.Join("/work/public", filename),

+						b.AssertFileContent(filepath.Join("/work/public", filename),

 							"TheContent",

 							"Single RelPermalink: "+rel,

 						)

@@ -134,15 +136,15 @@

 						rel, filename = relFilename("/cpath/a/1/", "cindex.html")

-						th.assertFileContent(filepath.Join("/work/public", filename),

+						b.AssertFileContent(filepath.Join("/work/public", filename),

 							"TheContent",

 							"Single RelPermalink: "+rel,

 						)

-						th.assertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content")

+						b.AssertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content")

 						// This should be just copied to destination.

-						th.assertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content")

+						b.AssertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content")

 						leafBundle1 := s.getPage(page.KindPage, "b/my-bundle/index.md")

 						assert.NotNil(leafBundle1)

@@ -159,8 +161,8 @@

 						assert.NotNil(rootBundle)

 						assert.True(rootBundle.Parent().IsHome())

 						if !ugly {

-							th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single RelPermalink: "+relURLBase+"/root/")

-							th.assertFileContent(filepath.FromSlash("/work/public/cpath/root/cindex.html"), "Single RelPermalink: "+relURLBase+"/cpath/root/")

+							b.AssertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single RelPermalink: "+relURLBase+"/root/")

+							b.AssertFileContent(filepath.FromSlash("/work/public/cpath/root/cindex.html"), "Single RelPermalink: "+relURLBase+"/cpath/root/")

 						}

 						leafBundle2 := s.getPage(page.KindPage, "a/b/index.md")

@@ -172,6 +174,7 @@

 						assert.Len(pageResources, 2)

 						firstPage := pageResources[0].(page.Page)

 						secondPage := pageResources[1].(page.Page)

+

 						assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/1.md"), firstPage.File().Filename(), secondPage.File().Filename())

 						assert.Contains(content(firstPage), "TheContent")

 						assert.Equal(6, len(leafBundle1.Resources()))

@@ -201,17 +204,17 @@

 						}

 						if ugly {

-							th.assertFileContent("/work/public/2017/pageslug.html",

+							b.AssertFileContent("/work/public/2017/pageslug.html",

 								relPermalinker("Single RelPermalink: %s/2017/pageslug.html"),

 								permalinker("Single Permalink: %s/2017/pageslug.html"),

 								relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"),

 								permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg"))

 						} else {

-							th.assertFileContent("/work/public/2017/pageslug/index.html",

+							b.AssertFileContent("/work/public/2017/pageslug/index.html",

 								relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"),

 								permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg"))

-							th.assertFileContent("/work/public/cpath/2017/pageslug/cindex.html",

+							b.AssertFileContent("/work/public/cpath/2017/pageslug/cindex.html",

 								relPermalinker("Single RelPermalink: %s/cpath/2017/pageslug/"),

 								relPermalinker("Short Sunset RelPermalink: %s/cpath/2017/pageslug/sunset2.jpg"),

 								relPermalinker("Sunset RelPermalink: %s/cpath/2017/pageslug/sunset1.jpg"),

@@ -219,15 +222,15 @@

 							)

 						}

-						th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content")

-						th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content")

-						th.assertFileNotExist("/work/public/cpath/cpath/2017/pageslug/c/logo.png")

+						b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content")

+						b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content")

+						assert.False(b.CheckExists("/work/public/cpath/cpath/2017/pageslug/c/logo.png"))

 						// Custom media type defined in site config.

 						assert.Len(leafBundle1.Resources().ByType("bepsays"), 1)

 						if ugly {

-							th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"),

+							b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"),

 								"TheContent",

 								relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"),

 								permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg"),

@@ -245,19 +248,19 @@

 							)

 							// https://github.com/gohugoio/hugo/issues/5882

-							th.assertFileContent(

+							b.AssertFileContent(

 								filepath.FromSlash("/work/public/2017/pageslug.html"), "0: Page RelPermalink: |")

-							th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent")

+							b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent")

 							// 은행

-							th.assertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG")

+							b.AssertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG")

 						} else {

-							th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent")

-							th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent")

-							th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title")

-							th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title")

+							b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent")

+							b.AssertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent")

+							b.AssertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title")

+							b.AssertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title")

 						}

@@ -272,25 +275,26 @@

 	t.Parallel()

 	for _, ugly := range []bool{false, true} {

+		ugly := ugly

 		t.Run(fmt.Sprintf("ugly=%t", ugly),

 			func(t *testing.T) {

-

+				t.Parallel()

 				assert := require.New(t)

 				fs, cfg := newTestBundleSourcesMultilingual(t)

 				cfg.Set("uglyURLs", ugly)

-				assert.NoError(loadDefaultSettingsFor(cfg))

-				assert.NoError(loadLanguageSettings(cfg, nil))

-				sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})

-				assert.NoError(err)

-				assert.Equal(2, len(sites.Sites))

+				b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded()

+				b.Build(BuildCfg{SkipRender: true})

-				assert.NoError(sites.Build(BuildCfg{}))

+				sites := b.H

+				assert.Equal(2, len(sites.Sites))

+

 				s := sites.Sites[0]

 				assert.Equal(8, len(s.RegularPages()))

 				assert.Equal(16, len(s.Pages()))

+				//dumpPages(s.AllPages()...)

 				assert.Equal(31, len(s.AllPages()))

 				bundleWithSubPath := s.getPage(page.KindPage, "lb/index")

@@ -358,15 +362,12 @@

 	fs, cfg := newTestBundleSourcesMultilingual(t)

 	cfg.Set("disableLanguages", []string{"nn"})

-	assert.NoError(loadDefaultSettingsFor(cfg))

-	assert.NoError(loadLanguageSettings(cfg, nil))

+	b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded()

+	b.Build(BuildCfg{})

+	sites := b.H

-	sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})

-	assert.NoError(err)

 	assert.Equal(1, len(sites.Sites))

-	assert.NoError(sites.Build(BuildCfg{}))

-

 	s := sites.Sites[0]

 	assert.Equal(8, len(s.RegularPages()))

@@ -383,21 +384,99 @@

 }

 func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) {

-	if runtime.GOOS == "windows" && os.Getenv("CI") == "" {

-		t.Skip("Skip TestPageBundlerSiteWitSymbolicLinksInContent as os.Symlink needs administrator rights on Windows")

-	}

+	skipSymlink(t)

+	wd, _ := os.Getwd()

+	defer func() {

+		os.Chdir(wd)

+	}()

+

 	assert := require.New(t)

-	ps, clean, workDir := newTestBundleSymbolicSources(t)

+	// We need to use the OS fs for this.

+	cfg := viper.New()

+	fs := hugofs.NewFrom(hugofs.Os, cfg)

+

+	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugosym")

+	assert.NoError(err)

+

+	contentDirName := "content"

+

+	contentDir := filepath.Join(workDir, contentDirName)

+	assert.NoError(os.MkdirAll(filepath.Join(contentDir, "a"), 0777))

+

+	for i := 1; i <= 3; i++ {

+		assert.NoError(os.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777))

+	}

+

+	assert.NoError(os.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777))

+

+	// Symlinked sections inside content.

+	os.Chdir(contentDir)

+	for i := 1; i <= 3; i++ {

+		assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i)))

+	}

+

+	assert.NoError(os.Chdir(filepath.Join(contentDir, "a")))

+

+	// Create a symlink to one single content file

+	assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md"))

+

+	assert.NoError(os.Chdir(filepath.FromSlash("../../symcontent3")))

+

+	// Create a circular symlink. Will print some warnings.

+	assert.NoError(os.Symlink(filepath.Join("..", contentDirName), filepath.FromSlash("circus")))

+

+	assert.NoError(os.Chdir(workDir))

+

 	defer clean()

-	cfg := ps.Cfg

-	fs := ps.Fs

+	cfg.Set("workingDir", workDir)

+	cfg.Set("contentDir", contentDirName)

+	cfg.Set("baseURL", "https://example.com")

-	s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{})

+	layout := `{{ .Title }}|{{ .Content }}`

+	pageContent := `---

+slug: %s

+date: 2017-10-09

+---

-	th := testHelper{s.Cfg, s.Fs, t}

+TheContent.

+`

+	b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{

+		Fs:  fs,

+		Cfg: cfg,

+	})

+

+	b.WithTemplates(

+		"_default/single.html", layout,

+		"_default/list.html", layout,

+	)

+

+	b.WithContent(

+		"a/regular.md", fmt.Sprintf(pageContent, "a1"),

+	)

+

+	b.WithSourceFile(

+		"symcontent1/s1.md", fmt.Sprintf(pageContent, "s1"),

+		"symcontent1/s2.md", fmt.Sprintf(pageContent, "s2"),

+		// Regular files inside symlinked folder.

+		"symcontent1/s1.md", fmt.Sprintf(pageContent, "s1"),

+		"symcontent1/s2.md", fmt.Sprintf(pageContent, "s2"),

+

+		// A bundle

+		"symcontent2/a1/index.md", fmt.Sprintf(pageContent, ""),

+		"symcontent2/a1/page.md", fmt.Sprintf(pageContent, "page"),

+		"symcontent2/a1/logo.png", "image",

+

+		// Assets

+		"symcontent3/s1.png", "image",

+		"symcontent3/s2.png", "image",

+	)

+

+	b.Build(BuildCfg{})

+	s := b.H.Sites[0]

+

 	assert.Equal(7, len(s.RegularPages()))

 	a1Bundle := s.getPage(page.KindPage, "symbolic2/a1/index.md")

 	assert.NotNil(a1Bundle)

@@ -404,9 +483,9 @@

 	assert.Equal(2, len(a1Bundle.Resources()))

 	assert.Equal(1, len(a1Bundle.Resources().ByType(pageResourceType)))

-	th.assertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent")

-	th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent")

-	th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent")

+	b.AssertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent")

+	b.AssertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent")

+	b.AssertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent")

 }

@@ -576,7 +655,7 @@

 }

 func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) {

-	cfg, fs := newTestCfg()

+	cfg, fs := newTestCfgBasic()

 	assert := require.New(t)

 	workDir := "/work"

@@ -743,7 +822,7 @@

 }

 func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) {

-	cfg, fs := newTestCfg()

+	cfg, fs := newTestCfgBasic()

 	workDir := "/work"

 	cfg.Set("workingDir", workDir)

@@ -825,90 +904,6 @@

 	return fs, cfg

 }

-func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, func(), string) {

-	assert := require.New(t)

-	// We need to use the OS fs for this.

-	cfg := viper.New()

-	fs := hugofs.NewFrom(hugofs.Os, cfg)

-	fs.Destination = &afero.MemMapFs{}

-	loadDefaultSettingsFor(cfg)

-

-	workDir, clean, err := createTempDir("hugosym")

-	assert.NoError(err)

-

-	contentDir := "base"

-	cfg.Set("workingDir", workDir)

-	cfg.Set("contentDir", contentDir)

-	cfg.Set("baseURL", "https://example.com")

-

-	if err := loadLanguageSettings(cfg, nil); err != nil {

-		t.Fatal(err)

-	}

-

-	layout := `{{ .Title }}|{{ .Content }}`

-	pageContent := `---

-slug: %s

-date: 2017-10-09

----

-

-TheContent.

-`

-

-	fs.Source.MkdirAll(filepath.Join(workDir, "layouts", "_default"), 0777)

-	fs.Source.MkdirAll(filepath.Join(workDir, contentDir), 0777)

-	fs.Source.MkdirAll(filepath.Join(workDir, contentDir, "a"), 0777)

-	for i := 1; i <= 3; i++ {

-		fs.Source.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777)

-

-	}

-	fs.Source.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777)

-

-	writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), layout)

-	writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), layout)

-

-	writeSource(t, fs, filepath.Join(workDir, contentDir, "a", "regular.md"), fmt.Sprintf(pageContent, "a1"))

-

-	// Regular files inside symlinked folder.

-	writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s1.md"), fmt.Sprintf(pageContent, "s1"))

-	writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s2.md"), fmt.Sprintf(pageContent, "s2"))

-

-	// A bundle

-	writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "index.md"), fmt.Sprintf(pageContent, ""))

-	writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "page.md"), fmt.Sprintf(pageContent, "page"))

-	writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "logo.png"), "image")

-

-	// Assets

-	writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s1.png"), "image")

-	writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s2.png"), "image")

-

-	wd, _ := os.Getwd()

-	defer func() {

-		os.Chdir(wd)

-	}()

-	// Symlinked sections inside content.

-	os.Chdir(filepath.Join(workDir, contentDir))

-	for i := 1; i <= 3; i++ {

-		assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i)))

-	}

-

-	os.Chdir(filepath.Join(workDir, contentDir, "a"))

-

-	// Create a symlink to one single content file

-	assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md"))

-

-	os.Chdir(filepath.FromSlash("../../symcontent3"))

-

-	// Create a circular symlink. Will print some warnings.

-	assert.NoError(os.Symlink(filepath.Join("..", contentDir), filepath.FromSlash("circus")))

-

-	os.Chdir(workDir)

-	assert.NoError(err)

-

-	ps, _ := helpers.NewPathSpec(fs, cfg)

-

-	return ps, clean, workDir

-}

-

 // https://github.com/gohugoio/hugo/issues/5858

 func TestBundledResourcesWhenMultipleOutputFormats(t *testing.T) {

 	t.Parallel()

@@ -969,5 +964,114 @@

 	assert.True(b.CheckExists("public/about/services1/this-is-the-slug/index.html"))

 	assert.True(b.CheckExists("public/about/services2/this-is-another-slug/index.html"))

+

+}

+

+func TestBundleMisc(t *testing.T) {

+	config := `

+baseURL = "https://example.com"

+defaultContentLanguage = "en"

+defaultContentLanguageInSubdir = true

+ignoreFiles = ["README\\.md", "content/en/ignore"]

+

+[Languages]

+[Languages.en]

+weight = 99999

+contentDir = "content/en"

+[Languages.nn]

+weight = 20

+contentDir = "content/nn"

+[Languages.sv]

+weight = 30

+contentDir = "content/sv"

+[Languages.nb]

+weight = 40

+contentDir = "content/nb"

+

+`

+

+	const pageContent = `---

+title: %q

+---

+`

+	createPage := func(s string) string {

+		return fmt.Sprintf(pageContent, s)

+	}

+

+	b := newTestSitesBuilder(t).WithConfigFile("toml", config)

+	b.WithLogger(loggers.NewWarningLogger())

+

+	b.WithTemplates("_default/list.html", `{{ range .Site.Pages }}

+{{ .Kind }}|{{ .Path }}|{{ with .CurrentSection }}CurrentSection: {{ .Path }}{{ end }}|{{ .RelPermalink }}{{ end }}

+`)

+

+	b.WithTemplates("_default/single.html", `Single: {{ .Title }}`)

+

+	b.WithContent("en/sect1/sect2/_index.md", createPage("en: Sect 2"))

+	b.WithContent("en/sect1/sect2/page.md", createPage("en: Page"))

+	b.WithContent("en/sect1/sect2/data-branch.json", "mydata")

+	b.WithContent("nn/sect1/sect2/page.md", createPage("nn: Page"))

+	b.WithContent("nn/sect1/sect2/data-branch.json", "my nn data")

+

+	// En only

+	b.WithContent("en/enonly/myen.md", createPage("en: Page"))

+	b.WithContent("en/enonly/myendata.json", "mydata")

+

+	// Leaf

+

+	b.WithContent("nn/b1/index.md", createPage("nn: leaf"))

+	b.WithContent("en/b1/index.md", createPage("en: leaf"))

+	b.WithContent("sv/b1/index.md", createPage("sv: leaf"))

+	b.WithContent("nb/b1/index.md", createPage("nb: leaf"))

+

+	// Should be ignored

+	b.WithContent("en/ignore/page.md", createPage("en: ignore"))

+	b.WithContent("en/README.md", createPage("en: ignore"))

+

+	// Both leaf and branch bundle in same dir

+	b.WithContent("en/b2/index.md", `---

+slug: leaf

+---

+`)

+	b.WithContent("en/b2/_index.md", createPage("en: branch"))

+

+	b.WithContent("en/b1/data1.json", "en: data")

+	b.WithContent("sv/b1/data1.json", "sv: data")

+	b.WithContent("sv/b1/data2.json", "sv: data2")

+	b.WithContent("nb/b1/data2.json", "nb: data2")

+

+	b.Build(BuildCfg{})

+

+	b.AssertFileContent("public/en/index.html",

+		filepath.FromSlash("section|sect1/sect2/_index.md|CurrentSection: sect1/sect2/_index.md"),

+		"myen.md|CurrentSection: enonly")

+

+	b.AssertFileContentFn("public/en/index.html", func(s string) bool {

+		// Check ignored files

+		return !regexp.MustCompile("README|ignore").MatchString(s)

+

+	})

+

+	b.AssertFileContent("public/nn/index.html", filepath.FromSlash("page|sect1/sect2/page.md|CurrentSection: sect1"))

+	b.AssertFileContentFn("public/nn/index.html", func(s string) bool {

+		return !strings.Contains(s, "enonly")

+

+	})

+

+	// Check order of inherited data file

+	b.AssertFileContent("public/nb/b1/data1.json", "en: data") // Default content

+	b.AssertFileContent("public/nn/b1/data2.json", "sv: data") // First match

+

+	b.AssertFileContent("public/en/enonly/myen/index.html", "Single: en: Page")

+	b.AssertFileContent("public/en/enonly/myendata.json", "mydata")

+

+	assert := require.New(t)

+	assert.False(b.CheckExists("public/sv/enonly/myen/index.html"))

+

+	// Both leaf and branch bundle in same dir

+	// We log a warning about it, but we keep both.

+	b.AssertFileContent("public/en/b2/index.html",

+		"/en/b2/leaf/",

+		filepath.FromSlash("section|sect1/sect2/_index.md|CurrentSection: sect1/sect2/_index.md"))

 }

--- a/hugolib/pagecollections.go

+++ b/hugolib/pagecollections.go

@@ -148,7 +148,6 @@

 			for _, p := range pageCollection {

 				if p.IsPage() {

 					sourceRef := p.sourceRef()

-

 					if sourceRef != "" {

 						// index the canonical ref

 						// e.g. /section/article.md

--- /dev/null

+++ b/hugolib/pages_capture.go

@@ -1,0 +1,779 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugolib

+

+import (

+	"context"

+	"fmt"

+	"os"

+	pth "path"

+	"path/filepath"

+	"strings"

+

+	"github.com/gohugoio/hugo/config"

+

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/gohugoio/hugo/resources"

+

+	"github.com/pkg/errors"

+	"golang.org/x/sync/errgroup"

+

+	"github.com/gohugoio/hugo/common/hugio"

+

+	"github.com/gohugoio/hugo/resources/resource"

+

+	"github.com/gohugoio/hugo/source"

+

+	"github.com/gohugoio/hugo/hugofs"

+

+	"github.com/gohugoio/hugo/common/loggers"

+	"github.com/spf13/afero"

+)

+

+func newPagesCollector(

+	sp *source.SourceSpec,

+	logger *loggers.Logger,

+	contentTracker *contentChangeMap,

+	proc pagesCollectorProcessorProvider, filenames ...string) *pagesCollector {

+

+	return &pagesCollector{

+		fs:        sp.SourceFs,

+		proc:      proc,

+		sp:        sp,

+		logger:    logger,

+		filenames: filenames,

+		tracker:   contentTracker,

+	}

+}

+

+func newPagesProcessor(h *HugoSites, sp *source.SourceSpec, partialBuild bool) *pagesProcessor {

+

+	return &pagesProcessor{

+		h:            h,

+		sp:           sp,

+		partialBuild: partialBuild,

+		numWorkers:   config.GetNumWorkerMultiplier() * 3,

+	}

+}

+

+type fileinfoBundle struct {

+	header    hugofs.FileMetaInfo

+	resources []hugofs.FileMetaInfo

+}

+

+func (b *fileinfoBundle) containsResource(name string) bool {

+	for _, r := range b.resources {

+		if r.Name() == name {

+			return true

+		}

+	}

+

+	return false

+

+}

+

+type pageBundles map[string]*fileinfoBundle

+

+type pagesCollector struct {

+	sp     *source.SourceSpec

+	fs     afero.Fs

+	logger *loggers.Logger

+

+	// Ordered list (bundle headers first) used in partial builds.

+	filenames []string

+

+	// Content files tracker used in partial builds.

+	tracker *contentChangeMap

+

+	proc pagesCollectorProcessorProvider

+}

+

+type contentDirKey struct {

+	dirname  string

+	filename string

+	tp       bundleDirType

+}

+

+// Collect.

+func (c *pagesCollector) Collect() error {

+	c.proc.Start(context.Background())

+	if c.tracker != nil {

+		c.tracker.start()

+		defer c.tracker.stop()

+	}

+

+	var collectErr error

+	if len(c.filenames) == 0 {

+		// Collect everything.

+		collectErr = c.collectDir("", false, nil)

+	} else {

+		dirs := make(map[contentDirKey]bool)

+		for _, filename := range c.filenames {

+			dir, filename, btype := c.tracker.resolveAndRemove(filename)

+			dirs[contentDirKey{dir, filename, btype}] = true

+		}

+

+		for dir, _ := range dirs {

+			switch dir.tp {

+			case bundleLeaf, bundleBranch:

+				collectErr = c.collectDir(dir.dirname, true, nil)

+			default:

+				// We always start from a directory.

+				collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool {

+					return strings.HasSuffix(dir.filename, fim.Meta().Path())

+				})

+			}

+

+			if collectErr != nil {

+				break

+			}

+		}

+

+	}

+

+	err := c.proc.Wait()

+

+	if collectErr != nil {

+		return collectErr

+	}

+

+	return err

+}

+

+func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func(fim hugofs.FileMetaInfo) bool) error {

+	fi, err := c.fs.Stat(dirname)

+	if err != nil {

+		if os.IsNotExist(err) {

+			// May have been deleted.

+			return nil

+		}

+		return err

+	}

+

+	handleDir := func(

+		btype bundleDirType,

+		dir hugofs.FileMetaInfo,

+		path string,

+		readdir []hugofs.FileMetaInfo) error {

+

+		if btype > bundleNot && c.tracker != nil {

+			c.tracker.add(path, btype)

+		}

+

+		if btype == bundleBranch {

+			if err := c.handleBundleBranch(readdir); err != nil {

+				return err

+			}

+			// A branch bundle is only this directory level, so keep walking.

+			return nil

+		} else if btype == bundleLeaf {

+			if err := c.handleBundleLeaf(dir, path, readdir); err != nil {

+				return err

+			}

+

+			return nil

+		}

+

+		if err := c.handleFiles(readdir...); err != nil {

+			return err

+		}

+

+		return nil

+

+	}

+

+	filter := func(fim hugofs.FileMetaInfo) bool {

+		if fim.Meta().SkipDir() {

+			return false

+		}

+

+		if c.sp.IgnoreFile(fim.Meta().Filename()) {

+			return false

+		}

+

+		if inFilter != nil {

+			return inFilter(fim)

+		}

+		return true

+	}

+

+	preHook := func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) {

+		var btype bundleDirType

+

+		filtered := readdir[:0]

+		for _, fi := range readdir {

+			if filter(fi) {

+				filtered = append(filtered, fi)

+				if c.tracker != nil {

+					// Track symlinks.

+					c.tracker.addSymbolicLinkMapping(fi)

+				}

+			}

+		}

+		readdir = filtered

+

+		// We merge language directories, so there can be duplicates, but they

+		// will be ordered, most important first.

+		var duplicates []int

+		seen := make(map[string]bool)

+

+		for i, fi := range readdir {

+

+			if fi.IsDir() {

+				continue

+			}

+

+			meta := fi.Meta()

+			class := meta.Classifier()

+			translationBase := meta.TranslationBaseNameWithExt()

+			key := pth.Join(meta.Lang(), translationBase)

+

+			if seen[key] {

+				duplicates = append(duplicates, i)

+				continue

+			}

+			seen[key] = true

+

+			var thisBtype bundleDirType

+

+			switch class {

+			case files.ContentClassLeaf:

+				thisBtype = bundleLeaf

+			case files.ContentClassBranch:

+				thisBtype = bundleBranch

+			}

+

+			// Folders with both index.md and _index.md type of files have

+			// undefined behaviour and can never work.

+			// The branch variant will win because of sort order, but log

+			// a warning about it.

+			if thisBtype > bundleNot && btype > bundleNot && thisBtype != btype {

+				c.logger.WARN.Printf("Content directory %q have both index.* and _index.* files, pick one.", dir.Meta().Filename())

+				// Reclassify it so it will be handled as a content file inside the

+				// section, which is in line with the <= 0.55 behaviour.

+				meta["classifier"] = files.ContentClassContent

+			} else if thisBtype > bundleNot {

+				btype = thisBtype

+			}

+

+		}

+

+		if len(duplicates) > 0 {

+			for i := len(duplicates) - 1; i >= 0; i-- {

+				idx := duplicates[i]

+				readdir = append(readdir[:idx], readdir[idx+1:]...)

+			}

+		}

+

+		err := handleDir(btype, dir, path, readdir)

+		if err != nil {

+			return nil, err

+		}

+

+		if btype == bundleLeaf || partial {

+			return nil, filepath.SkipDir

+		}

+

+		// Keep walking.

+		return readdir, nil

+

+	}

+

+	var postHook hugofs.WalkHook

+	if c.tracker != nil {

+		postHook = func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) {

+			if c.tracker == nil {

+				// Nothing to do.

+				return readdir, nil

+			}

+

+			return readdir, nil

+		}

+	}

+

+	wfn := func(path string, info hugofs.FileMetaInfo, err error) error {

+		if err != nil {

+			return err

+		}

+

+		return nil

+	}

+

+	w := hugofs.NewWalkway(hugofs.WalkwayConfig{

+		Fs:       c.fs,

+		Logger:   c.logger,

+		Root:     dirname,

+		Info:     fi.(hugofs.FileMetaInfo),

+		HookPre:  preHook,

+		HookPost: postHook,

+		WalkFn:   wfn})

+

+	return w.Walk()

+

+}

+

+func (c *pagesCollector) isBundleHeader(fi hugofs.FileMetaInfo) bool {

+	class := fi.Meta().Classifier()

+	return class == files.ContentClassLeaf || class == files.ContentClassBranch

+}

+

+func (c *pagesCollector) getLang(fi hugofs.FileMetaInfo) string {

+	lang := fi.Meta().Lang()

+	if lang != "" {

+		return lang

+	}

+

+	return c.sp.DefaultContentLanguage

+}

+

+func (c *pagesCollector) addToBundle(info hugofs.FileMetaInfo, btyp bundleDirType, bundles pageBundles) error {

+	getBundle := func(lang string) *fileinfoBundle {

+		return bundles[lang]

+	}

+

+	cloneBundle := func(lang string) *fileinfoBundle {

+		// Every bundled file needs a content file header.

+		// Use the default content language if found, else just

+		// pick one.

+		var (

+			source *fileinfoBundle

+			found  bool

+		)

+

+		source, found = bundles[c.sp.DefaultContentLanguage]

+		if !found {

+			for _, b := range bundles {

+				source = b

+				break

+			}

+		}

+

+		if source == nil {

+			panic(fmt.Sprintf("no source found, %d", len(bundles)))

+		}

+

+		clone := c.cloneFileInfo(source.header)

+		clone.Meta()["lang"] = lang

+

+		return &fileinfoBundle{

+			header: clone,

+		}

+	}

+

+	lang := c.getLang(info)

+	bundle := getBundle(lang)

+	isBundleHeader := c.isBundleHeader(info)

+	classifier := info.Meta().Classifier()

+	if bundle == nil {

+		if isBundleHeader {

+			bundle = &fileinfoBundle{header: info}

+			bundles[lang] = bundle

+		} else {

+			if btyp == bundleBranch {

+				// No special logic for branch bundles.

+				// Every language needs its own _index.md file.

+				return c.handleFiles(info)

+			}

+

+			bundle = cloneBundle(lang)

+			bundles[lang] = bundle

+		}

+	}

+

+	if !isBundleHeader {

+		bundle.resources = append(bundle.resources, info)

+	}

+

+	if classifier == files.ContentClassFile {

+		translations := info.Meta().Translations()

+		if len(translations) < len(bundles) {

+			for lang, b := range bundles {

+				if !stringSliceContains(lang, translations...) && !b.containsResource(info.Name()) {

+					// Clone and add it to the bundle.

+					clone := c.cloneFileInfo(info)

+					clone.Meta()["lang"] = lang

+					b.resources = append(b.resources, clone)

+				}

+			}

+		}

+	}

+

+	return nil

+}

+

+func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaInfo {

+	cm := hugofs.FileMeta{}

+	meta := fi.Meta()

+	if meta == nil {

+		panic(fmt.Sprintf("not meta: %v", fi.Name()))

+	}

+	for k, v := range meta {

+		cm[k] = v

+	}

+

+	return hugofs.NewFileMetaInfo(fi, cm)

+}

+

+func (c *pagesCollector) handleBundleBranch(readdir []hugofs.FileMetaInfo) error {

+

+	// Maps bundles to its language.

+	bundles := pageBundles{}

+

+	for _, fim := range readdir {

+

+		if fim.IsDir() {

+			continue

+		}

+

+		meta := fim.Meta()

+

+		switch meta.Classifier() {

+		case files.ContentClassContent:

+			if err := c.handleFiles(fim); err != nil {

+				return err

+			}

+		default:

+			if err := c.addToBundle(fim, bundleBranch, bundles); err != nil {

+				return err

+			}

+		}

+

+	}

+

+	return c.proc.Process(bundles)

+

+}

+

+func (c *pagesCollector) handleBundleLeaf(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) error {

+	// Maps bundles to its language.

+	bundles := pageBundles{}

+

+	walk := func(path string, info hugofs.FileMetaInfo, err error) error {

+		if err != nil {

+			return err

+		}

+		if info.IsDir() {

+			return nil

+		}

+

+		return c.addToBundle(info, bundleLeaf, bundles)

+

+	}

+

+	// Start a new walker from the given path.

+	w := hugofs.NewWalkway(hugofs.WalkwayConfig{

+		Root:       path,

+		Fs:         c.fs,

+		Logger:     c.logger,

+		Info:       dir,

+		DirEntries: readdir,

+		WalkFn:     walk})

+

+	if err := w.Walk(); err != nil {

+		return err

+	}

+

+	return c.proc.Process(bundles)

+

+}

+

+func (c *pagesCollector) handleFiles(fis ...hugofs.FileMetaInfo) error {

+	for _, fi := range fis {

+		if fi.IsDir() {

+			continue

+		}

+

+		if err := c.proc.Process(fi); err != nil {

+			return err

+		}

+	}

+	return nil

+}

+

+type pagesCollectorProcessorProvider interface {

+	Process(item interface{}) error

+	Start(ctx context.Context) context.Context

+	Wait() error

+}

+

+type pagesProcessor struct {

+	h  *HugoSites

+	sp *source.SourceSpec

+

+	itemChan  chan interface{}

+	itemGroup *errgroup.Group

+

+	// The output Pages

+	pagesChan  chan *pageState

+	pagesGroup *errgroup.Group

+

+	numWorkers int

+

+	partialBuild bool

+}

+

+func (proc *pagesProcessor) Process(item interface{}) error {

+	proc.itemChan <- item

+	return nil

+}

+

+func (proc *pagesProcessor) Start(ctx context.Context) context.Context {

+	proc.pagesChan = make(chan *pageState, proc.numWorkers)

+	proc.pagesGroup, ctx = errgroup.WithContext(ctx)

+	proc.itemChan = make(chan interface{}, proc.numWorkers)

+	proc.itemGroup, ctx = errgroup.WithContext(ctx)

+

+	proc.pagesGroup.Go(func() error {

+		for p := range proc.pagesChan {

+			s := p.s

+			p.forceRender = proc.partialBuild

+

+			if p.forceRender {

+				s.replacePage(p)

+			} else {

+				s.addPage(p)

+			}

+		}

+		return nil

+	})

+

+	for i := 0; i < proc.numWorkers; i++ {

+		proc.itemGroup.Go(func() error {

+			for item := range proc.itemChan {

+				select {

+				case <-proc.h.Done():

+					return nil

+				default:

+					if err := proc.process(item); err != nil {

+						proc.h.SendError(err)

+					}

+				}

+			}

+

+			return nil

+		})

+	}

+

+	return ctx

+}

+

+func (proc *pagesProcessor) Wait() error {

+	close(proc.itemChan)

+

+	err := proc.itemGroup.Wait()

+

+	close(proc.pagesChan)

+

+	if err != nil {

+		return err

+	}

+

+	return proc.pagesGroup.Wait()

+}

+

+func (proc *pagesProcessor) newPageFromBundle(b *fileinfoBundle) (*pageState, error) {

+	p, err := proc.newPageFromFi(b.header, nil)

+	if err != nil {

+		return nil, err

+	}

+

+	if len(b.resources) > 0 {

+

+		resources := make(resource.Resources, len(b.resources))

+

+		for i, rfi := range b.resources {

+			meta := rfi.Meta()

+			classifier := meta.Classifier()

+			var r resource.Resource

+			switch classifier {

+			case files.ContentClassContent:

+				rp, err := proc.newPageFromFi(rfi, p)

+				if err != nil {

+					return nil, err

+				}

+				rp.m.resourcePath = filepath.ToSlash(strings.TrimPrefix(rp.Path(), p.File().Dir()))

+

+				r = rp

+

+			case files.ContentClassFile:

+				r, err = proc.newResource(rfi, p)

+				if err != nil {

+					return nil, err

+				}

+			default:

+				panic(fmt.Sprintf("invalid classifier: %q", classifier))

+			}

+

+			resources[i] = r

+

+		}

+

+		p.addResources(resources...)

+	}

+

+	return p, nil

+}

+

+func (proc *pagesProcessor) newPageFromFi(fim hugofs.FileMetaInfo, owner *pageState) (*pageState, error) {

+	fi, err := newFileInfo(proc.sp, fim)

+	if err != nil {

+		return nil, err

+	}

+

+	var s *Site

+	meta := fim.Meta()

+

+	if owner != nil {

+		s = owner.s

+	} else {

+		lang := meta.Lang()

+		s = proc.getSite(lang)

+	}

+

+	r := func() (hugio.ReadSeekCloser, error) {

+		return meta.Open()

+	}

+

+	p, err := newPageWithContent(fi, s, owner != nil, r)

+	if err != nil {

+		return nil, err

+	}

+	p.parent = owner

+	return p, nil

+}

+

+func (proc *pagesProcessor) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resource.Resource, error) {

+

+	// TODO(bep) consolidate with multihost logic + clean up

+	outputFormats := owner.m.outputFormats()

+	seen := make(map[string]bool)

+	var targetBasePaths []string

+	// Make sure bundled resources are published to all of the ouptput formats'

+	// sub paths.

+	for _, f := range outputFormats {

+		p := f.Path

+		if seen[p] {

+			continue

+		}

+		seen[p] = true

+		targetBasePaths = append(targetBasePaths, p)

+

+	}

+

+	meta := fim.Meta()

+	r := func() (hugio.ReadSeekCloser, error) {

+		return meta.Open()

+	}

+

+	target := strings.TrimPrefix(meta.Path(), owner.File().Dir())

+

+	return owner.s.ResourceSpec.New(

+		resources.ResourceSourceDescriptor{

+			TargetPaths:        owner.getTargetPaths,

+			OpenReadSeekCloser: r,

+			FileInfo:           fim,

+			RelTargetFilename:  target,

+			TargetBasePaths:    targetBasePaths,

+		})

+}

+

+func (proc *pagesProcessor) getSite(lang string) *Site {

+	if lang == "" {

+		return proc.h.Sites[0]

+	}

+

+	for _, s := range proc.h.Sites {

+		if lang == s.Lang() {

+			return s

+		}

+	}

+	return proc.h.Sites[0]

+}

+

+func (proc *pagesProcessor) copyFile(fim hugofs.FileMetaInfo) error {

+	meta := fim.Meta()

+	s := proc.getSite(meta.Lang())

+	f, err := meta.Open()

+	if err != nil {

+		return errors.Wrap(err, "copyFile: failed to open")

+	}

+

+	target := filepath.Join(s.PathSpec.GetTargetLanguageBasePath(), meta.Path())

+

+	defer f.Close()

+

+	return s.publish(&s.PathSpec.ProcessingStats.Files, target, f)

+

+}

+

+func (proc *pagesProcessor) process(item interface{}) error {

+	send := func(p *pageState, err error) {

+		if err != nil {

+			proc.sendError(err)

+		} else {

+			proc.pagesChan <- p

+		}

+	}

+

+	switch v := item.(type) {

+	// Page bundles mapped to their language.

+	case pageBundles:

+		for _, bundle := range v {

+			if proc.shouldSkip(bundle.header) {

+				continue

+			}

+			send(proc.newPageFromBundle(bundle))

+		}

+	case hugofs.FileMetaInfo:

+		if proc.shouldSkip(v) {

+			return nil

+		}

+		meta := v.Meta()

+

+		classifier := meta.Classifier()

+		switch classifier {

+		case files.ContentClassContent:

+			send(proc.newPageFromFi(v, nil))

+		case files.ContentClassFile:

+			proc.sendError(proc.copyFile(v))

+		default:

+			panic(fmt.Sprintf("invalid classifier: %q", classifier))

+		}

+	default:

+		panic(fmt.Sprintf("unrecognized item type in Process: %T", item))

+	}

+

+	return nil

+}

+

+func (proc *pagesProcessor) sendError(err error) {

+	if err == nil {

+		return

+	}

+	proc.h.SendError(err)

+}

+

+func (proc *pagesProcessor) shouldSkip(fim hugofs.FileMetaInfo) bool {

+	return proc.sp.DisabledLanguages[fim.Meta().Lang()]

+}

+

+func stringSliceContains(k string, values ...string) bool {

+	for _, v := range values {

+		if k == v {

+			return true

+		}

+	}

+	return false

+}

--- /dev/null

+++ b/hugolib/pages_capture_test.go

@@ -1,0 +1,88 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package hugolib

+

+import (

+	"context"

+	"fmt"

+	"path/filepath"

+	"testing"

+

+	"github.com/pkg/errors"

+

+	"github.com/gohugoio/hugo/helpers"

+	"github.com/gohugoio/hugo/source"

+

+	"github.com/gohugoio/hugo/common/loggers"

+

+	"github.com/gohugoio/hugo/hugofs"

+	"github.com/spf13/afero"

+	"github.com/stretchr/testify/require"

+)

+

+func TestPagesCapture(t *testing.T) {

+

+	cfg, hfs := newTestCfg()

+	fs := hfs.Source

+

+	assert := require.New(t)

+

+	var writeFile = func(filename string) {

+		assert.NoError(afero.WriteFile(fs, filepath.FromSlash(filename), []byte(fmt.Sprintf("content-%s", filename)), 0755))

+	}

+

+	writeFile("_index.md")

+	writeFile("logo.png")

+	writeFile("root.md")

+	writeFile("blog/index.md")

+	writeFile("blog/hello.md")

+	writeFile("blog/images/sunset.png")

+	writeFile("pages/page1.md")

+	writeFile("pages/page2.md")

+	writeFile("pages/page.png")

+

+	ps, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg)

+	assert.NoError(err)

+	sourceSpec := source.NewSourceSpec(ps, fs)

+

+	t.Run("Collect", func(t *testing.T) {

+		assert := require.New(t)

+		proc := &testPagesCollectorProcessor{}

+		c := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil, proc)

+		assert.NoError(c.Collect())

+		assert.Equal(4, len(proc.items))

+	})

+

+	t.Run("error in Wait", func(t *testing.T) {

+		assert := require.New(t)

+		c := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil,

+			&testPagesCollectorProcessor{waitErr: errors.New("failed")})

+		assert.Error(c.Collect())

+	})

+}

+

+type testPagesCollectorProcessor struct {

+	items   []interface{}

+	waitErr error

+}

+

+func (proc *testPagesCollectorProcessor) Process(item interface{}) error {

+	proc.items = append(proc.items, item)

+	return nil

+}

+func (proc *testPagesCollectorProcessor) Start(ctx context.Context) context.Context {

+	return ctx

+}

+

+func (proc *testPagesCollectorProcessor) Wait() error { return proc.waitErr }

--- a/hugolib/paths/paths.go

+++ b/hugolib/paths/paths.go

@@ -20,6 +20,7 @@

 	"github.com/gohugoio/hugo/config"

 	"github.com/gohugoio/hugo/langs"

+	"github.com/gohugoio/hugo/modules"

 	"github.com/pkg/errors"

 	"github.com/gohugoio/hugo/hugofs"

@@ -39,7 +40,6 @@

 	// Directories

 	// TODO(bep) when we have trimmed down mos of the dirs usage outside of this package, make

 	// these into an interface.

-	ContentDir string

 	ThemesDir  string

 	WorkingDir string

@@ -62,8 +62,9 @@

 	UglyURLs           bool

 	CanonifyURLs       bool

-	Language  *langs.Language

-	Languages langs.Languages

+	Language              *langs.Language

+	Languages             langs.Languages

+	LanguagesDefaultFirst langs.Languages

 	// The PathSpec looks up its config settings in both the current language

 	// and then in the global Viper config.

@@ -74,8 +75,8 @@

 	DefaultContentLanguage         string

 	multilingual                   bool

-	themes    []string

-	AllThemes []ThemeConfig

+	AllModules    modules.Modules

+	ModulesClient *modules.Client

 }

 func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) {

@@ -91,12 +92,6 @@

 	resourceDir := filepath.Clean(cfg.GetString("resourceDir"))

 	publishDir := filepath.Clean(cfg.GetString("publishDir"))

-	if contentDir == "" {

-		return nil, fmt.Errorf("contentDir not set")

-	}

-	if resourceDir == "" {

-		return nil, fmt.Errorf("resourceDir not set")

-	}

 	if publishDir == "" {

 		return nil, fmt.Errorf("publishDir not set")

 	}

@@ -104,8 +99,9 @@

 	defaultContentLanguage := cfg.GetString("defaultContentLanguage")

 	var (

-		language  *langs.Language

-		languages langs.Languages

+		language              *langs.Language

+		languages             langs.Languages

+		languagesDefaultFirst langs.Languages

 	)

 	if l, ok := cfg.(*langs.Language); ok {

@@ -117,6 +113,12 @@

 		languages = l

 	}

+	if l, ok := cfg.Get("languagesSortedDefaultFirst").(langs.Languages); ok {

+		languagesDefaultFirst = l

+	}

+

+	//

+

 	if len(languages) == 0 {

 		// We have some old tests that does not test the entire chain, hence

 		// they have no languages. So create one so we get the proper filesystem.

@@ -156,7 +158,6 @@

 		UglyURLs:           cfg.GetBool("uglyURLs"),

 		CanonifyURLs:       cfg.GetBool("canonifyURLs"),

-		ContentDir: contentDir,

 		ThemesDir:  cfg.GetString("themesDir"),

 		WorkingDir: workingDir,

@@ -163,8 +164,6 @@

 		AbsResourcesDir: absResourcesDir,

 		AbsPublishDir:   absPublishDir,

-		themes: config.GetStringSlicePreserveString(cfg, "theme"),

-

 		multilingual:                   cfg.GetBool("multilingual"),

 		defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"),

 		DefaultContentLanguage:         defaultContentLanguage,

@@ -171,20 +170,20 @@

 		Language:                 language,

 		Languages:                languages,

+		LanguagesDefaultFirst:    languagesDefaultFirst,

 		MultihostTargetBasePaths: multihostTargetBasePaths,

 		PaginatePath: cfg.GetString("paginatePath"),

 	}

-	if !cfg.IsSet("theme") && cfg.IsSet("allThemes") {

-		p.AllThemes = cfg.Get("allThemes").([]ThemeConfig)

-	} else {

-		p.AllThemes, err = collectThemeNames(p)

-		if err != nil {

-			return nil, err

-		}

+	if cfg.IsSet("allModules") {

+		p.AllModules = cfg.Get("allModules").(modules.Modules)

 	}

+	if cfg.IsSet("modulesClient") {

+		p.ModulesClient = cfg.Get("modulesClient").(*modules.Client)

+	}

+

 	// TODO(bep) remove this, eventually

 	p.PublishDir = absPublishDir

@@ -207,15 +206,6 @@

 	return p.Language.Lang

 }

-// ThemeSet checks whether a theme is in use or not.

-func (p *Paths) ThemeSet() bool {

-	return len(p.themes) > 0

-}

-

-func (p *Paths) Themes() []string {

-	return p.themes

-}

-

 func (p *Paths) GetTargetLanguageBasePath() string {

 	if p.Languages.IsMultihost() {

 		// In a multihost configuration all assets will be published below the language code.

@@ -267,6 +257,18 @@

 // absolute, the path is just cleaned.

 func (p *Paths) AbsPathify(inPath string) string {

 	return AbsPathify(p.WorkingDir, inPath)

+}

+

+// RelPathify trims any WorkingDir prefix from the given filename. If

+// the filename is not considered to be absolute, the path is just cleaned.

+func (p *Paths) RelPathify(filename string) string {

+	filename = filepath.Clean(filename)

+	if !filepath.IsAbs(filename) {

+		return filename

+	}

+

+	return strings.TrimPrefix(strings.TrimPrefix(filename, p.WorkingDir), FilePathSeparator)

+

 }

 // AbsPathify creates an absolute path if given a working dir and arelative path.

--- a/hugolib/paths/paths_test.go

+++ b/hugolib/paths/paths_test.go

@@ -16,6 +16,8 @@

 import (

 	"testing"

+	"github.com/gohugoio/hugo/langs"

+

 	"github.com/gohugoio/hugo/hugofs"

 	"github.com/spf13/viper"

 	"github.com/stretchr/testify/require"

@@ -27,13 +29,18 @@

 	v := viper.New()

 	fs := hugofs.NewMem(v)

+	v.Set("languages", map[string]interface{}{

+		"no": map[string]interface{}{},

+		"en": map[string]interface{}{},

+	})

 	v.Set("defaultContentLanguageInSubdir", true)

 	v.Set("defaultContentLanguage", "no")

-	v.Set("multilingual", true)

 	v.Set("contentDir", "content")

 	v.Set("workingDir", "work")

 	v.Set("resourceDir", "resources")

 	v.Set("publishDir", "public")

+

+	langs.LoadLanguageSettings(v, nil)

 	p, err := New(fs, v)

 	assert.NoError(err)

--- a/hugolib/paths/themes.go

+++ /dev/null

@@ -1,154 +1,0 @@

-// Copyright 2019 The Hugo Authors. All rights reserved.

-//

-// Licensed under the Apache License, Version 2.0 (the "License");

-// you may not use this file except in compliance with the License.

-// You may obtain a copy of the License at

-// http://www.apache.org/licenses/LICENSE-2.0

-//

-// Unless required by applicable law or agreed to in writing, software

-// distributed under the License is distributed on an "AS IS" BASIS,

-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-// See the License for the specific language governing permissions and

-// limitations under the License.

-

-package paths

-

-import (

-	"path/filepath"

-	"strings"

-

-	"github.com/gohugoio/hugo/config"

-	"github.com/spf13/afero"

-	"github.com/spf13/cast"

-)

-

-type ThemeConfig struct {

-	// The theme name as provided by the folder name below /themes.

-	Name string

-

-	// Optional configuration filename (e.g. "/themes/mytheme/config.json").

-	ConfigFilename string

-

-	// Optional config read from the ConfigFile above.

-	Cfg config.Provider

-}

-

-// Create file system, an ordered theme list from left to right, no duplicates.

-type themesCollector struct {

-	themesDir string

-	fs        afero.Fs

-	seen      map[string]bool

-	themes    []ThemeConfig

-}

-

-func (c *themesCollector) isSeen(theme string) bool {

-	loki := strings.ToLower(theme)

-	if c.seen[loki] {

-		return true

-	}

-	c.seen[loki] = true

-	return false

-}

-

-func (c *themesCollector) addAndRecurse(themes ...string) error {

-	for i := 0; i < len(themes); i++ {

-		theme := themes[i]

-		configFilename := c.getConfigFileIfProvided(theme)

-		if !c.isSeen(theme) {

-			tc, err := c.add(theme, configFilename)

-			if err != nil {

-				return err

-			}

-			if err := c.addThemeNamesFromTheme(tc); err != nil {

-				return err

-			}

-		}

-	}

-	return nil

-}

-

-func (c *themesCollector) add(name, configFilename string) (ThemeConfig, error) {

-	var cfg config.Provider

-	var tc ThemeConfig

-

-	if configFilename != "" {

-		var err error

-		cfg, err = config.FromFile(c.fs, configFilename)

-		if err != nil {

-			return tc, err

-		}

-	}

-

-	tc = ThemeConfig{Name: name, ConfigFilename: configFilename, Cfg: cfg}

-	c.themes = append(c.themes, tc)

-	return tc, nil

-

-}

-

-func collectThemeNames(p *Paths) ([]ThemeConfig, error) {

-	return CollectThemes(p.Fs.Source, p.AbsPathify(p.ThemesDir), p.Themes())

-

-}

-

-func CollectThemes(fs afero.Fs, themesDir string, themes []string) ([]ThemeConfig, error) {

-	if len(themes) == 0 {

-		return nil, nil

-	}

-

-	c := &themesCollector{

-		fs:        fs,

-		themesDir: themesDir,

-		seen:      make(map[string]bool)}

-

-	for i := 0; i < len(themes); i++ {

-		theme := themes[i]

-		if err := c.addAndRecurse(theme); err != nil {

-			return nil, err

-		}

-	}

-

-	return c.themes, nil

-

-}

-

-func (c *themesCollector) getConfigFileIfProvided(theme string) string {

-	configDir := filepath.Join(c.themesDir, theme)

-

-	var (

-		configFilename string

-		exists         bool

-	)

-

-	// Viper supports more, but this is the sub-set supported by Hugo.

-	for _, configFormats := range config.ValidConfigFileExtensions {

-		configFilename = filepath.Join(configDir, "config."+configFormats)

-		exists, _ = afero.Exists(c.fs, configFilename)

-		if exists {

-			break

-		}

-	}

-

-	if !exists {

-		// No theme config set.

-		return ""

-	}

-

-	return configFilename

-

-}

-

-func (c *themesCollector) addThemeNamesFromTheme(theme ThemeConfig) error {

-	if theme.Cfg != nil && theme.Cfg.IsSet("theme") {

-		v := theme.Cfg.Get("theme")

-		switch vv := v.(type) {

-		case []string:

-			return c.addAndRecurse(vv...)

-		case []interface{}:

-			return c.addAndRecurse(cast.ToStringSlice(vv)...)

-		default:

-			return c.addAndRecurse(cast.ToString(vv))

-		}

-	}

-

-	return nil

-}

--- a/hugolib/resource_chain_test.go

+++ b/hugolib/resource_chain_test.go

@@ -18,6 +18,8 @@

 	"path/filepath"

 	"testing"

+	"github.com/gohugoio/hugo/htesting"

+

 	"github.com/spf13/viper"

 	"github.com/stretchr/testify/require"

@@ -33,7 +35,7 @@

 		t.Skip("Skip SCSS")

 	}

 	assert := require.New(t)

-	workDir, clean, err := createTempDir("hugo-scss-include")

+	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-scss-include")

 	assert.NoError(err)

 	defer clean()

@@ -40,10 +42,10 @@

 	v := viper.New()

 	v.Set("workingDir", workDir)

 	b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger())

-	b.WithViper(v)

-	b.WithWorkingDir(workDir)

 	// Need to use OS fs for this.

 	b.Fs = hugofs.NewDefault(v)

+	b.WithWorkingDir(workDir)

+	b.WithViper(v)

 	fooDir := filepath.Join(workDir, "node_modules", "foo")

 	scssDir := filepath.Join(workDir, "assets", "scss")

@@ -84,7 +86,7 @@

 		t.Skip("Skip SCSS")

 	}

 	assert := require.New(t)

-	workDir, clean, err := createTempDir("hugo-scss-include")

+	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-scss-include")

 	assert.NoError(err)

 	defer clean()

@@ -95,10 +97,10 @@

 	v.Set("workingDir", workDir)

 	v.Set("theme", theme)

 	b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger())

-	b.WithViper(v)

-	b.WithWorkingDir(workDir)

 	// Need to use OS fs for this.

 	b.Fs = hugofs.NewDefault(v)

+	b.WithWorkingDir(workDir)

+	b.WithViper(v)

 	fooDir := filepath.Join(workDir, "node_modules", "foo")

 	scssDir := filepath.Join(workDir, "assets", "scss")

@@ -385,14 +387,15 @@

 	}

 	for _, test := range tests {

-		if !test.shouldRun() {

-			t.Log("Skip", test.name)

-			continue

-		}

+		test := test

+		t.Run(test.name, func(t *testing.T) {

+			if !test.shouldRun() {

+				t.Skip()

+			}

+			t.Parallel()

-		b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger())

-		b.WithSimpleConfigFile()

-		b.WithContent("_index.md", `

+			b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger())

+			b.WithContent("_index.md", `

 ---

 title: Home

 ---

@@ -400,7 +403,7 @@

 Home.

 `,

-			"page1.md", `

+				"page1.md", `

 ---

 title: Hello1

 ---

@@ -407,7 +410,7 @@

 Hello1

 `,

-			"page2.md", `

+				"page2.md", `

 ---

 title: Hello2

 ---

@@ -414,23 +417,23 @@

 Hello2

 `,

-			"t1.txt", "t1t|",

-			"t2.txt", "t2t|",

-		)

+				"t1.txt", "t1t|",

+				"t2.txt", "t2t|",

+			)

-		b.WithSourceFile(filepath.Join("assets", "css", "styles1.css"), `

+			b.WithSourceFile(filepath.Join("assets", "css", "styles1.css"), `

 h1 {

 	 font-style: bold;

 }

 `)

-		b.WithSourceFile(filepath.Join("assets", "js", "script1.js"), `

+			b.WithSourceFile(filepath.Join("assets", "js", "script1.js"), `

 var x;

 x = 5;

 document.getElementById("demo").innerHTML = x * 10;

 `)

-		b.WithSourceFile(filepath.Join("assets", "mydata", "json1.json"), `

+			b.WithSourceFile(filepath.Join("assets", "mydata", "json1.json"), `

 {

 "employees":[

     {"firstName":"John", "lastName":"Doe"}, 

@@ -440,19 +443,19 @@

 }

 `)

-		b.WithSourceFile(filepath.Join("assets", "mydata", "svg1.svg"), `

+			b.WithSourceFile(filepath.Join("assets", "mydata", "svg1.svg"), `

 <svg height="100" width="100">

   <line x1="5" y1="10" x2="20" y2="40"/>

 </svg> 

 `)

-		b.WithSourceFile(filepath.Join("assets", "mydata", "xml1.xml"), `

+			b.WithSourceFile(filepath.Join("assets", "mydata", "xml1.xml"), `

 <hello>

 <world>Hugo Rocks!</<world>

 </hello>

 `)

-		b.WithSourceFile(filepath.Join("assets", "mydata", "html1.html"), `

+			b.WithSourceFile(filepath.Join("assets", "mydata", "html1.html"), `

 <html>

 <a  href="#">

 Cool

@@ -460,7 +463,7 @@

 </html>

 `)

-		b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), `

+			b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), `

 $color: #333;

 body {

@@ -468,7 +471,7 @@

 }

 `)

-		b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), `

+			b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), `

 $color: #333;

 .content-navigation

@@ -476,10 +479,11 @@

 `)

-		t.Log("Test", test.name)

-		test.prepare(b)

-		b.Build(BuildCfg{})

-		test.verify(b)

+			test.prepare(b)

+			b.Build(BuildCfg{})

+			test.verify(b)

+

+		})

 	}

 }

--- a/hugolib/shortcode_test.go

+++ b/hugolib/shortcode_test.go

@@ -26,10 +26,6 @@

 	"strings"

 	"testing"

-	"github.com/spf13/viper"

-

-	"github.com/spf13/afero"

-

 	"github.com/gohugoio/hugo/deps"

 	"github.com/gohugoio/hugo/helpers"

 	"github.com/gohugoio/hugo/tpl"

@@ -54,13 +50,9 @@

 	writeSource(t, fs, "content/simple.md", contentFile)

-	h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate})

+	b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}).WithNothingAdded()

+	err := b.BuildE(BuildCfg{})

-	require.NoError(t, err)

-	require.Len(t, h.Sites, 1)

-

-	err = h.Build(BuildCfg{})

-

 	if err != nil && !expectError {

 		t.Fatalf("Shortcode rendered error %s.", err)

 	}

@@ -69,6 +61,9 @@

 		t.Fatalf("No error from shortcode")

 	}

+	h := b.H

+	require.Len(t, h.Sites, 1)

+

 	require.Len(t, h.Sites[0].RegularPages(), 1)

 	output := strings.TrimSpace(content(h.Sites[0].RegularPages()[0]))

@@ -78,7 +73,7 @@

 	expected = strings.TrimSpace(expected)

 	if output != expected {

-		Fatalf(t, "Shortcode render didn't match. got \n%q but expected \n%q", output, expected)

+		t.Fatalf("Shortcode render didn't match. got \n%q but expected \n%q", output, expected)

 	}

 }

@@ -341,7 +336,6 @@

 }

 func TestExtractShortcodes(t *testing.T) {

-	t.Parallel()

 	b := newTestSitesBuilder(t).WithSimpleConfigFile()

 	b.WithTemplates(

@@ -413,7 +407,10 @@

 		{"inline", `{{< my.inline >}}Hi{{< /my.inline >}}`, regexpCheck("my.inline;inline:true;closing:true;inner:{Hi};")},

 	} {

+		test := test

+

 		t.Run(test.name, func(t *testing.T) {

+			t.Parallel()

 			assert := require.New(t)

 			counter := 0

@@ -437,7 +434,6 @@

 }

 func TestShortcodesInSite(t *testing.T) {

-	t.Parallel()

 	baseURL := "http://foo/bar"

 	tests := []struct {

@@ -577,7 +573,9 @@

 	s := buildSingleSite(t, deps.DepsCfg{WithTemplate: addTemplates, Fs: fs, Cfg: cfg}, BuildCfg{})

 	for i, test := range tests {

+		test := test

 		t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) {

+			t.Parallel()

 			if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() {

 				t.Skip("Skip Asciidoc test case as no Asciidoc present.")

 			} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {

@@ -632,9 +630,8 @@

 CSV: {{< myShort >}}

 `

-	mf := afero.NewMemMapFs()

-

-	th, h := newTestSitesFromConfig(t, mf, siteConfig,

+	b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)

+	b.WithTemplates(

 		"layouts/_default/single.html", `Single HTML: {{ .Title }}|{{ .Content }}`,

 		"layouts/_default/single.json", `Single JSON: {{ .Title }}|{{ .Content }}`,

 		"layouts/_default/single.csv", `Single CSV: {{ .Title }}|{{ .Content }}`,

@@ -651,14 +648,13 @@

 		"layouts/shortcodes/myInner.html", `myInner:--{{- .Inner -}}--`,

 	)

-	fs := th.Fs

+	b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "Home"),

+		"sect/mypage.md", fmt.Sprintf(pageTemplate, "Single"),

+		"sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV"),

+	)

-	writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "Home"))

-	writeSource(t, fs, "content/sect/mypage.md", fmt.Sprintf(pageTemplate, "Single"))

-	writeSource(t, fs, "content/sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV"))

-

-	err := h.Build(BuildCfg{})

-	require.NoError(t, err)

+	b.Build(BuildCfg{})

+	h := b.H

 	require.Len(t, h.Sites, 1)

 	s := h.Sites[0]

@@ -666,7 +662,7 @@

 	require.NotNil(t, home)

 	require.Len(t, home.OutputFormats(), 3)

-	th.assertFileContent("public/index.html",

+	b.AssertFileContent("public/index.html",

 		"Home HTML",

 		"ShortHTML",

 		"ShortNoExt",

@@ -674,7 +670,7 @@

 		"myInner:--ShortHTML--",

 	)

-	th.assertFileContent("public/amp/index.html",

+	b.AssertFileContent("public/amp/index.html",

 		"Home AMP",

 		"ShortAMP",

 		"ShortNoExt",

@@ -682,7 +678,7 @@

 		"myInner:--ShortAMP--",

 	)

-	th.assertFileContent("public/index.ics",

+	b.AssertFileContent("public/index.ics",

 		"Home Calendar",

 		"ShortCalendar",

 		"ShortNoExt",

@@ -690,7 +686,7 @@

 		"myInner:--ShortCalendar--",

 	)

-	th.assertFileContent("public/sect/mypage/index.html",

+	b.AssertFileContent("public/sect/mypage/index.html",

 		"Single HTML",

 		"ShortHTML",

 		"ShortNoExt",

@@ -698,7 +694,7 @@

 		"myInner:--ShortHTML--",

 	)

-	th.assertFileContent("public/sect/mypage/index.json",

+	b.AssertFileContent("public/sect/mypage/index.json",

 		"Single JSON",

 		"ShortJSON",

 		"ShortNoExt",

@@ -706,7 +702,7 @@

 		"myInner:--ShortJSON--",

 	)

-	th.assertFileContent("public/amp/sect/mypage/index.html",

+	b.AssertFileContent("public/amp/sect/mypage/index.html",

 		// No special AMP template

 		"Single HTML",

 		"ShortAMP",

@@ -715,7 +711,7 @@

 		"myInner:--ShortAMP--",

 	)

-	th.assertFileContent("public/sect/mycsvpage/index.csv",

+	b.AssertFileContent("public/sect/mycsvpage/index.csv",

 		"Single CSV",

 		"ShortCSV",

 	)

@@ -864,10 +860,6 @@

 ---

 C-%s`

-	v := viper.New()

-

-	v.Set("timeout", 500)

-

 	templates = append(templates, []string{"shortcodes/c.html", contentShortcode}...)

 	templates = append(templates, []string{"_default/single.html", "Single Content: {{ .Content }}"}...)

 	templates = append(templates, []string{"_default/list.html", "List Content: {{ .Content }}"}...)

@@ -884,21 +876,21 @@

 	builder := newTestSitesBuilder(t).WithDefaultMultiSiteConfig()

-	builder.WithViper(v).WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{})

+	builder.WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{})

 	s := builder.H.Sites[0]

 	assert.Equal(3, len(s.RegularPages()))

-	builder.AssertFileContent("public/section1/index.html",

+	builder.AssertFileContent("public/en/section1/index.html",

 		"List Content: <p>Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/<p>C-s1p1</p>\n|",

 		"BP1:P1:|P2:docbp1/<p>C-bp1</p>",

 	)

-	builder.AssertFileContent("public/b1/index.html",

+	builder.AssertFileContent("public/en/b1/index.html",

 		"Single Content: <p>Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/<p>C-s1p1</p>\n|",

 		"P2:docbp1/<p>C-bp1</p>",

 	)

-	builder.AssertFileContent("public/section2/s2p1/index.html",

+	builder.AssertFileContent("public/en/section2/s2p1/index.html",

 		"Single Content: <p>Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/<p>C-s1p1</p>\n|",

 		"P2:docbp1/<p>C-bp1</p>",

 	)

@@ -1062,8 +1054,10 @@

 func TestInlineShortcodes(t *testing.T) {

 	for _, enableInlineShortcodes := range []bool{true, false} {

+		enableInlineShortcodes := enableInlineShortcodes

 		t.Run(fmt.Sprintf("enableInlineShortcodes=%t", enableInlineShortcodes),

 			func(t *testing.T) {

+				t.Parallel()

 				conf := fmt.Sprintf(`

 baseURL = "https://example.com"

 enableInlineShortcodes = %t

--- a/hugolib/site.go

+++ b/hugolib/site.go

@@ -14,7 +14,6 @@

 package hugolib

 import (

-	"context"

 	"fmt"

 	"html/template"

 	"io"

@@ -29,6 +28,8 @@

 	"strings"

 	"time"

+	"github.com/gohugoio/hugo/hugofs/files"

+

 	"github.com/gohugoio/hugo/common/maps"

 	"github.com/pkg/errors"

@@ -45,7 +46,6 @@

 	"github.com/gohugoio/hugo/config"

 	"github.com/gohugoio/hugo/lazy"

-	"golang.org/x/sync/errgroup"

 	"github.com/gohugoio/hugo/media"

@@ -1028,7 +1028,8 @@

 				removed = true

 			}

 		}

-		if removed && IsContentFile(ev.Name) {

+

+		if removed && files.IsContentFile(ev.Name) {

 			h.removePageByFilename(ev.Name)

 		}

@@ -1058,7 +1059,7 @@

 			filenamesChanged = append(filenamesChanged, contentFilesChanged...)

 		}

-		filenamesChanged = helpers.UniqueStrings(filenamesChanged)

+		filenamesChanged = helpers.UniqueStringsReuse(filenamesChanged)

 		if err := s.readAndProcessContent(filenamesChanged...); err != nil {

 			return whatChanged{}, err

@@ -1078,10 +1079,12 @@

 func (s *Site) process(config BuildCfg) (err error) {

 	if err = s.initialize(); err != nil {

+		err = errors.Wrap(err, "initialize")

 		return

 	}

-	if err := s.readAndProcessContent(); err != nil {

-		return err

+	if err = s.readAndProcessContent(); err != nil {

+		err = errors.Wrap(err, "readAndProcessContent")

+		return

 	}

 	return err

@@ -1304,93 +1307,14 @@

 	return s.BaseFs.IsContent(e.Name)

 }

-type contentCaptureResultHandler struct {

-	defaultContentProcessor *siteContentProcessor

-	contentProcessors       map[string]*siteContentProcessor

-}

-

-func (c *contentCaptureResultHandler) getContentProcessor(lang string) *siteContentProcessor {

-	proc, found := c.contentProcessors[lang]

-	if found {

-		return proc

-	}

-	return c.defaultContentProcessor

-}

-

-func (c *contentCaptureResultHandler) handleSingles(fis ...*fileInfo) {

-	for _, fi := range fis {

-		proc := c.getContentProcessor(fi.Lang())

-		proc.processSingle(fi)

-	}

-}

-func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) {

-	for _, b := range d.bundles {

-		proc := c.getContentProcessor(b.fi.Lang())

-		proc.processBundle(b)

-	}

-}

-

-func (c *contentCaptureResultHandler) handleCopyFile(f pathLangFile) {

-	proc := c.getContentProcessor(f.Lang())

-	proc.processAsset(f)

-}

-

 func (s *Site) readAndProcessContent(filenames ...string) error {

-

-	ctx := context.Background()

-	g, ctx := errgroup.WithContext(ctx)

-

-	defaultContentLanguage := s.SourceSpec.DefaultContentLanguage

-

-	contentProcessors := make(map[string]*siteContentProcessor)

-	var defaultContentProcessor *siteContentProcessor

-	sites := s.h.langSite()

-	for k, v := range sites {

-		if v.language.Disabled {

-			continue

-		}

-		proc := newSiteContentProcessor(ctx, len(filenames) > 0, v)

-		contentProcessors[k] = proc

-		if k == defaultContentLanguage {

-			defaultContentProcessor = proc

-		}

-		g.Go(func() error {

-			return proc.process(ctx)

-		})

-	}

-

-	var (

-		handler   captureResultHandler

-		bundleMap *contentChangeMap

-	)

-

-	mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor}

-

 	sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs)

-	if s.running() {

-		// Need to track changes.

-		bundleMap = s.h.ContentChanges

-		handler = &captureResultHandlerChain{handlers: []captureBundlesHandler{mainHandler, bundleMap}}

+	proc := newPagesProcessor(s.h, sourceSpec, len(filenames) > 0)

-	} else {

-		handler = mainHandler

-	}

+	c := newPagesCollector(sourceSpec, s.Log, s.h.ContentChanges, proc, filenames...)

-	c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...)

-

-	err1 := c.capture()

-

-	for _, proc := range contentProcessors {

-		proc.closeInput()

-	}

-

-	err2 := g.Wait()

-

-	if err1 != nil {

-		return err1

-	}

-	return err2

+	return c.Collect()

 }

 func (s *Site) getMenusFromConfig() navigation.Menus {

@@ -1831,8 +1755,8 @@

 }

 func (s *Site) kindFromSections(sections []string) string {

-	if len(sections) == 0 || len(s.siteCfg.taxonomiesConfig) == 0 {

-		return page.KindSection

+	if len(sections) == 0 {

+		return page.KindHome

 	}

 	sectionPath := path.Join(sections...)

--- a/hugolib/site_output_test.go

+++ b/hugolib/site_output_test.go

@@ -32,7 +32,9 @@

 func TestSiteWithPageOutputs(t *testing.T) {

 	for _, outputs := range [][]string{{"html", "json", "calendar"}, {"json"}} {

+		outputs := outputs

 		t.Run(fmt.Sprintf("%v", outputs), func(t *testing.T) {

+			t.Parallel()

 			doTestSiteWithPageOutputs(t, outputs)

 		})

 	}

@@ -39,7 +41,6 @@

 }

 func doTestSiteWithPageOutputs(t *testing.T, outputs []string) {

-	t.Parallel()

 	outputsStr := strings.Replace(fmt.Sprintf("%q", outputs), " ", ", ", -1)

@@ -84,19 +85,16 @@

 `

-	mf := afero.NewMemMapFs()

-

-	writeToFs(t, mf, "i18n/en.toml", `

+	b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)

+	b.WithI18n("en.toml", `

 [elbow]

 other = "Elbow"

-`)

-	writeToFs(t, mf, "i18n/nn.toml", `

+`, "nn.toml", `

 [elbow]

 other = "Olboge"

 `)

-	th, h := newTestSitesFromConfig(t, mf, siteConfig,

-

+	b.WithTemplates(

 		// Case issue partials #3333

 		"layouts/partials/GoHugo.html", `Go Hugo Partial`,

 		"layouts/_default/baseof.json", `START JSON:{{block "main" .}}default content{{ end }}:END JSON`,

@@ -133,23 +131,17 @@

 `,

 		"layouts/_default/single.html", `{{ define "main" }}{{ .Content }}{{ end }}`,

 	)

-	require.Len(t, h.Sites, 2)

-	fs := th.Fs

+	b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr))

+	b.WithContent("_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr))

-	writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr))

-	writeSource(t, fs, "content/_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr))

-

 	for i := 1; i <= 10; i++ {

-		writeSource(t, fs, fmt.Sprintf("content/p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr))

-

+		b.WithContent(fmt.Sprintf("p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr))

 	}

-	err := h.Build(BuildCfg{})

+	b.Build(BuildCfg{})

-	require.NoError(t, err)

-

-	s := h.Sites[0]

+	s := b.H.Sites[0]

 	require.Equal(t, "en", s.language.Lang)

 	home := s.getPage(page.KindHome)

@@ -163,13 +155,13 @@

 	// There is currently always a JSON output to make it simpler ...

 	altFormats := lenOut - 1

 	hasHTML := helpers.InStringArray(outputs, "html")

-	th.assertFileContent("public/index.json",

+	b.AssertFileContent("public/index.json",

 		"List JSON",

 		fmt.Sprintf("Alt formats: %d", altFormats),

 	)

 	if hasHTML {

-		th.assertFileContent("public/index.json",

+		b.AssertFileContent("public/index.json",

 			"Alt Output: HTML",

 			"Output/Rel: JSON/alternate|",

 			"Output/Rel: HTML/canonical|",

@@ -178,7 +170,7 @@

 			"OtherShort: <h1>Hi!</h1>",

 		)

-		th.assertFileContent("public/index.html",

+		b.AssertFileContent("public/index.html",

 			// The HTML entity is a deliberate part of this test: The HTML templates are

 			// parsed with html/template.

 			`List HTML|JSON Home|<atom:link href=http://example.com/blog/ rel="self" type="text/html" />`,

@@ -187,14 +179,15 @@

 			"OtherShort: <h1>Hi!</h1>",

 			"Len Pages: home 10",

 		)

-		th.assertFileContent("public/page/2/index.html", "Page Number: 2")

-		th.assertFileNotExist("public/page/2/index.json")

+		assert := require.New(t)

+		b.AssertFileContent("public/page/2/index.html", "Page Number: 2")

+		assert.False(b.CheckExists("public/page/2/index.json"))

-		th.assertFileContent("public/nn/index.html",

+		b.AssertFileContent("public/nn/index.html",

 			"List HTML|JSON Nynorsk Heim|",

 			"nn: Olboge")

 	} else {

-		th.assertFileContent("public/index.json",

+		b.AssertFileContent("public/index.json",

 			"Output/Rel: JSON/canonical|",

 			// JSON is plain text, so no need to safeHTML this and that

 			`<atom:link href=http://example.com/blog/index.json rel="self" type="application/json" />`,

@@ -201,7 +194,7 @@

 			"ShortJSON",

 			"OtherShort: <h1>Hi!</h1>",

 		)

-		th.assertFileContent("public/nn/index.json",

+		b.AssertFileContent("public/nn/index.json",

 			"List JSON|JSON Nynorsk Heim|",

 			"nn: Olboge",

 			"ShortJSON",

--- a/hugolib/site_sections_test.go

+++ b/hugolib/site_sections_test.go

@@ -25,7 +25,6 @@

 )

 func TestNestedSections(t *testing.T) {

-	t.Parallel()

 	var (

 		assert  = require.New(t)

@@ -294,7 +293,9 @@

 	home := s.getPage(page.KindHome)

 	for _, test := range tests {

+		test := test

 		t.Run(fmt.Sprintf("sections %s", test.sections), func(t *testing.T) {

+			t.Parallel()

 			assert := require.New(t)

 			sections := strings.Split(test.sections, ",")

 			p := s.getPage(page.KindSection, sections...)

--- a/hugolib/site_stats_test.go

+++ b/hugolib/site_stats_test.go

@@ -20,7 +20,6 @@

 	"testing"

 	"github.com/gohugoio/hugo/helpers"

-	"github.com/spf13/afero"

 	"github.com/stretchr/testify/require"

 )

@@ -60,30 +59,28 @@

 # Doc

 `

-	th, h := newTestSitesFromConfig(t, afero.NewMemMapFs(), siteConfig,

-		"layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}",

-		"layouts/_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`,

-		"layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",

+	b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)

+

+	b.WithTemplates(

+		"_default/single.html", "Single|{{ .Title }}|{{ .Content }}",

+		"_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`,

+		"_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",

 	)

-	require.Len(t, h.Sites, 2)

-	fs := th.Fs

-

 	for i := 0; i < 2; i++ {

 		for j := 0; j < 2; j++ {

 			pageID := i + j + 1

-			writeSource(t, fs, fmt.Sprintf("content/sect/p%d.md", pageID),

+			b.WithContent(fmt.Sprintf("content/sect/p%d.md", pageID),

 				fmt.Sprintf(pageTemplate, pageID, fmt.Sprintf("- tag%d", j), fmt.Sprintf("- category%d", j), pageID))

 		}

 	}

 	for i := 0; i < 5; i++ {

-		writeSource(t, fs, fmt.Sprintf("content/assets/image%d.png", i+1), "image")

+		b.WithContent(fmt.Sprintf("assets/image%d.png", i+1), "image")

 	}

-	err := h.Build(BuildCfg{})

-

-	assert.NoError(err)

+	b.Build(BuildCfg{})

+	h := b.H

 	stats := []*helpers.ProcessingStats{

 		h.Sites[0].PathSpec.ProcessingStats,

--- a/hugolib/site_test.go

+++ b/hugolib/site_test.go

@@ -362,11 +362,14 @@

 // Issue #1176

 func TestSectionNaming(t *testing.T) {

-	t.Parallel()

 	for _, canonify := range []bool{true, false} {

 		for _, uglify := range []bool{true, false} {

 			for _, pluralize := range []bool{true, false} {

+				canonify := canonify

+				uglify := uglify

+				pluralize := pluralize

 				t.Run(fmt.Sprintf("canonify=%t,uglify=%t,pluralize=%t", canonify, uglify, pluralize), func(t *testing.T) {

+					t.Parallel()

 					doTestSectionNaming(t, canonify, uglify, pluralize)

 				})

 			}

--- a/hugolib/taxonomy_test.go

+++ b/hugolib/taxonomy_test.go

@@ -64,7 +64,9 @@

 //

 func TestTaxonomiesWithAndWithoutContentFile(t *testing.T) {

 	for _, uglyURLs := range []bool{false, true} {

+		uglyURLs := uglyURLs

 		t.Run(fmt.Sprintf("uglyURLs=%t", uglyURLs), func(t *testing.T) {

+			t.Parallel()

 			doTestTaxonomiesWithAndWithoutContentFile(t, uglyURLs)

 		})

 	}

@@ -71,7 +73,6 @@

 }

 func doTestTaxonomiesWithAndWithoutContentFile(t *testing.T, uglyURLs bool) {

-	t.Parallel()

 	siteConfig := `

 baseURL = "http://example.com/blog"

@@ -104,26 +105,21 @@

 	siteConfig = fmt.Sprintf(siteConfig, uglyURLs)

-	th, h := newTestSitesFromConfigWithDefaultTemplates(t, siteConfig)

-	require.Len(t, h.Sites, 1)

+	b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)

-	fs := th.Fs

+	b.WithContent(

+		"p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1"),

+		"p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1"),

+		"p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1"),

+		"p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1"),

+		"categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10),

+		"tags/Tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10),

+		// https://github.com/gohugoio/hugo/issues/5847

+		"/tags/not-used/_index.md", newTestPage("Unused Tag List", "2018-01-01", 10),

+	)

-	writeSource(t, fs, "content/p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1"))

-	writeSource(t, fs, "content/p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1"))

-	writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1"))

-	writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1"))

+	b.Build(BuildCfg{})

-	writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)

-	writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10)

-

-	// https://github.com/gohugoio/hugo/issues/5847

-	writeNewContentFile(t, fs.Source, "Unused Tag List", "2018-01-01", "content/tags/not-used/_index.md", 10)

-

-	err := h.Build(BuildCfg{})

-

-	require.NoError(t, err)

-

 	// So what we have now is:

 	// 1. categories with terms content page, but no content page for the only c1 category

 	// 2. tags with no terms content page, but content page for one of 2 tags (tag1)

@@ -138,26 +134,26 @@

 	}

 	// 1.

-	th.assertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1")

-	th.assertFileContent(pathFunc("public/categories/index.html"), "Terms List", "Category Terms")

+	b.AssertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1")

+	b.AssertFileContent(pathFunc("public/categories/index.html"), "Taxonomy Term Page", "Category Terms")

 	// 2.

-	th.assertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2")

-	th.assertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1")

-	th.assertFileContent(pathFunc("public/tags/index.html"), "Terms List", "Tags")

+	b.AssertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2")

+	b.AssertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1")

+	b.AssertFileContent(pathFunc("public/tags/index.html"), "Taxonomy Term Page", "Tags")

 	// 3.

-	th.assertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1")

-	th.assertFileContent(pathFunc("public/others/index.html"), "Terms List", "Others")

+	b.AssertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1")

+	b.AssertFileContent(pathFunc("public/others/index.html"), "Taxonomy Term Page", "Others")

 	// 4.

-	th.assertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1")

+	b.AssertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1")

 	// This looks kind of funky, but the taxonomy terms do not have a permalinks definition,

 	// for good reasons.

-	th.assertFileContent(pathFunc("public/permalinkeds/index.html"), "Terms List", "Permalinkeds")

+	b.AssertFileContent(pathFunc("public/permalinkeds/index.html"), "Taxonomy Term Page", "Permalinkeds")

-	s := h.Sites[0]

+	s := b.H.Sites[0]

 	// Make sure that each page.KindTaxonomyTerm page has an appropriate number

 	// of page.KindTaxonomy pages in its Pages slice.

@@ -204,7 +200,7 @@

 	require.Equal(t, "Hello Hugo world", helloWorld.Title())

 	// Issue #2977

-	th.assertFileContent(pathFunc("public/empties/index.html"), "Terms List", "Empties")

+	b.AssertFileContent(pathFunc("public/empties/index.html"), "Taxonomy Term Page", "Empties")

 }

--- a/hugolib/template_engines_test.go

+++ b/hugolib/template_engines_test.go

@@ -24,7 +24,6 @@

 )

 func TestAllTemplateEngines(t *testing.T) {

-	t.Parallel()

 	noOp := func(s string) string {

 		return s

 	}

@@ -48,8 +47,10 @@

 		{"html", noOp},

 		{"ace", noOp},

 	} {

+		config := config

 		t.Run(config.suffix,

 			func(t *testing.T) {

+				t.Parallel()

 				doTestTemplateEngine(t, config.suffix, config.templateFixer)

 			})

 	}

--- a/hugolib/template_test.go

+++ b/hugolib/template_test.go

@@ -25,7 +25,6 @@

 )

 func TestTemplateLookupOrder(t *testing.T) {

-	t.Parallel()

 	var (

 		fs  *hugofs.Fs

 		cfg *viper.Viper

@@ -193,22 +192,26 @@

 		},

 	} {

-		cfg, fs = newTestCfg()

-		th = testHelper{cfg, fs, t}

+		this := this

+		t.Run(this.name, func(t *testing.T) {

+			// TODO(bep) there are some function vars need to pull down here to enable => t.Parallel()

+			cfg, fs = newTestCfg()

+			th = testHelper{cfg, fs, t}

-		for i := 1; i <= 3; i++ {

-			writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `---

+			for i := 1; i <= 3; i++ {

+				writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `---

 title: Template test

 ---

 Some content

 `)

-		}

+			}

-		this.setup(t)

+			this.setup(t)

-		buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})

-		t.Log(this.name)

-		this.assert(t)

+			buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})

+			//helpers.PrintFs(s.BaseFs.Layouts.Fs, "", os.Stdout)

+			this.assert(t)

+		})

 	}

 }

--- a/hugolib/testhelpers_test.go

+++ b/hugolib/testhelpers_test.go

@@ -2,7 +2,6 @@

 import (

 	"io"

-	"io/ioutil"

 	"path/filepath"

 	"runtime"

 	"strconv"

@@ -9,6 +8,11 @@

 	"testing"

 	"unicode/utf8"

+	"github.com/gohugoio/hugo/parser/metadecoders"

+

+	"github.com/gohugoio/hugo/parser"

+	"github.com/pkg/errors"

+

 	"bytes"

 	"fmt"

 	"regexp"

@@ -39,10 +43,13 @@

 )

 type sitesBuilder struct {

-	Cfg config.Provider

-	Fs  *hugofs.Fs

-	T   testing.TB

+	Cfg     config.Provider

+	environ []string

+	Fs      *hugofs.Fs

+	T       testing.TB

+	depsCfg deps.DepsCfg

+

 	*require.Assertions

 	logger *loggers.Logger

@@ -60,7 +67,9 @@

 	theme string

 	// Default toml

-	configFormat string

+	configFormat  string

+	configFileSet bool

+	viperSet      bool

 	// Default is empty.

 	// TODO(bep) revisit this and consider always setting it to something.

@@ -67,6 +76,7 @@

 	// Consider this in relation to using the BaseFs.PublishFs to all publishing.

 	workingDir string

+	addNothing bool

 	// Base data/content

 	contentFilePairs  []string

 	templateFilePairs []string

@@ -94,18 +104,22 @@

 	return &sitesBuilder{T: t, Assertions: require.New(t), Fs: fs, configFormat: "toml", dumper: litterOptions}

 }

-func createTempDir(prefix string) (string, func(), error) {

-	workDir, err := ioutil.TempDir("", prefix)

-	if err != nil {

-		return "", nil, err

-	}

+func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder {

+	assert := require.New(t)

-	if runtime.GOOS == "darwin" && !strings.HasPrefix(workDir, "/private") {

-		// To get the entry folder in line with the rest. This its a little bit

-		// mysterious, but so be it.

-		workDir = "/private" + workDir

+	litterOptions := litter.Options{

+		HidePrivateFields: true,

+		StripPackageNames: true,

+		Separator:         " ",

 	}

-	return workDir, func() { os.RemoveAll(workDir) }, nil

+

+	b := &sitesBuilder{T: t, Assertions: assert, depsCfg: d, Fs: d.Fs, dumper: litterOptions}

+	workingDir := d.Cfg.GetString("workingDir")

+

+	b.WithWorkingDir(workingDir)

+

+	return b.WithViper(d.Cfg.(*viper.Viper))

+

 }

 func (s *sitesBuilder) Running() *sitesBuilder {

@@ -113,6 +127,11 @@

 	return s

 }

+func (s *sitesBuilder) WithNothingAdded() *sitesBuilder {

+	s.addNothing = true

+	return s

+}

+

 func (s *sitesBuilder) WithLogger(logger *loggers.Logger) *sitesBuilder {

 	s.logger = logger

 	return s

@@ -119,11 +138,20 @@

 }

 func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder {

-	s.workingDir = dir

+	s.workingDir = filepath.FromSlash(dir)

 	return s

 }

+func (s *sitesBuilder) WithEnviron(env ...string) *sitesBuilder {

+	for i := 0; i < len(env); i += 2 {

+		s.environ = append(s.environ, fmt.Sprintf("%s=%s", env[i], env[i+1]))

+	}

+	return s

+}

+

 func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTemplate string) *sitesBuilder {

+	s.T.Helper()

+

 	if format == "" {

 		format = "toml"

 	}

@@ -138,32 +166,59 @@

 }

 func (s *sitesBuilder) WithViper(v *viper.Viper) *sitesBuilder {

-	loadDefaultSettingsFor(v)

-	s.Cfg = v

+	s.T.Helper()

+	if s.configFileSet {

+		s.T.Fatal("WithViper: use Viper or config.toml, not both")

+	}

+	defer func() {

+		s.viperSet = true

+	}()

-	return s

+	// Write to a config file to make sure the tests follow the same code path.

+	var buff bytes.Buffer

+	m := v.AllSettings()

+	s.Assertions.NoError(parser.InterfaceToConfig(m, metadecoders.TOML, &buff))

+	return s.WithConfigFile("toml", buff.String())

 }

 func (s *sitesBuilder) WithConfigFile(format, conf string) *sitesBuilder {

-	writeSource(s.T, s.Fs, "config."+format, conf)

+	s.T.Helper()

+	if s.viperSet {

+		s.T.Fatal("WithConfigFile: use Viper or config.toml, not both")

+	}

+	s.configFileSet = true

+	filename := s.absFilename("config." + format)

+	writeSource(s.T, s.Fs, filename, conf)

 	s.configFormat = format

 	return s

 }

 func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder {

+	s.T.Helper()

 	if s.theme == "" {

 		s.theme = "test-theme"

 	}

 	filename := filepath.Join("themes", s.theme, "config."+format)

-	writeSource(s.T, s.Fs, filename, conf)

+	writeSource(s.T, s.Fs, s.absFilename(filename), conf)

 	return s

 }

-func (s *sitesBuilder) WithSourceFile(filename, content string) *sitesBuilder {

-	writeSource(s.T, s.Fs, filepath.FromSlash(filename), content)

+func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder {

+	s.T.Helper()

+	for i := 0; i < len(filenameContent); i += 2 {

+		writeSource(s.T, s.Fs, s.absFilename(filenameContent[i]), filenameContent[i+1])

+	}

 	return s

 }

+func (s *sitesBuilder) absFilename(filename string) string {

+	filename = filepath.FromSlash(filename)

+	if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) {

+		filename = filepath.Join(s.workingDir, filename)

+	}

+	return filename

+}

+

 const commonConfigSections = `

 [services]

@@ -191,10 +246,12 @@

 `

 func (s *sitesBuilder) WithSimpleConfigFile() *sitesBuilder {

+	s.T.Helper()

 	return s.WithSimpleConfigFileAndBaseURL("http://example.com/")

 }

 func (s *sitesBuilder) WithSimpleConfigFileAndBaseURL(baseURL string) *sitesBuilder {

+	s.T.Helper()

 	config := fmt.Sprintf("baseURL = %q", baseURL)

 	config = config + commonConfigSections

@@ -323,7 +380,7 @@

 	for i := 0; i < len(filenameContent); i += 2 {

 		filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1]

 		changedFiles = append(changedFiles, filename)

-		writeSource(s.T, s.Fs, filename, content)

+		writeSource(s.T, s.Fs, s.absFilename(filename), content)

 	}

 	s.changedFiles = changedFiles

@@ -354,6 +411,7 @@

 func (s *sitesBuilder) CreateSites() *sitesBuilder {

 	if err := s.CreateSitesE(); err != nil {

+		herrors.PrintStackTrace(err)

 		s.Fatalf("Failed to create sites: %s", err)

 	}

@@ -361,34 +419,72 @@

 }

 func (s *sitesBuilder) LoadConfig() error {

-	cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat})

+	if !s.configFileSet {

+		s.WithSimpleConfigFile()

+	}

+

+	cfg, _, err := LoadConfig(ConfigSourceDescriptor{

+		WorkingDir: s.workingDir,

+		Fs:         s.Fs.Source,

+		Logger:     s.logger,

+		Environ:    s.environ,

+		Filename:   "config." + s.configFormat}, func(cfg config.Provider) error {

+

+		return nil

+	})

+

 	if err != nil {

 		return err

 	}

+

 	s.Cfg = cfg

+

 	return nil

 }

 func (s *sitesBuilder) CreateSitesE() error {

-	s.addDefaults()

-	s.writeFilePairs("content", s.contentFilePairs)

-	s.writeFilePairs("content", s.contentFilePairsAdded)

-	s.writeFilePairs("layouts", s.templateFilePairs)

-	s.writeFilePairs("layouts", s.templateFilePairsAdded)

-	s.writeFilePairs("data", s.dataFilePairs)

-	s.writeFilePairs("data", s.dataFilePairsAdded)

-	s.writeFilePairs("i18n", s.i18nFilePairs)

-	s.writeFilePairs("i18n", s.i18nFilePairsAdded)

-

-	if s.Cfg == nil {

-		if err := s.LoadConfig(); err != nil {

-			return err

+	if !s.addNothing {

+		if _, ok := s.Fs.Source.(*afero.OsFs); ok {

+			for _, dir := range []string{

+				"content/sect",

+				"layouts/_default",

+				"layouts/partials",

+				"layouts/shortcodes",

+				"data",

+				"i18n",

+			} {

+				if err := os.MkdirAll(filepath.Join(s.workingDir, dir), 0777); err != nil {

+					return errors.Wrapf(err, "failed to create %q", dir)

+				}

+			}

 		}

+

+		s.addDefaults()

+		s.writeFilePairs("content", s.contentFilePairsAdded)

+		s.writeFilePairs("layouts", s.templateFilePairsAdded)

+		s.writeFilePairs("data", s.dataFilePairsAdded)

+		s.writeFilePairs("i18n", s.i18nFilePairsAdded)

+

+		s.writeFilePairs("i18n", s.i18nFilePairs)

+		s.writeFilePairs("data", s.dataFilePairs)

+		s.writeFilePairs("content", s.contentFilePairs)

+		s.writeFilePairs("layouts", s.templateFilePairs)

+

 	}

-	sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Logger: s.logger, Running: s.running})

+	if err := s.LoadConfig(); err != nil {

+		return errors.Wrap(err, "failed to load config")

+	}

+

+	depsCfg := s.depsCfg

+	depsCfg.Fs = s.Fs

+	depsCfg.Cfg = s.Cfg

+	depsCfg.Logger = s.logger

+	depsCfg.Running = s.running

+

+	sites, err := NewHugoSites(depsCfg)

 	if err != nil {

-		return err

+		return errors.Wrap(err, "failed to create sites")

 	}

 	s.H = sites

@@ -404,10 +500,12 @@

 }

 func (s *sitesBuilder) Build(cfg BuildCfg) *sitesBuilder {

+	s.T.Helper()

 	return s.build(cfg, false)

 }

 func (s *sitesBuilder) BuildFail(cfg BuildCfg) *sitesBuilder {

+	s.T.Helper()

 	return s.build(cfg, true)

 }

@@ -528,24 +626,19 @@

 }

 func (s *sitesBuilder) Fatalf(format string, args ...interface{}) {

-	Fatalf(s.T, format, args...)

+	s.T.Helper()

+	s.T.Fatalf(format, args...)

 }

-func Fatalf(t testing.TB, format string, args ...interface{}) {

-	trace := stackTrace()

-	format = format + "\n%s"

-	args = append(args, trace)

-	t.Fatalf(format, args...)

-}

-

 func stackTrace() string {

 	return strings.Join(assert.CallerInfo(), "\n\r\t\t\t")

 }

 func (s *sitesBuilder) AssertFileContentFn(filename string, f func(s string) bool) {

+	s.T.Helper()

 	content := s.FileContent(filename)

 	if !f(content) {

-		s.Fatalf("Assert failed for %q", filename)

+		s.Fatalf("Assert failed for %q in content\n%s", filename, content)

 	}

 }

@@ -554,6 +647,7 @@

 }

 func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {

+	s.T.Helper()

 	content := s.FileContent(filename)

 	for _, match := range matches {

 		if !strings.Contains(content, match) {

@@ -563,10 +657,16 @@

 }

 func (s *sitesBuilder) FileContent(filename string) string {

+	s.T.Helper()

+	filename = filepath.FromSlash(filename)

+	if !strings.HasPrefix(filename, s.workingDir) {

+		filename = filepath.Join(s.workingDir, filename)

+	}

 	return readDestination(s.T, s.Fs, filename)

 }

 func (s *sitesBuilder) AssertObject(expected string, object interface{}) {

+	s.T.Helper()

 	got := s.dumper.Sdump(object)

 	expected = strings.TrimSpace(expected)

@@ -633,18 +733,42 @@

 	return value

 }

-func newTestCfg() (*viper.Viper, *hugofs.Fs) {

+func loadTestConfig(fs afero.Fs, withConfig ...func(cfg config.Provider) error) (*viper.Viper, error) {

+	v, _, err := LoadConfig(ConfigSourceDescriptor{Fs: fs}, withConfig...)

+	return v, err

+}

+func newTestCfgBasic() (*viper.Viper, *hugofs.Fs) {

+	mm := afero.NewMemMapFs()

 	v := viper.New()

-	fs := hugofs.NewMem(v)

+	v.Set("defaultContentLanguageInSubdir", true)

-	v.SetFs(fs.Source)

+	fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v)

-	loadDefaultSettingsFor(v)

+	return v, fs

-	// Default is false, but true is easier to use as default in tests

-	v.Set("defaultContentLanguageInSubdir", true)

+}

+func newTestCfg(withConfig ...func(cfg config.Provider) error) (*viper.Viper, *hugofs.Fs) {

+	mm := afero.NewMemMapFs()

+

+	v, err := loadTestConfig(mm, func(cfg config.Provider) error {

+		// Default is false, but true is easier to use as default in tests

+		cfg.Set("defaultContentLanguageInSubdir", true)

+

+		for _, w := range withConfig {

+			w(cfg)

+		}

+

+		return nil

+	})

+

+	if err != nil && err != ErrNoConfigFile {

+		panic(err)

+	}

+

+	fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v)

+

 	return v, fs

 }

@@ -651,9 +775,10 @@

 func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layoutPathContentPairs ...string) (testHelper, *HugoSites) {

 	if len(layoutPathContentPairs)%2 != 0 {

-		Fatalf(t, "Layouts must be provided in pairs")

+		t.Fatalf("Layouts must be provided in pairs")

 	}

+	writeToFs(t, afs, filepath.Join("content", ".gitkeep"), "")

 	writeToFs(t, afs, "config.toml", tomlConfig)

 	cfg, err := LoadConfigDefault(afs)

@@ -673,14 +798,6 @@

 	return th, h

 }

-func newTestSitesFromConfigWithDefaultTemplates(t testing.TB, tomlConfig string) (testHelper, *HugoSites) {

-	return newTestSitesFromConfig(t, afero.NewMemMapFs(), tomlConfig,

-		"layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}",

-		"layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}",

-		"layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",

-	)

-}

-

 func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ tpl.TemplateHandler) error {

 	return func(templ tpl.TemplateHandler) error {

@@ -694,13 +811,17 @@

 	}

 }

+// TODO(bep) replace these with the builder

 func buildSingleSite(t testing.TB, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site {

 	return buildSingleSiteExpected(t, false, false, depsCfg, buildCfg)

 }

 func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError bool, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site {

-	h, err := NewHugoSites(depsCfg)

+	t.Helper()

+	b := newTestSitesBuilderFromDepsCfg(t, depsCfg).WithNothingAdded()

+	err := b.CreateSitesE()

+

 	if expectSiteInitEror {

 		require.Error(t, err)

 		return nil

@@ -708,6 +829,8 @@

 		require.NoError(t, err)

 	}

+	h := b.H

+

 	require.Len(t, h.Sites, 1)

 	if expectBuildError {

@@ -751,9 +874,13 @@

 func dumpPages(pages ...page.Page) {

 	fmt.Println("---------")

 	for i, p := range pages {

-		fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n",

+		var meta interface{}

+		if p.File() != nil && p.File().FileInfo() != nil {

+			meta = p.File().FileInfo().Meta()

+		}

+		fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s Lang: %s Meta: %v\n",

 			i+1,

-			p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath())

+			p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath(), p.Lang(), meta)

 	}

 }

@@ -801,4 +928,11 @@

 	if parallelEnabled {

 		t.Parallel()

 	}

+}

+

+func skipSymlink(t *testing.T) {

+	if runtime.GOOS == "windows" && os.Getenv("CI") == "" {

+		t.Skip("skip symlink test on local Windows (needs admin)")

+	}

+

 }

--- /dev/null

+++ b/langs/config.go

@@ -1,0 +1,217 @@

+// Copyright 2018 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package langs

+

+import (

+	"fmt"

+	"path/filepath"

+	"sort"

+	"strings"

+

+	"github.com/gohugoio/hugo/common/maps"

+

+	"github.com/spf13/cast"

+

+	"github.com/pkg/errors"

+

+	"github.com/gohugoio/hugo/config"

+)

+

+type LanguagesConfig struct {

+	Languages                      Languages

+	Multihost                      bool

+	DefaultContentLanguageInSubdir bool

+}

+

+func LoadLanguageSettings(cfg config.Provider, oldLangs Languages) (c LanguagesConfig, err error) {

+

+	defaultLang := cfg.GetString("defaultContentLanguage")

+	if defaultLang == "" {

+		defaultLang = "en"

+		cfg.Set("defaultContentLanguage", defaultLang)

+	}

+

+	var languages map[string]interface{}

+

+	languagesFromConfig := cfg.GetStringMap("languages")

+	disableLanguages := cfg.GetStringSlice("disableLanguages")

+

+	if len(disableLanguages) == 0 {

+		languages = languagesFromConfig

+	} else {

+		languages = make(map[string]interface{})

+		for k, v := range languagesFromConfig {

+			for _, disabled := range disableLanguages {

+				if disabled == defaultLang {

+					return c, fmt.Errorf("cannot disable default language %q", defaultLang)

+				}

+

+				if strings.EqualFold(k, disabled) {

+					v.(map[string]interface{})["disabled"] = true

+					break

+				}

+			}

+			languages[k] = v

+		}

+	}

+

+	var languages2 Languages

+

+	if len(languages) == 0 {

+		languages2 = append(languages2, NewDefaultLanguage(cfg))

+	} else {

+		languages2, err = toSortedLanguages(cfg, languages)

+		if err != nil {

+			return c, errors.Wrap(err, "Failed to parse multilingual config")

+		}

+	}

+

+	if oldLangs != nil {

+		// When in multihost mode, the languages are mapped to a server, so

+		// some structural language changes will need a restart of the dev server.

+		// The validation below isn't complete, but should cover the most

+		// important cases.

+		var invalid bool

+		if languages2.IsMultihost() != oldLangs.IsMultihost() {

+			invalid = true

+		} else {

+			if languages2.IsMultihost() && len(languages2) != len(oldLangs) {

+				invalid = true

+			}

+		}

+

+		if invalid {

+			return c, errors.New("language change needing a server restart detected")

+		}

+

+		if languages2.IsMultihost() {

+			// We need to transfer any server baseURL to the new language

+			for i, ol := range oldLangs {

+				nl := languages2[i]

+				nl.Set("baseURL", ol.GetString("baseURL"))

+			}

+		}

+	}

+

+	// The defaultContentLanguage is something the user has to decide, but it needs

+	// to match a language in the language definition list.

+	langExists := false

+	for _, lang := range languages2 {

+		if lang.Lang == defaultLang {

+			langExists = true

+			break

+		}

+	}

+

+	if !langExists {

+		return c, fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang)

+	}

+

+	c.Languages = languages2

+	c.Multihost = languages2.IsMultihost()

+	c.DefaultContentLanguageInSubdir = c.Multihost

+

+	sortedDefaultFirst := make(Languages, len(c.Languages))

+	for i, v := range c.Languages {

+		sortedDefaultFirst[i] = v

+	}

+	sort.Slice(sortedDefaultFirst, func(i, j int) bool {

+		li, lj := sortedDefaultFirst[i], sortedDefaultFirst[j]

+		if li.Lang == defaultLang {

+			return true

+		}

+

+		if lj.Lang == defaultLang {

+			return false

+		}

+

+		return i < j

+	})

+

+	cfg.Set("languagesSorted", c.Languages)

+	cfg.Set("languagesSortedDefaultFirst", sortedDefaultFirst)

+	cfg.Set("multilingual", len(languages2) > 1)

+

+	multihost := c.Multihost

+

+	if multihost {

+		cfg.Set("defaultContentLanguageInSubdir", true)

+		cfg.Set("multihost", true)

+	}

+

+	if multihost {

+		// The baseURL may be provided at the language level. If that is true,

+		// then every language must have a baseURL. In this case we always render

+		// to a language sub folder, which is then stripped from all the Permalink URLs etc.

+		for _, l := range languages2 {

+			burl := l.GetLocal("baseURL")

+			if burl == nil {

+				return c, errors.New("baseURL must be set on all or none of the languages")

+			}

+		}

+

+	}

+

+	return c, nil

+}

+

+func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (Languages, error) {

+	languages := make(Languages, len(l))

+	i := 0

+

+	for lang, langConf := range l {

+		langsMap, err := cast.ToStringMapE(langConf)

+

+		if err != nil {

+			return nil, fmt.Errorf("Language config is not a map: %T", langConf)

+		}

+

+		language := NewLanguage(lang, cfg)

+

+		for loki, v := range langsMap {

+			switch loki {

+			case "title":

+				language.Title = cast.ToString(v)

+			case "languagename":

+				language.LanguageName = cast.ToString(v)

+			case "weight":

+				language.Weight = cast.ToInt(v)

+			case "contentdir":

+				language.ContentDir = filepath.Clean(cast.ToString(v))

+			case "disabled":

+				language.Disabled = cast.ToBool(v)

+			case "params":

+				m := cast.ToStringMap(v)

+				// Needed for case insensitive fetching of params values

+				maps.ToLower(m)

+				for k, vv := range m {

+					language.SetParam(k, vv)

+				}

+			}

+

+			// Put all into the Params map

+			language.SetParam(loki, v)

+

+			// Also set it in the configuration map (for baseURL etc.)

+			language.Set(loki, v)

+		}

+

+		languages[i] = language

+		i++

+	}

+

+	sort.Sort(languages)

+

+	return languages, nil

+}

--- a/langs/i18n/i18n_test.go

+++ b/langs/i18n/i18n_test.go

@@ -17,11 +17,13 @@

 	"path/filepath"

 	"testing"

+	"github.com/gohugoio/hugo/modules"

+

 	"github.com/gohugoio/hugo/tpl/tplimpl"

 	"github.com/gohugoio/hugo/common/loggers"

-	"github.com/gohugoio/hugo/htesting"

 	"github.com/gohugoio/hugo/langs"

+	"github.com/gohugoio/hugo/resources/page"

 	"github.com/spf13/afero"

 	"github.com/spf13/viper"

@@ -199,7 +201,7 @@

 	l.Set("i18nDir", "i18n")

 	return deps.DepsCfg{

 		Language:            l,

-		Site:                htesting.NewTestHugoSite(),

+		Site:                page.NewDummyHugoSite(cfg),

 		Cfg:                 cfg,

 		Fs:                  fs,

 		Logger:              logger,

@@ -219,6 +221,13 @@

 	v.Set("assetDir", "assets")

 	v.Set("resourceDir", "resources")

 	v.Set("publishDir", "public")

+	langs.LoadLanguageSettings(v, nil)

+	mod, err := modules.CreateProjectModule(v)

+	if err != nil {

+		panic(err)

+	}

+	v.Set("allModules", modules.Modules{mod})

+

 	return v

 }

--- a/langs/i18n/translationProvider.go

+++ b/langs/i18n/translationProvider.go

@@ -40,8 +40,7 @@

 // Update updates the i18n func in the provided Deps.

 func (tp *TranslationProvider) Update(d *deps.Deps) error {

-	sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs)

-	src := sp.NewFilesystem("")

+	spec := source.NewSourceSpec(d.PathSpec, nil)

 	i18nBundle := bundle.New()

@@ -51,26 +50,34 @@

 	}

 	var newLangs []string

-	for _, r := range src.Files() {

-		currentSpec := language.GetPluralSpec(r.BaseFileName())

-		if currentSpec == nil {

-			// This may is a language code not supported by go-i18n, it may be

-			// Klingon or ... not even a fake language. Make sure it works.

-			newLangs = append(newLangs, r.BaseFileName())

+	for _, dir := range d.BaseFs.I18n.Dirs {

+		src := spec.NewFilesystemFromFileMetaInfo(dir)

+

+		files, err := src.Files()

+		if err != nil {

+			return err

 		}

-	}

-	if len(newLangs) > 0 {

-		language.RegisterPluralSpec(newLangs, en)

-	}

+		for _, r := range files {

+			currentSpec := language.GetPluralSpec(r.BaseFileName())

+			if currentSpec == nil {

+				// This may is a language code not supported by go-i18n, it may be

+				// Klingon or ... not even a fake language. Make sure it works.

+				newLangs = append(newLangs, r.BaseFileName())

+			}

+		}

-	// The source files are ordered so the most important comes first. Since this is a

-	// last key win situation, we have to reverse the iteration order.

-	files := src.Files()

-	for i := len(files) - 1; i >= 0; i-- {

-		if err := addTranslationFile(i18nBundle, files[i]); err != nil {

-			return err

+		if len(newLangs) > 0 {

+			language.RegisterPluralSpec(newLangs, en)

 		}

+

+		// The source files are ordered so the most important comes first. Since this is a

+		// last key win situation, we have to reverse the iteration order.

+		for i := len(files) - 1; i >= 0; i-- {

+			if err := addTranslationFile(i18nBundle, files[i]); err != nil {

+				return err

+			}

+		}

 	}

 	tp.t = NewTranslator(i18nBundle, d.Cfg, d.Log)

@@ -81,8 +88,8 @@

 }

-func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error {

-	f, err := r.Open()

+func addTranslationFile(bundle *bundle.Bundle, r source.File) error {

+	f, err := r.FileInfo().Meta().Open()

 	if err != nil {

 		return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName())

 	}

@@ -101,14 +108,15 @@

 	return nil

 }

-func errWithFileContext(inerr error, r source.ReadableFile) error {

-	rfi, ok := r.FileInfo().(hugofs.RealFilenameInfo)

+func errWithFileContext(inerr error, r source.File) error {

+	fim, ok := r.FileInfo().(hugofs.FileMetaInfo)

 	if !ok {

 		return inerr

 	}

-	realFilename := rfi.RealFilename()

-	f, err := r.Open()

+	meta := fim.Meta()

+	realFilename := meta.Filename()

+	f, err := meta.Open()

 	if err != nil {

 		return inerr

 	}

--- a/langs/language.go

+++ b/langs/language.go

@@ -78,12 +78,7 @@

 	}

 	maps.ToLower(params)

-	defaultContentDir := cfg.GetString("contentDir")

-	if defaultContentDir == "" {

-		panic("contentDir not set")

-	}

-

-	l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})}

+	l := &Language{Lang: lang, ContentDir: cfg.GetString("contentDir"), Cfg: cfg, params: params, settings: make(map[string]interface{})}

 	return l

 }

@@ -130,6 +125,24 @@

 // Params retunrs language-specific params merged with the global params.

 func (l *Language) Params() map[string]interface{} {

 	return l.params

+}

+

+func (l Languages) AsSet() map[string]bool {

+	m := make(map[string]bool)

+	for _, lang := range l {

+		m[lang.Lang] = true

+	}

+

+	return m

+}

+

+func (l Languages) AsOrdinalSet() map[string]int {

+	m := make(map[string]int)

+	for i, lang := range l {

+		m[lang.Lang] = i

+	}

+

+	return m

 }

 // IsMultihost returns whether there are more than one language and at least one of

--- a/magefile.go

+++ b/magefile.go

@@ -143,20 +143,31 @@

 	mg.Deps(TestRace)

 }

+func testGoFlags() string {

+	if isCI() {

+		return ""

+	}

+

+	return "-test.short"

+}

+

 // Run tests in 32-bit mode

 // Note that we don't run with the extended tag. Currently not supported in 32 bit.

 func Test386() error {

-	return sh.RunWith(map[string]string{"GOARCH": "386"}, goexe, "test", "./...")

+	env := map[string]string{"GOARCH": "386", "GOFLAGS": testGoFlags()}

+	return sh.RunWith(env, goexe, "test", "./...")

 }

 // Run tests

 func Test() error {

-	return sh.Run(goexe, "test", "./...", "-tags", buildTags())

+	env := map[string]string{"GOFLAGS": testGoFlags()}

+	return sh.RunWith(env, goexe, "test", "./...", "-tags", buildTags())

 }

 // Run tests with race detector

 func TestRace() error {

-	return sh.Run(goexe, "test", "-race", "./...", "-tags", buildTags())

+	env := map[string]string{"GOFLAGS": testGoFlags()}

+	return sh.RunWith(env, goexe, "test", "-race", "./...", "-tags", buildTags())

 }

 // Run gofmt linter

@@ -294,6 +305,10 @@

 func isGoLatest() bool {

 	return strings.Contains(runtime.Version(), "1.11")

+}

+

+func isCI() bool {

+	return os.Getenv("CI") != ""

 }

 func buildTags() string {

--- /dev/null

+++ b/modules/client.go

@@ -1,0 +1,570 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package modules

+

+import (

+	"bufio"

+	"bytes"

+	"context"

+	"encoding/json"

+	"fmt"

+	"io"

+	"io/ioutil"

+	"os"

+	"os/exec"

+	"path/filepath"

+

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/gohugoio/hugo/common/loggers"

+

+	"strings"

+	"time"

+

+	"github.com/gohugoio/hugo/config"

+

+	"github.com/rogpeppe/go-internal/module"

+

+	"github.com/gohugoio/hugo/common/hugio"

+

+	"github.com/pkg/errors"

+	"github.com/spf13/afero"

+)

+

+var (

+	fileSeparator = string(os.PathSeparator)

+)

+

+const (

+	goBinaryStatusOK goBinaryStatus = iota

+	goBinaryStatusNotFound

+	goBinaryStatusTooOld

+)

+

+// The "vendor" dir is reserved for Go Modules.

+const vendord = "_vendor"

+

+const (

+	goModFilename = "go.mod"

+	goSumFilename = "go.sum"

+)

+

+// NewClient creates a new Client that can be used to manage the Hugo Components

+// in a given workingDir.

+// The Client will resolve the dependencies recursively, but needs the top

+// level imports to start out.

+func NewClient(cfg ClientConfig) *Client {

+	fs := cfg.Fs

+

+	n := filepath.Join(cfg.WorkingDir, goModFilename)

+	goModEnabled, _ := afero.Exists(fs, n)

+	var goModFilename string

+	if goModEnabled {

+		goModFilename = n

+	}

+

+	env := os.Environ()

+	mcfg := cfg.ModuleConfig

+

+	config.SetEnvVars(&env,

+		"PWD", cfg.WorkingDir,

+		"GOPROXY", mcfg.Proxy,

+		"GOPRIVATE", mcfg.Private,

+		"GONOPROXY", mcfg.NoProxy)

+

+	if cfg.CacheDir != "" {

+		// Module cache stored below $GOPATH/pkg

+		config.SetEnvVars(&env, "GOPATH", cfg.CacheDir)

+

+	}

+

+	logger := cfg.Logger

+	if logger == nil {

+		logger = loggers.NewWarningLogger()

+	}

+

+	return &Client{

+		fs:                fs,

+		ignoreVendor:      cfg.IgnoreVendor,

+		workingDir:        cfg.WorkingDir,

+		themesDir:         cfg.ThemesDir,

+		logger:            logger,

+		moduleConfig:      mcfg,

+		environ:           env,

+		GoModulesFilename: goModFilename}

+}

+

+// Client contains most of the API provided by this package.

+type Client struct {

+	fs     afero.Fs

+	logger *loggers.Logger

+

+	// Ignore any _vendor directory.

+	ignoreVendor bool

+

+	// Absolute path to the project dir.

+	workingDir string

+

+	// Absolute path to the project's themes dir.

+	themesDir string

+

+	// The top level module config

+	moduleConfig Config

+

+	// Environment variables used in "go get" etc.

+	environ []string

+

+	// Set when Go modules are initialized in the current repo, that is:

+	// a go.mod file exists.

+	GoModulesFilename string

+

+	// Set if we get a exec.ErrNotFound when running Go, which is most likely

+	// due to being run on a system without Go installed. We record it here

+	// so we can give an instructional error at the end if module/theme

+	// resolution fails.

+	goBinaryStatus goBinaryStatus

+}

+

+// Graph writes a module dependenchy graph to the given writer.

+func (c *Client) Graph(w io.Writer) error {

+	mc, coll := c.collect(true)

+	if coll.err != nil {

+		return coll.err

+	}

+	for _, module := range mc.AllModules {

+		if module.Owner() == nil {

+			continue

+		}

+

+		prefix := ""

+		if module.Disabled() {

+			prefix = "DISABLED "

+		}

+		dep := pathVersion(module.Owner()) + " " + pathVersion(module)

+		if replace := module.Replace(); replace != nil {

+			if replace.Version() != "" {

+				dep += " => " + pathVersion(replace)

+			} else {

+				// Local dir.

+				dep += " => " + replace.Dir()

+			}

+

+		}

+		fmt.Fprintln(w, prefix+dep)

+	}

+

+	return nil

+}

+

+// Tidy can be used to remove unused dependencies from go.mod and go.sum.

+func (c *Client) Tidy() error {

+	tc, coll := c.collect(false)

+	if coll.err != nil {

+		return coll.err

+	}

+

+	if coll.skipTidy {

+		return nil

+	}

+

+	return c.tidy(tc.AllModules, false)

+}

+

+// Vendor writes all the module dependencies to a _vendor folder.

+//

+// Unlike Go, we support it for any level.

+//

+// We, by default, use the /_vendor folder first, if found. To disable,

+// run with

+//    hugo --ignoreVendor

+//

+// Given a module tree, Hugo will pick the first module for a given path,

+// meaning that if the top-level module is vendored, that will be the full

+// set of dependencies.

+func (c *Client) Vendor() error {

+	vendorDir := filepath.Join(c.workingDir, vendord)

+	if err := c.rmVendorDir(vendorDir); err != nil {

+		return err

+	}

+

+	// Write the modules list to modules.txt.

+	//

+	// On the form:

+	//

+	// # github.com/alecthomas/chroma v0.6.3

+	//

+	// This is how "go mod vendor" does it. Go also lists

+	// the packages below it, but that is currently not applicable to us.

+	//

+	var modulesContent bytes.Buffer

+

+	tc, coll := c.collect(true)

+	if coll.err != nil {

+		return coll.err

+	}

+

+	for _, t := range tc.AllModules {

+		if t.Owner() == nil {

+			// This is the project.

+			continue

+		}

+		// We respect the --ignoreVendor flag even for the vendor command.

+		if !t.IsGoMod() && !t.Vendor() {

+			// We currently do not vendor components living in the

+			// theme directory, see https://github.com/gohugoio/hugo/issues/5993

+			continue

+		}

+

+		fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version())

+

+		dir := t.Dir()

+

+		for _, mount := range t.Mounts() {

+			if err := hugio.CopyDir(c.fs, filepath.Join(dir, mount.Source), filepath.Join(vendorDir, t.Path(), mount.Source), nil); err != nil {

+				return errors.Wrap(err, "failed to copy module to vendor dir")

+			}

+		}

+

+		// Include the resource cache if present.

+		resourcesDir := filepath.Join(dir, files.FolderResources)

+		_, err := c.fs.Stat(resourcesDir)

+		if err == nil {

+			if err := hugio.CopyDir(c.fs, resourcesDir, filepath.Join(vendorDir, t.Path(), files.FolderResources), nil); err != nil {

+				return errors.Wrap(err, "failed to copy resources to vendor dir")

+			}

+		}

+

+		// Also include any theme.toml or config.* files in the root.

+		configFiles, _ := afero.Glob(c.fs, filepath.Join(dir, "config.*"))

+		configFiles = append(configFiles, filepath.Join(dir, "theme.toml"))

+		for _, configFile := range configFiles {

+			if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil {

+				if !os.IsNotExist(err) {

+					return err

+				}

+			}

+		}

+	}

+

+	if modulesContent.Len() > 0 {

+		if err := afero.WriteFile(c.fs, filepath.Join(vendorDir, vendorModulesFilename), modulesContent.Bytes(), 0666); err != nil {

+			return err

+		}

+	}

+

+	return nil

+}

+

+// Get runs "go get" with the supplied arguments.

+func (c *Client) Get(args ...string) error {

+	if err := c.runGo(context.Background(), os.Stdout, append([]string{"get"}, args...)...); err != nil {

+		errors.Wrapf(err, "failed to get %q", args)

+	}

+	return nil

+}

+

+// Init initializes this as a Go Module with the given path.

+// If path is empty, Go will try to guess.

+// If this succeeds, this project will be marked as Go Module.

+func (c *Client) Init(path string) error {

+	err := c.runGo(context.Background(), os.Stdout, "mod", "init", path)

+	if err != nil {

+		return errors.Wrap(err, "failed to init modules")

+	}

+

+	c.GoModulesFilename = filepath.Join(c.workingDir, goModFilename)

+

+	return nil

+}

+

+func (c *Client) isProbablyModule(path string) bool {

+	return module.CheckPath(path) == nil

+}

+

+func (c *Client) listGoMods() (goModules, error) {

+	if c.GoModulesFilename == "" {

+		return nil, nil

+	}

+

+	out := ioutil.Discard

+	err := c.runGo(context.Background(), out, "mod", "download")

+	if err != nil {

+		return nil, errors.Wrap(err, "failed to download modules")

+	}

+

+	b := &bytes.Buffer{}

+	err = c.runGo(context.Background(), b, "list", "-m", "-json", "all")

+	if err != nil {

+		return nil, errors.Wrap(err, "failed to list modules")

+	}

+

+	var modules goModules

+

+	dec := json.NewDecoder(b)

+	for {

+		m := &goModule{}

+		if err := dec.Decode(m); err != nil {

+			if err == io.EOF {

+				break

+			}

+			return nil, errors.Wrap(err, "failed to decode modules list")

+		}

+

+		modules = append(modules, m)

+	}

+

+	return modules, err

+

+}

+

+func (c *Client) rewriteGoMod(name string, isGoMod map[string]bool) error {

+	data, err := c.rewriteGoModRewrite(name, isGoMod)

+	if err != nil {

+		return err

+	}

+	if data != nil {

+		if err := afero.WriteFile(c.fs, filepath.Join(c.workingDir, name), data, 0666); err != nil {

+			return err

+		}

+	}

+

+	return nil

+}

+

+func (c *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]byte, error) {

+	if name == goModFilename && c.GoModulesFilename == "" {

+		// Already checked.

+		return nil, nil

+	}

+

+	modlineSplitter := getModlineSplitter(name == goModFilename)

+

+	b := &bytes.Buffer{}

+	f, err := c.fs.Open(filepath.Join(c.workingDir, name))

+	if err != nil {

+		if os.IsNotExist(err) {

+			// It's been deleted.

+			return nil, nil

+		}

+		return nil, err

+	}

+	defer f.Close()

+

+	scanner := bufio.NewScanner(f)

+	var dirty bool

+

+	for scanner.Scan() {

+		line := scanner.Text()

+		var doWrite bool

+

+		if parts := modlineSplitter(line); parts != nil {

+			modname, modver := parts[0], parts[1]

+			modver = strings.TrimSuffix(modver, "/"+goModFilename)

+			modnameVer := modname + " " + modver

+			doWrite = isGoMod[modnameVer]

+		} else {

+			doWrite = true

+		}

+

+		if doWrite {

+			fmt.Fprintln(b, line)

+		} else {

+			dirty = true

+		}

+	}

+

+	if !dirty {

+		// Nothing changed

+		return nil, nil

+	}

+

+	return b.Bytes(), nil

+

+}

+

+func (c *Client) rmVendorDir(vendorDir string) error {

+	modulestxt := filepath.Join(vendorDir, vendorModulesFilename)

+

+	if _, err := c.fs.Stat(vendorDir); err != nil {

+		return nil

+	}

+

+	_, err := c.fs.Stat(modulestxt)

+	if err != nil {

+		// If we have a _vendor dir without modules.txt it sounds like

+		// a _vendor dir created by others.

+		return errors.New("found _vendor dir without modules.txt, skip delete")

+	}

+

+	return c.fs.RemoveAll(vendorDir)

+}

+

+func (c *Client) runGo(

+	ctx context.Context,

+	stdout io.Writer,

+	args ...string) error {

+

+	if c.goBinaryStatus != 0 {

+		return nil

+	}

+

+	stderr := new(bytes.Buffer)

+	cmd := exec.CommandContext(ctx, "go", args...)

+

+	cmd.Env = c.environ

+	cmd.Dir = c.workingDir

+	cmd.Stdout = stdout

+	cmd.Stderr = io.MultiWriter(stderr, os.Stderr)

+

+	if err := cmd.Run(); err != nil {

+		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {

+			c.goBinaryStatus = goBinaryStatusNotFound

+			return nil

+		}

+

+		_, ok := err.(*exec.ExitError)

+		if !ok {

+			return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err)

+		}

+

+		// Too old Go version

+		if strings.Contains(stderr.String(), "flag provided but not defined") {

+			c.goBinaryStatus = goBinaryStatusTooOld

+			return nil

+		}

+

+		return errors.Errorf("go command failed: %s", stderr)

+

+	}

+

+	return nil

+}

+

+func (c *Client) tidy(mods Modules, goModOnly bool) error {

+	isGoMod := make(map[string]bool)

+	for _, m := range mods {

+		if m.Owner() == nil {

+			continue

+		}

+		if m.IsGoMod() {

+			// Matching the format in go.mod

+			pathVer := m.Path() + " " + m.Version()

+			isGoMod[pathVer] = true

+		}

+	}

+

+	if err := c.rewriteGoMod(goModFilename, isGoMod); err != nil {

+		return err

+	}

+

+	if goModOnly {

+		return nil

+	}

+

+	if err := c.rewriteGoMod(goSumFilename, isGoMod); err != nil {

+		return err

+	}

+

+	return nil

+}

+

+// ClientConfig configures the module Client.

+type ClientConfig struct {

+	Fs           afero.Fs

+	Logger       *loggers.Logger

+	IgnoreVendor bool

+	WorkingDir   string

+	ThemesDir    string // Absolute directory path

+	CacheDir     string // Module cache

+	ModuleConfig Config

+}

+

+type goBinaryStatus int

+

+type goModule struct {

+	Path     string         // module path

+	Version  string         // module version

+	Versions []string       // available module versions (with -versions)

+	Replace  *goModule      // replaced by this module

+	Time     *time.Time     // time version was created

+	Update   *goModule      // available update, if any (with -u)

+	Main     bool           // is this the main module?

+	Indirect bool           // is this module only an indirect dependency of main module?

+	Dir      string         // directory holding files for this module, if any

+	GoMod    string         // path to go.mod file for this module, if any

+	Error    *goModuleError // error loading module

+}

+

+type goModuleError struct {

+	Err string // the error itself

+}

+

+type goModules []*goModule

+

+func (modules goModules) GetByPath(p string) *goModule {

+	if modules == nil {

+		return nil

+	}

+

+	for _, m := range modules {

+		if strings.EqualFold(p, m.Path) {

+			return m

+		}

+	}

+

+	return nil

+}

+

+func (modules goModules) GetMain() *goModule {

+	for _, m := range modules {

+		if m.Main {

+			return m

+		}

+	}

+

+	return nil

+}

+

+func getModlineSplitter(isGoMod bool) func(line string) []string {

+	if isGoMod {

+		return func(line string) []string {

+			if strings.HasPrefix(line, "require (") {

+				return nil

+			}

+			if !strings.HasPrefix(line, "require") && !strings.HasPrefix(line, "\t") {

+				return nil

+			}

+			line = strings.TrimPrefix(line, "require")

+			line = strings.TrimSpace(line)

+			line = strings.TrimSuffix(line, "// indirect")

+

+			return strings.Fields(line)

+		}

+	}

+

+	return func(line string) []string {

+		return strings.Fields(line)

+	}

+}

+

+func pathVersion(m Module) string {

+	versionStr := m.Version()

+	if m.Vendor() {

+		versionStr += "+vendor"

+	}

+	if versionStr == "" {

+		return m.Path()

+	}

+	return fmt.Sprintf("%s@%s", m.Path(), versionStr)

+}

--- /dev/null

+++ b/modules/client_test.go

@@ -1,0 +1,117 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package modules

+

+import (

+	"bytes"

+	"testing"

+

+	"github.com/gohugoio/hugo/common/hugo"

+

+	"github.com/gohugoio/hugo/htesting"

+

+	"github.com/gohugoio/hugo/hugofs"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestClient(t *testing.T) {

+	if hugo.GoMinorVersion() < 12 {

+		// https://github.com/golang/go/issues/26794

+		// There were some concurrent issues with Go modules in < Go 12.

+		t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib")

+	}

+

+	t.Parallel()

+

+	modName := "hugo-modules-basic-test"

+	modPath := "github.com/gohugoio/tests/" + modName

+	modConfig := DefaultModuleConfig

+	modConfig.Imports = []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}}

+

+	assert := require.New(t)

+

+	workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName)

+	assert.NoError(err)

+	defer clean()

+

+	client := NewClient(ClientConfig{

+		Fs:           hugofs.Os,

+		WorkingDir:   workingDir,

+		ModuleConfig: modConfig,

+	})

+

+	// Test Init

+	assert.NoError(client.Init(modPath))

+

+	// Test Collect

+	mc, err := client.Collect()

+	assert.NoError(err)

+	assert.Equal(4, len(mc.AllModules))

+	for _, m := range mc.AllModules {

+		assert.NotNil(m)

+	}

+

+	// Test Graph

+	var graphb bytes.Buffer

+	assert.NoError(client.Graph(&graphb))

+

+	expect := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/[email protected]

+github.com/gohugoio/hugoTestModules1_darwin/[email protected] github.com/gohugoio/hugoTestModules1_darwin/[email protected]

+github.com/gohugoio/hugoTestModules1_darwin/[email protected] github.com/gohugoio/hugoTestModules1_darwin/[email protected]

+`

+

+	assert.Equal(expect, graphb.String())

+

+	// Test Vendor

+	assert.NoError(client.Vendor())

+	graphb.Reset()

+	assert.NoError(client.Graph(&graphb))

+	expectVendored := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/[email protected]+vendor

+github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/[email protected]+vendor

+github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/[email protected]+vendor

+`

+	assert.Equal(expectVendored, graphb.String())

+

+	// Test the ignoreVendor setting

+	clientIgnoreVendor := NewClient(ClientConfig{

+		Fs:           hugofs.Os,

+		WorkingDir:   workingDir,

+		ModuleConfig: modConfig,

+		IgnoreVendor: true,

+	})

+

+	graphb.Reset()

+	assert.NoError(clientIgnoreVendor.Graph(&graphb))

+	assert.Equal(expect, graphb.String())

+

+	// Test Tidy

+	assert.NoError(client.Tidy())

+

+}

+

+func TestGetModlineSplitter(t *testing.T) {

+

+	assert := require.New(t)

+

+	gomodSplitter := getModlineSplitter(true)

+

+	assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gomodSplitter("\tgithub.com/BurntSushi/toml v0.3.1"))

+	assert.Equal([]string{"github.com/cpuguy83/go-md2man", "v1.0.8"}, gomodSplitter("\tgithub.com/cpuguy83/go-md2man v1.0.8 // indirect"))

+	assert.Nil(gomodSplitter("require ("))

+

+	gosumSplitter := getModlineSplitter(false)

+	assert.Equal([]string{"github.com/BurntSushi/toml", "v0.3.1"}, gosumSplitter("github.com/BurntSushi/toml v0.3.1"))

+

+}

--- /dev/null

+++ b/modules/collect.go

@@ -1,0 +1,574 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package modules

+

+import (

+	"bufio"

+	"fmt"

+	"os"

+	"path/filepath"

+	"strings"

+

+	"github.com/spf13/cast"

+

+	"github.com/gohugoio/hugo/common/maps"

+

+	"github.com/gohugoio/hugo/common/hugo"

+	"github.com/gohugoio/hugo/parser/metadecoders"

+

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/rogpeppe/go-internal/module"

+

+	"github.com/pkg/errors"

+

+	"github.com/gohugoio/hugo/config"

+	"github.com/spf13/afero"

+)

+

+var ErrNotExist = errors.New("module does not exist")

+

+const vendorModulesFilename = "modules.txt"

+

+// IsNotExist returns whether an error means that a module could not be found.

+func IsNotExist(err error) bool {

+	return errors.Cause(err) == ErrNotExist

+}

+

+// CreateProjectModule creates modules from the given config.

+// This is used in tests only.

+func CreateProjectModule(cfg config.Provider) (Module, error) {

+	workingDir := cfg.GetString("workingDir")

+	var modConfig Config

+

+	mod := createProjectModule(nil, workingDir, modConfig)

+	if err := ApplyProjectConfigDefaults(cfg, mod); err != nil {

+		return nil, err

+	}

+

+	return mod, nil

+}

+

+func (h *Client) Collect() (ModulesConfig, error) {

+	mc, coll := h.collect(true)

+	return mc, coll.err

+

+}

+

+func (h *Client) collect(tidy bool) (ModulesConfig, *collector) {

+	c := &collector{

+		Client: h,

+	}

+

+	c.collect()

+	if c.err != nil {

+		return ModulesConfig{}, c

+	}

+

+	if !c.skipTidy && tidy {

+		if err := h.tidy(c.modules, true); err != nil {

+			c.err = err

+			return ModulesConfig{}, c

+		}

+	}

+

+	// TODO(bep) consider --ignoreVendor vs removing from go.mod

+	var activeMods Modules

+	for _, mod := range c.modules {

+		if !mod.Config().HugoVersion.IsValid() {

+			h.logger.WARN.Printf(`Module %q is not compatible with this Hugo version; run "hugo mod graph" for more information.`, mod.Path())

+		}

+		if !mod.Disabled() {

+			activeMods = append(activeMods, mod)

+		}

+	}

+

+	return ModulesConfig{

+		AllModules:        c.modules,

+		ActiveModules:     activeMods,

+		GoModulesFilename: c.GoModulesFilename,

+	}, c

+

+}

+

+type ModulesConfig struct {

+	// All modules, including any disabled.

+	AllModules Modules

+

+	// All active modules.

+	ActiveModules Modules

+

+	// Set if this is a Go modules enabled project.

+	GoModulesFilename string

+}

+

+type collected struct {

+	// Pick the first and prevent circular loops.

+	seen map[string]bool

+

+	// Maps module path to a _vendor dir. These values are fetched from

+	// _vendor/modules.txt, and the first (top-most) will win.

+	vendored map[string]vendoredModule

+

+	// Set if a Go modules enabled project.

+	gomods goModules

+

+	// Ordered list of collected modules, including Go Modules and theme

+	// components stored below /themes.

+	modules Modules

+}

+

+// Collects and creates a module tree.

+type collector struct {

+	*Client

+

+	// Store away any non-fatal error and return at the end.

+	err error

+

+	// Set to disable any Tidy operation in the end.

+	skipTidy bool

+

+	*collected

+}

+

+func (c *collector) initModules() error {

+	c.collected = &collected{

+		seen:     make(map[string]bool),

+		vendored: make(map[string]vendoredModule),

+	}

+

+	// We may fail later if we don't find the mods.

+	return c.loadModules()

+}

+

+func (c *collector) isSeen(path string) bool {

+	key := pathKey(path)

+	if c.seen[key] {

+		return true

+	}

+	c.seen[key] = true

+	return false

+}

+

+func (c *collector) getVendoredDir(path string) (vendoredModule, bool) {

+	v, found := c.vendored[path]

+	return v, found

+}

+

+func (c *collector) add(owner *moduleAdapter, moduleImport Import, disabled bool) (*moduleAdapter, error) {

+	var (

+		mod       *goModule

+		moduleDir string

+		version   string

+		vendored  bool

+	)

+

+	modulePath := moduleImport.Path

+	var realOwner Module = owner

+

+	if !c.ignoreVendor {

+		if err := c.collectModulesTXT(owner); err != nil {

+			return nil, err

+		}

+

+		// Try _vendor first.

+		var vm vendoredModule

+		vm, vendored = c.getVendoredDir(modulePath)

+		if vendored {

+			moduleDir = vm.Dir

+			realOwner = vm.Owner

+			version = vm.Version

+

+			if owner.projectMod {

+				// We want to keep the go.mod intact with the versions and all.

+				c.skipTidy = true

+			}

+

+		}

+	}

+

+	if moduleDir == "" {

+		mod = c.gomods.GetByPath(modulePath)

+		if mod != nil {

+			moduleDir = mod.Dir

+		}

+

+		if moduleDir == "" {

+

+			if c.GoModulesFilename != "" && c.isProbablyModule(modulePath) {

+				// Try to "go get" it and reload the module configuration.

+				if err := c.Get(modulePath); err != nil {

+					return nil, err

+				}

+				if err := c.loadModules(); err != nil {

+					return nil, err

+				}

+

+				mod = c.gomods.GetByPath(modulePath)

+				if mod != nil {

+					moduleDir = mod.Dir

+				}

+			}

+

+			// Fall back to /themes/<mymodule>

+			if moduleDir == "" {

+				moduleDir = filepath.Join(c.themesDir, modulePath)

+

+				if found, _ := afero.Exists(c.fs, moduleDir); !found {

+					c.err = c.wrapModuleNotFound(errors.Errorf(`module %q not found; either add it as a Hugo Module or store it in %q.`, modulePath, c.themesDir))

+					return nil, nil

+				}

+			}

+		}

+	}

+

+	if found, _ := afero.Exists(c.fs, moduleDir); !found {

+		c.err = c.wrapModuleNotFound(errors.Errorf("%q not found", moduleDir))

+		return nil, nil

+	}

+

+	if !strings.HasSuffix(moduleDir, fileSeparator) {

+		moduleDir += fileSeparator

+	}

+

+	ma := &moduleAdapter{

+		dir:      moduleDir,

+		vendor:   vendored,

+		disabled: disabled,

+		gomod:    mod,

+		version:  version,

+		// This may be the owner of the _vendor dir

+		owner: realOwner,

+	}

+

+	if mod == nil {

+		ma.path = modulePath

+	}

+

+	if err := ma.validateAndApplyDefaults(c.fs); err != nil {

+		return nil, err

+	}

+

+	if !moduleImport.IgnoreConfig {

+		if err := c.applyThemeConfig(ma); err != nil {

+			return nil, err

+		}

+	}

+

+	if err := c.applyMounts(moduleImport, ma); err != nil {

+		return nil, err

+	}

+

+	c.modules = append(c.modules, ma)

+	return ma, nil

+

+}

+

+func (c *collector) addAndRecurse(owner *moduleAdapter, disabled bool) error {

+	moduleConfig := owner.Config()

+	if owner.projectMod {

+		if err := c.applyMounts(Import{}, owner); err != nil {

+			return err

+		}

+	}

+

+	for _, moduleImport := range moduleConfig.Imports {

+		disabled := disabled || moduleImport.Disabled

+

+		if !c.isSeen(moduleImport.Path) {

+			tc, err := c.add(owner, moduleImport, disabled)

+			if err != nil {

+				return err

+			}

+			if tc == nil {

+				continue

+			}

+			if err := c.addAndRecurse(tc, disabled); err != nil {

+				return err

+			}

+		}

+	}

+	return nil

+}

+

+func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error {

+	mounts := moduleImport.Mounts

+

+	if !mod.projectMod && len(mounts) == 0 {

+		modConfig := mod.Config()

+		mounts = modConfig.Mounts

+		if len(mounts) == 0 {

+			// Create default mount points for every component folder that

+			// exists in the module.

+			for _, componentFolder := range files.ComponentFolders {

+				sourceDir := filepath.Join(mod.Dir(), componentFolder)

+				_, err := c.fs.Stat(sourceDir)

+				if err == nil {

+					mounts = append(mounts, Mount{

+						Source: componentFolder,

+						Target: componentFolder,

+					})

+				}

+			}

+		}

+	}

+

+	var err error

+	mounts, err = c.normalizeMounts(mod, mounts)

+	if err != nil {

+		return err

+	}

+

+	mod.mounts = mounts

+	return nil

+}

+

+func (c *collector) applyThemeConfig(tc *moduleAdapter) error {

+

+	var (

+		configFilename string

+		cfg            config.Provider

+		themeCfg       map[string]interface{}

+		hasConfig      bool

+		err            error

+	)

+

+	// Viper supports more, but this is the sub-set supported by Hugo.

+	for _, configFormats := range config.ValidConfigFileExtensions {

+		configFilename = filepath.Join(tc.Dir(), "config."+configFormats)

+		hasConfig, _ = afero.Exists(c.fs, configFilename)

+		if hasConfig {

+			break

+		}

+	}

+

+	// The old theme information file.

+	themeTOML := filepath.Join(tc.Dir(), "theme.toml")

+

+	hasThemeTOML, _ := afero.Exists(c.fs, themeTOML)

+	if hasThemeTOML {

+		data, err := afero.ReadFile(c.fs, themeTOML)

+		if err != nil {

+			return err

+		}

+		themeCfg, err = metadecoders.Default.UnmarshalToMap(data, metadecoders.TOML)

+		if err != nil {

+			return errors.Wrapf(err, "failed to read module config for %q in %q", tc.Path(), themeTOML)

+		}

+		maps.ToLower(themeCfg)

+	}

+

+	if hasConfig {

+		if configFilename != "" {

+			var err error

+			cfg, err = config.FromFile(c.fs, configFilename)

+			if err != nil {

+				return errors.Wrapf(err, "failed to read module config for %q in %q", tc.Path(), configFilename)

+			}

+		}

+

+		tc.configFilename = configFilename

+		tc.cfg = cfg

+	}

+

+	config, err := DecodeConfig(cfg)

+	if err != nil {

+		return err

+	}

+

+	const oldVersionKey = "min_version"

+

+	if hasThemeTOML {

+

+		// Merge old with new

+		if minVersion, found := themeCfg[oldVersionKey]; found {

+			if config.HugoVersion.Min == "" {

+				config.HugoVersion.Min = hugo.VersionString(cast.ToString(minVersion))

+			}

+		}

+

+		if config.Params == nil {

+			config.Params = make(map[string]interface{})

+		}

+

+		for k, v := range themeCfg {

+			if k == oldVersionKey {

+				continue

+			}

+			config.Params[k] = v

+		}

+

+	}

+

+	tc.config = config

+

+	return nil

+

+}

+

+func (c *collector) collect() {

+	if err := c.initModules(); err != nil {

+		c.err = err

+		return

+	}

+

+	projectMod := createProjectModule(c.gomods.GetMain(), c.workingDir, c.moduleConfig)

+

+	if err := c.addAndRecurse(projectMod, false); err != nil {

+		c.err = err

+		return

+	}

+

+	// Append the project module at the tail.

+	c.modules = append(c.modules, projectMod)

+

+}

+

+func (c *collector) collectModulesTXT(owner Module) error {

+	vendorDir := filepath.Join(owner.Dir(), vendord)

+	filename := filepath.Join(vendorDir, vendorModulesFilename)

+

+	f, err := c.fs.Open(filename)

+

+	if err != nil {

+		if os.IsNotExist(err) {

+			return nil

+		}

+

+		return err

+	}

+

+	defer f.Close()

+

+	scanner := bufio.NewScanner(f)

+

+	for scanner.Scan() {

+		// # github.com/alecthomas/chroma v0.6.3

+		line := scanner.Text()

+		line = strings.Trim(line, "# ")

+		line = strings.TrimSpace(line)

+		parts := strings.Fields(line)

+		if len(parts) != 2 {

+			return errors.Errorf("invalid modules list: %q", filename)

+		}

+		path := parts[0]

+		if _, found := c.vendored[path]; !found {

+			c.vendored[path] = vendoredModule{

+				Owner:   owner,

+				Dir:     filepath.Join(vendorDir, path),

+				Version: parts[1],

+			}

+		}

+

+	}

+	return nil

+}

+

+func (c *collector) loadModules() error {

+	modules, err := c.listGoMods()

+	if err != nil {

+		return err

+	}

+	c.gomods = modules

+	return nil

+}

+

+func (c *collector) normalizeMounts(owner Module, mounts []Mount) ([]Mount, error) {

+	var out []Mount

+	dir := owner.Dir()

+

+	for _, mnt := range mounts {

+		errMsg := fmt.Sprintf("invalid module config for %q", owner.Path())

+

+		if mnt.Source == "" || mnt.Target == "" {

+			return nil, errors.New(errMsg + ": both source and target must be set")

+		}

+

+		mnt.Source = filepath.Clean(mnt.Source)

+		mnt.Target = filepath.Clean(mnt.Target)

+

+		// Verify that Source exists

+		sourceDir := filepath.Join(dir, mnt.Source)

+		_, err := c.fs.Stat(sourceDir)

+		if err != nil {

+			continue

+		}

+

+		// Verify that target points to one of the predefined component dirs

+		targetBase := mnt.Target

+		idxPathSep := strings.Index(mnt.Target, string(os.PathSeparator))

+		if idxPathSep != -1 {

+			targetBase = mnt.Target[0:idxPathSep]

+		}

+		if !files.IsComponentFolder(targetBase) {

+			return nil, errors.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders)

+		}

+

+		out = append(out, mnt)

+	}

+

+	return out, nil

+}

+

+func (c *collector) wrapModuleNotFound(err error) error {

+	err = errors.Wrap(ErrNotExist, err.Error())

+	if c.GoModulesFilename == "" {

+		return err

+	}

+

+	baseMsg := "we found a go.mod file in your project, but"

+

+	switch c.goBinaryStatus {

+	case goBinaryStatusNotFound:

+		return errors.Wrap(err, baseMsg+" you need to install Go to use it. See https://golang.org/dl/.")

+	case goBinaryStatusTooOld:

+		return errors.Wrap(err, baseMsg+" you need to a newer version of Go to use it. See https://golang.org/dl/.")

+	}

+

+	return err

+

+}

+

+type vendoredModule struct {

+	Owner   Module

+	Dir     string

+	Version string

+}

+

+func createProjectModule(gomod *goModule, workingDir string, conf Config) *moduleAdapter {

+	// Create a pseudo module for the main project.

+	var path string

+	if gomod == nil {

+		path = "project"

+	}

+

+	return &moduleAdapter{

+		path:       path,

+		dir:        workingDir,

+		gomod:      gomod,

+		projectMod: true,

+		config:     conf,

+	}

+

+}

+

+// In the first iteration of Hugo Modules, we do not support multiple

+// major versions running at the same time, so we pick the first (upper most).

+// We will investigate namespaces in future versions.

+// TODO(bep) add a warning when the above happens.

+func pathKey(p string) string {

+	prefix, _, _ := module.SplitPathVersion(p)

+

+	return strings.ToLower(prefix)

+}

--- /dev/null

+++ b/modules/collect_test.go

@@ -1,0 +1,38 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package modules

+

+import (

+	"testing"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestPathKey(t *testing.T) {

+	assert := require.New(t)

+

+	for _, test := range []struct {

+		in     string

+		expect string

+	}{

+		{"github.com/foo", "github.com/foo"},

+		{"github.com/foo/v2", "github.com/foo"},

+		{"github.com/foo/v12", "github.com/foo"},

+		{"github.com/foo/v3d", "github.com/foo/v3d"},

+		{"MyTheme", "mytheme"},

+	} {

+		assert.Equal(test.expect, pathKey(test.in))

+	}

+

+}

--- /dev/null

+++ b/modules/config.go

@@ -1,0 +1,335 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package modules

+

+import (

+	"fmt"

+	"path"

+	"path/filepath"

+	"strings"

+

+	"github.com/gohugoio/hugo/common/hugo"

+

+	"github.com/gohugoio/hugo/config"

+	"github.com/gohugoio/hugo/hugofs/files"

+	"github.com/gohugoio/hugo/langs"

+	"github.com/mitchellh/mapstructure"

+)

+

+var DefaultModuleConfig = Config{

+

+	// Default to direct, which means "git clone" and similar. We

+	// will investigate proxy settings in more depth later.

+	// See https://github.com/golang/go/issues/26334

+	Proxy: "direct",

+

+	// Comma separated glob list matching paths that should not use the

+	// proxy configured above.

+	NoProxy: "none",

+

+	// Comma separated glob list matching paths that should be

+	// treated as private.

+	Private: "*.*",

+}

+

+// ApplyProjectConfigDefaults applies default/missing module configuration for

+// the main project.

+func ApplyProjectConfigDefaults(cfg config.Provider, mod Module) error {

+	moda := mod.(*moduleAdapter)

+

+	// Map legacy directory config into the new module.

+	languages := cfg.Get("languagesSortedDefaultFirst").(langs.Languages)

+	isMultiHost := languages.IsMultihost()

+

+	// To bridge between old and new configuration format we need

+	// a way to make sure all of the core components are configured on

+	// the basic level.

+	componentsConfigured := make(map[string]bool)

+	for _, mnt := range moda.mounts {

+		componentsConfigured[mnt.Component()] = true

+	}

+

+	type dirKeyComponent struct {

+		key          string

+		component    string

+		multilingual bool

+	}

+

+	dirKeys := []dirKeyComponent{

+		{"contentDir", files.ComponentFolderContent, true},

+		{"dataDir", files.ComponentFolderData, false},

+		{"layoutDir", files.ComponentFolderLayouts, false},

+		{"i18nDir", files.ComponentFolderI18n, false},

+		{"archetypeDir", files.ComponentFolderArchetypes, false},

+		{"assetDir", files.ComponentFolderAssets, false},

+		{"", files.ComponentFolderStatic, isMultiHost},

+	}

+

+	createMountsFor := func(d dirKeyComponent, cfg config.Provider) []Mount {

+		var lang string

+		if language, ok := cfg.(*langs.Language); ok {

+			lang = language.Lang

+		}

+

+		// Static mounts are a little special.

+		if d.component == files.ComponentFolderStatic {

+			var mounts []Mount

+			staticDirs := getStaticDirs(cfg)

+			if len(staticDirs) > 0 {

+				componentsConfigured[d.component] = true

+			}

+

+			for _, dir := range staticDirs {

+				mounts = append(mounts, Mount{Lang: lang, Source: dir, Target: d.component})

+			}

+

+			return mounts

+

+		}

+

+		if cfg.IsSet(d.key) {

+			source := cfg.GetString(d.key)

+			componentsConfigured[d.component] = true

+

+			return []Mount{Mount{

+				// No lang set for layouts etc.

+				Source: source,

+				Target: d.component}}

+		}

+

+		return nil

+	}

+

+	createMounts := func(d dirKeyComponent) []Mount {

+		var mounts []Mount

+		if d.multilingual {

+			if d.component == files.ComponentFolderContent {

+				seen := make(map[string]bool)

+				hasContentDir := false

+				for _, language := range languages {

+					if language.ContentDir != "" {

+						hasContentDir = true

+						break

+					}

+				}

+

+				if hasContentDir {

+					for _, language := range languages {

+						contentDir := language.ContentDir

+						if contentDir == "" {

+							contentDir = files.ComponentFolderContent

+						}

+						if contentDir == "" || seen[contentDir] {

+							continue

+						}

+						seen[contentDir] = true

+						mounts = append(mounts, Mount{Lang: language.Lang, Source: contentDir, Target: d.component})

+					}

+				}

+

+				componentsConfigured[d.component] = len(seen) > 0

+

+			} else {

+				for _, language := range languages {

+					mounts = append(mounts, createMountsFor(d, language)...)

+				}

+			}

+		} else {

+			mounts = append(mounts, createMountsFor(d, cfg)...)

+		}

+

+		return mounts

+	}

+

+	var mounts []Mount

+	for _, dirKey := range dirKeys {

+		if componentsConfigured[dirKey.component] {

+

+			continue

+		}

+

+		mounts = append(mounts, createMounts(dirKey)...)

+

+	}

+

+	// Add default configuration

+	for _, dirKey := range dirKeys {

+		if componentsConfigured[dirKey.component] {

+			continue

+		}

+		mounts = append(mounts, Mount{Source: dirKey.component, Target: dirKey.component})

+	}

+

+	// Remove duplicates

+	seen := make(map[string]bool)

+	tmp := mounts[:0]

+	for _, m := range mounts {

+		key := path.Join(m.Lang, m.Source, m.Target)

+		if !seen[key] {

+			tmp = append(tmp, m)

+		}

+		seen[key] = true

+	}

+

+	moda.mounts = tmp

+

+	return nil

+}

+

+// DecodeConfig creates a modules Config from a given Hugo configuration.

+func DecodeConfig(cfg config.Provider) (Config, error) {

+	c := DefaultModuleConfig

+

+	if cfg == nil {

+		return c, nil

+	}

+

+	themeSet := cfg.IsSet("theme")

+	moduleSet := cfg.IsSet("module")

+

+	if moduleSet {

+		m := cfg.GetStringMap("module")

+		if err := mapstructure.WeakDecode(m, &c); err != nil {

+			return c, err

+		}

+

+		for i, mnt := range c.Mounts {

+			mnt.Source = filepath.Clean(mnt.Source)

+			mnt.Target = filepath.Clean(mnt.Target)

+			c.Mounts[i] = mnt

+		}

+

+	}

+

+	if themeSet {

+		imports := config.GetStringSlicePreserveString(cfg, "theme")

+		for _, imp := range imports {

+			c.Imports = append(c.Imports, Import{

+				Path: imp,

+			})

+		}

+

+	}

+

+	return c, nil

+}

+

+// Config holds a module config.

+type Config struct {

+	Mounts  []Mount

+	Imports []Import

+

+	// Meta info about this module (license information etc.).

+	Params map[string]interface{}

+

+	// Will be validated against the running Hugo version.

+	HugoVersion HugoVersion

+

+	// Configures GOPROXY.

+	Proxy string

+	// Configures GONOPROXY.

+	NoProxy string

+	// Configures GOPRIVATE.

+	Private string

+}

+

+// HugoVersion holds Hugo binary version requirements for a module.

+type HugoVersion struct {

+	// The minimum Hugo version that this module works with.

+	Min hugo.VersionString

+

+	// The maxium Hugo version that this module works with.

+	Max hugo.VersionString

+

+	// Set if the extended version is needed.

+	Extended bool

+}

+

+func (v HugoVersion) String() string {

+	extended := ""

+	if v.Extended {

+		extended = " extended"

+	}

+

+	if v.Min != "" && v.Max != "" {

+		return fmt.Sprintf("%s/%s%s", v.Min, v.Max, extended)

+	}

+

+	if v.Min != "" {

+		return fmt.Sprintf("Min %s%s", v.Min, extended)

+	}

+

+	if v.Max != "" {

+		return fmt.Sprintf("Max %s%s", v.Max, extended)

+	}

+

+	return extended

+}

+

+// IsValid reports whether this version is valid compared to the running

+// Hugo binary.

+func (v HugoVersion) IsValid() bool {

+	current := hugo.CurrentVersion.Version()

+	if v.Extended && !hugo.IsExtended {

+		return false

+	}

+

+	isValid := true

+

+	if v.Min != "" && current.Compare(v.Min) > 0 {

+		isValid = false

+	}

+

+	if v.Max != "" && current.Compare(v.Max) < 0 {

+		isValid = false

+	}

+

+	return isValid

+}

+

+type Import struct {

+	Path         string // Module path

+	IgnoreConfig bool   // Ignore any config.toml found.

+	Disabled     bool   // Turn off this module.

+	Mounts       []Mount

+}

+

+type Mount struct {

+	Source string // relative path in source repo, e.g. "scss"

+	Target string // relative target path, e.g. "assets/bootstrap/scss"

+

+	Lang string // any language code associated with this mount.

+}

+

+func (m Mount) Component() string {

+	return strings.Split(m.Target, fileSeparator)[0]

+}

+

+func getStaticDirs(cfg config.Provider) []string {

+	var staticDirs []string

+	for i := -1; i <= 10; i++ {

+		staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)

+	}

+	return staticDirs

+}

+

+func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {

+

+	if id >= 0 {

+		key = fmt.Sprintf("%s%d", key, id)

+	}

+

+	return config.GetStringSlicePreserveString(cfg, key)

+

+}

--- /dev/null

+++ b/modules/config_test.go

@@ -1,0 +1,132 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package modules

+

+import (

+	"fmt"

+	"testing"

+

+	"github.com/gohugoio/hugo/common/hugo"

+

+	"github.com/gohugoio/hugo/config"

+

+	"github.com/stretchr/testify/require"

+)

+

+func TestConfigHugoVersionIsValid(t *testing.T) {

+	assert := require.New(t)

+

+	for i, test := range []struct {

+		in     HugoVersion

+		expect bool

+	}{

+		{HugoVersion{Min: "0.33.0"}, true},

+		{HugoVersion{Min: "0.56.0-DEV"}, true},

+		{HugoVersion{Min: "0.33.0", Max: "0.55.0"}, false},

+		{HugoVersion{Min: "0.33.0", Max: "0.99.0"}, true},

+	} {

+		assert.Equal(test.expect, test.in.IsValid(), fmt.Sprintf("test %d", i))

+	}

+}

+

+func TestDecodeConfig(t *testing.T) {

+	assert := require.New(t)

+	tomlConfig := `

+[module]

+

+[module.hugoVersion]

+min = "0.54.2"

+max = "0.99.0"

+extended = true

+

+[[module.mounts]]

+source="src/project/blog"

+target="content/blog"

+lang="en"

+[[module.imports]]

+path="github.com/bep/mycomponent"

+[[module.imports.mounts]]

+source="scss"

+target="assets/bootstrap/scss"

+[[module.imports.mounts]]

+source="src/markdown/blog"

+target="content/blog"

+lang="en"

+`

+	cfg, err := config.FromConfigString(tomlConfig, "toml")

+	assert.NoError(err)

+

+	mcfg, err := DecodeConfig(cfg)

+	assert.NoError(err)

+

+	v056 := hugo.VersionString("0.56.0")

+

+	hv := mcfg.HugoVersion

+

+	assert.Equal(-1, v056.Compare(hv.Min))

+	assert.Equal(1, v056.Compare(hv.Max))

+	assert.True(hv.Extended)

+

+	if hugo.IsExtended {

+		assert.True(hv.IsValid())

+	}

+

+	assert.Len(mcfg.Mounts, 1)

+	assert.Len(mcfg.Imports, 1)

+	imp := mcfg.Imports[0]

+	imp.Path = "github.com/bep/mycomponent"

+	assert.Equal("src/markdown/blog", imp.Mounts[1].Source)

+	assert.Equal("content/blog", imp.Mounts[1].Target)

+	assert.Equal("en", imp.Mounts[1].Lang)

+

+}

+

+func TestDecodeConfigBothOldAndNewProvided(t *testing.T) {

+	assert := require.New(t)

+	tomlConfig := `

+

+theme = ["b", "c"]

+

+[module]

+[[module.imports]]

+path="a"

+

+`

+	cfg, err := config.FromConfigString(tomlConfig, "toml")

+	assert.NoError(err)

+

+	modCfg, err := DecodeConfig(cfg)

+	assert.NoError(err)

+	assert.Len(modCfg.Imports, 3)

+	assert.Equal("a", modCfg.Imports[0].Path)

+

+}

+

+// Test old style theme import.

+func TestDecodeConfigTheme(t *testing.T) {

+	assert := require.New(t)

+	tomlConfig := `

+

+theme = ["a", "b"]

+`

+	cfg, err := config.FromConfigString(tomlConfig, "toml")

+	assert.NoError(err)

+

+	mcfg, err := DecodeConfig(cfg)

+	assert.NoError(err)

+

+	assert.Len(mcfg.Imports, 2)

+	assert.Equal("a", mcfg.Imports[0].Path)

+	assert.Equal("b", mcfg.Imports[1].Path)

+}

--- /dev/null

+++ b/modules/module.go

@@ -1,0 +1,196 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+// Package modules provides a client that can be used to manage Hugo Components,

+// what's refered to as Hugo Modules. Hugo Modules is built on top of Go Modules,

+// but also supports vendoring and components stored directly in the themes dir.

+package modules

+

+import (

+	"github.com/gohugoio/hugo/config"

+	"github.com/spf13/afero"

+)

+

+var _ Module = (*moduleAdapter)(nil)

+

+type Module interface {

+

+	// Optional config read from the configFilename above.

+	Cfg() config.Provider

+

+	// The decoded module config and mounts.

+	Config() Config

+

+	// Optional configuration filename (e.g. "/themes/mytheme/config.json").

+	// This will be added to the special configuration watch list when in

+	// server mode.

+	ConfigFilename() string

+

+	// Directory holding files for this module.

+	Dir() string

+

+	// This module is disabled.

+	Disabled() bool

+

+	// Returns whether this is a Go Module.

+	IsGoMod() bool

+

+	// Any directory remappings.

+	Mounts() []Mount

+

+	// In the dependency tree, this is the first module that defines this module

+	// as a dependency.

+	Owner() Module

+

+	// Returns the path to this module.

+	// This will either be the module path, e.g. "github.com/gohugoio/myshortcodes",

+	// or the path below your /theme folder, e.g. "mytheme".

+	Path() string

+

+	// Replaced by this module.

+	Replace() Module

+

+	// Returns whether Dir points below the _vendor dir.

+	Vendor() bool

+

+	// The module version.

+	Version() string

+

+	// Whether this module's dir is a watch candidate.

+	Watch() bool

+}

+

+type Modules []Module

+

+type moduleAdapter struct {

+	path       string

+	dir        string

+	version    string

+	vendor     bool

+	disabled   bool

+	projectMod bool

+	owner      Module

+

+	mounts []Mount

+

+	configFilename string

+	cfg            config.Provider

+	config         Config

+

+	// Set if a Go module.

+	gomod *goModule

+}

+

+func (m *moduleAdapter) Cfg() config.Provider {

+	return m.cfg

+}

+

+func (m *moduleAdapter) Config() Config {

+	return m.config

+}

+

+func (m *moduleAdapter) ConfigFilename() string {

+	return m.configFilename

+}

+

+func (m *moduleAdapter) Dir() string {

+	// This may point to the _vendor dir.

+	if !m.IsGoMod() || m.dir != "" {

+		return m.dir

+	}

+	return m.gomod.Dir

+}

+

+func (m *moduleAdapter) Disabled() bool {

+	return m.disabled

+}

+

+func (m *moduleAdapter) IsGoMod() bool {

+	return m.gomod != nil

+}

+

+func (m *moduleAdapter) Mounts() []Mount {

+	return m.mounts

+}

+

+func (m *moduleAdapter) Owner() Module {

+	return m.owner

+}

+

+func (m *moduleAdapter) Path() string {

+	if !m.IsGoMod() || m.path != "" {

+		return m.path

+	}

+	return m.gomod.Path

+}

+

+func (m *moduleAdapter) Replace() Module {

+	if m.IsGoMod() && !m.Vendor() && m.gomod.Replace != nil {

+		return &moduleAdapter{

+			gomod: m.gomod.Replace,

+			owner: m.owner,

+		}

+	}

+	return nil

+}

+

+func (m *moduleAdapter) Vendor() bool {

+	return m.vendor

+}

+

+func (m *moduleAdapter) Version() string {

+	if !m.IsGoMod() || m.version != "" {

+		return m.version

+	}

+	return m.gomod.Version

+}

+

+func (m *moduleAdapter) Watch() bool {

+	if m.Owner() == nil {

+		// Main project

+		return true

+	}

+

+	if !m.IsGoMod() {

+		// Module inside /themes

+		return true

+	}

+

+	if m.Replace() != nil {

+		// Version is not set when replaced by a local folder.

+		return m.Replace().Version() == ""

+	}

+

+	return false

+}

+

+func (m *moduleAdapter) validateAndApplyDefaults(fs afero.Fs) error {

+

+	/*if len(m.modImport.Mounts) == 0 {

+		// Create default mount points for every component folder that

+		// exists in the module.

+		for _, componentFolder := range files.ComponentFolders {

+			sourceDir := filepath.Join(dir, componentFolder)

+			_, err := fs.Stat(sourceDir)

+			if err == nil {

+				m.modImport.Mounts = append(m.modImport.Mounts, Mount{

+					Source: componentFolder,

+					Target: componentFolder,

+				})

+			}

+		}

+	}*/

+

+	return nil

+

+}

--- a/output/layout.go

+++ b/output/layout.go

@@ -71,7 +71,7 @@

 	layouts := resolvePageTemplate(d, f)

 	layouts = prependTextPrefixIfNeeded(f, layouts...)

-	layouts = helpers.UniqueStrings(layouts)

+	layouts = helpers.UniqueStringsReuse(layouts)

 	l.mu.Lock()

 	l.cache[key] = layouts

--- a/parser/metadecoders/decoder.go

+++ b/parser/metadecoders/decoder.go

@@ -82,6 +82,32 @@

 	return d.UnmarshalToMap(data, format)

 }

+// UnmarshalStringTo tries to unmarshal data to a new instance of type typ.

+func (d Decoder) UnmarshalStringTo(data string, typ interface{}) (interface{}, error) {

+	data = strings.TrimSpace(data)

+	// We only check for the possible types in YAML, JSON and TOML.

+	switch typ.(type) {

+	case string:

+		return data, nil

+	case map[string]interface{}:

+		format := d.FormatFromContentString(data)

+		return d.UnmarshalToMap([]byte(data), format)

+	case []interface{}:

+		// A standalone slice. Let YAML handle it.

+		return d.Unmarshal([]byte(data), YAML)

+	case bool:

+		return cast.ToBoolE(data)

+	case int:

+		return cast.ToIntE(data)

+	case int64:

+		return cast.ToInt64E(data)

+	case float64:

+		return cast.ToFloat64E(data)

+	default:

+		return nil, errors.Errorf("unmarshal: %T not supportedd", typ)

+	}

+}

+

 // Unmarshal will unmarshall data in format f into an interface{}.

 // This is what's needed for Hugo's /data handling.

 func (d Decoder) Unmarshal(data []byte, f Format) (interface{}, error) {

--- a/parser/metadecoders/decoder_test.go

+++ b/parser/metadecoders/decoder_test.go

@@ -90,6 +90,38 @@

 }

+func TestUnmarshalStringTo(t *testing.T) {

+	assert := require.New(t)

+

+	d := Default

+

+	expectMap := map[string]interface{}{"a": "b"}

+

+	for i, test := range []struct {

+		data   string

+		to     interface{}

+		expect interface{}

+	}{

+		{"a string", "string", "a string"},

+		{`{ "a": "b" }`, make(map[string]interface{}), expectMap},

+		{"32", int64(1234), int64(32)},

+		{"32", int(1234), int(32)},

+		{"3.14159", float64(1), float64(3.14159)},

+		{"[3,7,9]", []interface{}{}, []interface{}{3, 7, 9}},

+		{"[3.1,7.2,9.3]", []interface{}{}, []interface{}{3.1, 7.2, 9.3}},

+	} {

+		msg := fmt.Sprintf("%d: %T", i, test.to)

+		m, err := d.UnmarshalStringTo(test.data, test.to)

+		if b, ok := test.expect.(bool); ok && !b {

+			assert.Error(err, msg)

+		} else {

+			assert.NoError(err, msg)

+			assert.Equal(test.expect, m, msg)

+		}

+

+	}

+}

+

 func TestStringifyYAMLMapKeys(t *testing.T) {

 	cases := []struct {

 		input    interface{}

--- a/resources/image_cache.go

+++ b/resources/image_cache.go

@@ -145,7 +145,7 @@

 	}

 	// The file is now stored in this cache.

-	img.overriddenSourceFs = c.fileCache.Fs

+	img.sourceFs = c.fileCache.Fs

 	c.mu.Lock()

 	if img2, found := c.store[key]; found {

--- a/resources/page/page_nop.go

+++ b/resources/page/page_nop.go

@@ -17,9 +17,10 @@

 import (

 	"html/template"

-	"os"

 	"time"

+	"github.com/gohugoio/hugo/hugofs"

+

 	"github.com/bep/gitmap"

 	"github.com/gohugoio/hugo/navigation"

@@ -147,7 +148,7 @@

 	return nilFile

 }

-func (p *nopPage) FileInfo() os.FileInfo {

+func (p *nopPage) FileInfo() hugofs.FileMetaInfo {

 	return nil

 }

--- a/resources/page/page_wrappers.autogen.go

+++ b/resources/page/page_wrappers.autogen.go

@@ -18,8 +18,8 @@

 import (

 	"github.com/gohugoio/hugo/common/hugo"

 	"github.com/gohugoio/hugo/helpers"

+	"github.com/gohugoio/hugo/hugofs"

 	"html/template"

-	"os"

 )

 // NewDeprecatedWarningPage adds deprecation warnings to the given implementation.

@@ -91,7 +91,7 @@

 	helpers.Deprecated("Page", ".UniqueID", "Use .File.UniqueID", false)

 	return p.p.UniqueID()

 }

-func (p *pageDeprecated) FileInfo() os.FileInfo {

+func (p *pageDeprecated) FileInfo() hugofs.FileMetaInfo {

 	helpers.Deprecated("Page", ".FileInfo", "Use .File.FileInfo", false)

 	return p.p.FileInfo()

 }

--- a/resources/page/pagemeta/page_frontmatter.go

+++ b/resources/page/pagemeta/page_frontmatter.go

@@ -236,7 +236,7 @@

 			complete = append(complete, aliases...)

 		}

 	}

-	return helpers.UniqueStrings(complete)

+	return helpers.UniqueStringsReuse(complete)

 }

 func expandDefaultValues(values []string, defaults []string) []string {

--- a/resources/page/permalinks.go

+++ b/resources/page/permalinks.go

@@ -15,6 +15,7 @@

 import (

 	"fmt"

+	"os"

 	"path/filepath"

 	"regexp"

 	"strconv"

@@ -90,7 +91,11 @@

 	expanders := make(map[string]func(Page) (string, error))

+	// Allow " " and / to represent the root section.

+	const sectionCutSet = " /" + string(os.PathSeparator)

+

 	for k, pattern := range patterns {

+		k = strings.Trim(k, sectionCutSet)

 		if !l.validate(pattern) {

 			return nil, &permalinkExpandError{pattern: pattern, err: errPermalinkIllFormed}

 		}

--- a/resources/page/site.go

+++ b/resources/page/site.go

@@ -17,6 +17,8 @@

 	"html/template"

 	"time"

+	"github.com/gohugoio/hugo/config"

+

 	"github.com/gohugoio/hugo/common/hugo"

 	"github.com/gohugoio/hugo/langs"

 	"github.com/gohugoio/hugo/navigation"

@@ -50,4 +52,73 @@

 		return nil

 	}

 	return s[0]

+}

+

+type testSite struct {

+	h hugo.Info

+	l *langs.Language

+}

+

+func (t testSite) Hugo() hugo.Info {

+	return t.h

+}

+

+func (t testSite) ServerPort() int {

+	return 1313

+}

+

+func (testSite) LastChange() (t time.Time) {

+	return

+}

+

+func (t testSite) Title() string {

+	return "foo"

+}

+

+func (t testSite) Sites() Sites {

+	return nil

+}

+

+func (t testSite) IsServer() bool {

+	return false

+}

+

+func (t testSite) Language() *langs.Language {

+	return t.l

+}

+

+func (t testSite) Pages() Pages {

+	return nil

+}

+

+func (t testSite) RegularPages() Pages {

+	return nil

+}

+

+func (t testSite) Menus() navigation.Menus {

+	return nil

+}

+

+func (t testSite) Taxonomies() interface{} {

+	return nil

+}

+

+func (t testSite) BaseURL() template.URL {

+	return ""

+}

+

+func (t testSite) Params() map[string]interface{} {

+	return nil

+}

+

+func (t testSite) Data() map[string]interface{} {

+	return nil

+}

+

+// NewDummyHugoSite creates a new minimal test site.

+func NewDummyHugoSite(cfg config.Provider) Site {

+	return testSite{

+		h: hugo.NewInfo(hugo.EnvironmentProduction),

+		l: langs.NewLanguage("en", cfg),

+	}

 }

--- a/resources/page/testhelpers_test.go

+++ b/resources/page/testhelpers_test.go

@@ -16,10 +16,11 @@

 import (

 	"fmt"

 	"html/template"

-	"os"

 	"path/filepath"

 	"time"

+	"github.com/gohugoio/hugo/modules"

+

 	"github.com/bep/gitmap"

 	"github.com/gohugoio/hugo/helpers"

 	"github.com/gohugoio/hugo/resources/resource"

@@ -65,6 +66,12 @@

 func newTestPathSpecFor(cfg config.Provider) *helpers.PathSpec {

 	config.SetBaseTestDefaults(cfg)

+	langs.LoadLanguageSettings(cfg, nil)

+	mod, err := modules.CreateProjectModule(cfg)

+	if err != nil {

+		panic(err)

+	}

+	cfg.Set("allModules", modules.Modules{mod})

 	fs := hugofs.NewMem(cfg)

 	s, err := helpers.NewPathSpec(fs, cfg)

 	if err != nil {

@@ -189,7 +196,7 @@

 	return p.file

 }

-func (p *testPage) FileInfo() os.FileInfo {

+func (p *testPage) FileInfo() hugofs.FileMetaInfo {

 	panic("not implemented")

 }

--- a/resources/page/zero_file.autogen.go

+++ b/resources/page/zero_file.autogen.go

@@ -17,8 +17,8 @@

 import (

 	"github.com/gohugoio/hugo/helpers"

+	"github.com/gohugoio/hugo/hugofs"

 	"github.com/gohugoio/hugo/source"

-	"os"

 )

 // ZeroFile represents a zero value of source.File with warnings if invoked.

@@ -82,7 +82,7 @@

 	z.log.Println(".File.UniqueID on zero object. Wrap it in if or with: {{ with .File }}{{ .UniqueID }}{{ end }}")

 	return

 }

-func (z zeroFile) FileInfo() (o0 os.FileInfo) {

+func (z zeroFile) FileInfo() (o0 hugofs.FileMetaInfo) {

 	z.log.Println(".File.FileInfo on zero object. Wrap it in if or with: {{ with .File }}{{ .FileInfo }}{{ end }}")

 	return

 }

--- a/resources/resource.go

+++ b/resources/resource.go

@@ -133,9 +133,13 @@

 	SourceFile         source.File

 	OpenReadSeekCloser resource.OpenReadSeekCloser

+	FileInfo os.FileInfo

+

 	// If OpenReadSeekerCloser is not set, we use this to open the file.

 	SourceFilename string

+	Fs afero.Fs

+

 	// The relative target filename without any language code.

 	RelTargetFilename string

@@ -157,19 +161,11 @@

 	return r.SourceFilename

 }

-func (r *Spec) sourceFs() afero.Fs {

-	return r.PathSpec.BaseFs.Content.Fs

-}

-

 func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) {

-	return r.newResourceForFs(r.sourceFs(), fd)

+	return r.newResourceFor(fd)

 }

-func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {

-	return r.newResourceForFs(sourceFs, fd)

-}

-

-func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {

+func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) {

 	if fd.OpenReadSeekCloser == nil {

 		if fd.SourceFile != nil && fd.SourceFilename != "" {

 			return nil, errors.New("both SourceFile and AbsSourceFilename provided")

@@ -187,15 +183,14 @@

 		fd.TargetBasePaths = r.MultihostTargetBasePaths

 	}

-	return r.newResource(sourceFs, fd)

+	return r.newResource(fd.Fs, fd)

 }

 func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {

-	var fi os.FileInfo

+	fi := fd.FileInfo

 	var sourceFilename string

 	if fd.OpenReadSeekCloser != nil {

-

 	} else if fd.SourceFilename != "" {

 		var err error

 		fi, err = sourceFs.Stat(fd.SourceFilename)

@@ -207,7 +202,6 @@

 		}

 		sourceFilename = fd.SourceFilename

 	} else {

-		fi = fd.SourceFile.FileInfo()

 		sourceFilename = fd.SourceFile.Filename()

 	}

@@ -245,8 +239,6 @@

 		mimeType)

 	if mimeType.MainType == "image" {

-		ext := strings.ToLower(helpers.Ext(sourceFilename))

-

 		imgFormat, ok := imageFormats[ext]

 		if !ok {

 			// This allows SVG etc. to be used as resources. They will not have the methods of the Image, but

@@ -376,7 +368,7 @@

 	// This may be set to tell us to look in another filesystem for this resource.

 	// We, by default, use the sourceFs filesystem in the spec below.

-	overriddenSourceFs afero.Fs

+	sourceFs afero.Fs

 	spec *Spec

@@ -411,7 +403,8 @@

 	if l.openReadSeekerCloser != nil {

 		return l.openReadSeekerCloser()

 	}

-	f, err := l.sourceFs().Open(l.sourceFilename)

+

+	f, err := l.getSourceFs().Open(l.sourceFilename)

 	if err != nil {

 		return nil, err

 	}

@@ -497,11 +490,8 @@

 	return err

 }

-func (l *genericResource) sourceFs() afero.Fs {

-	if l.overriddenSourceFs != nil {

-		return l.overriddenSourceFs

-	}

-	return l.spec.sourceFs()

+func (l *genericResource) getSourceFs() afero.Fs {

+	return l.sourceFs

 }

 func (l *genericResource) publishIfNeeded() {

@@ -711,6 +701,10 @@

 	baseFilename string,

 	mediaType media.Type) *genericResource {

+	if osFileInfo != nil && osFileInfo.IsDir() {

+		panic(fmt.Sprintf("dirs nto supported resource types: %v", osFileInfo))

+	}

+

 	// This value is used both to construct URLs and file paths, but start

 	// with a Unix-styled path.

 	baseFilename = helpers.ToSlashTrimLeading(baseFilename)

@@ -724,7 +718,7 @@

 	}

 	pathDescriptor := resourcePathDescriptor{

-		baseTargetPathDirs: helpers.UniqueStrings(targetPathBaseDirs),

+		baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs),

 		targetPathBuilder:  targetPathBuilder,

 		relTargetDirFile:   dirFile{dir: fpath, file: fname},

 	}

@@ -738,7 +732,7 @@

 		openReadSeekerCloser:   openReadSeekerCloser,

 		publishOnce:            po,

 		resourcePathDescriptor: pathDescriptor,

-		overriddenSourceFs:     sourceFs,

+		sourceFs:               sourceFs,

 		osFileInfo:             osFileInfo,

 		sourceFilename:         sourceFilename,

 		mediaType:              mediaType,

--- a/resources/resource/params.go

+++ b/resources/resource/params.go

@@ -14,7 +14,7 @@

 package resource

 import (

-	"strings"

+	"github.com/gohugoio/hugo/common/maps"

 	"github.com/spf13/cast"

 )

@@ -25,65 +25,6 @@

 		return nil, err

 	}

-	keyStr = strings.ToLower(keyStr)

-	result, _ := traverseDirectParams(r, fallback, keyStr)

-	if result != nil {

-		return result, nil

-	}

+	return maps.GetNestedParam(keyStr, ".", r.Params(), fallback)

-	keySegments := strings.Split(keyStr, ".")

-	if len(keySegments) == 1 {

-		return nil, nil

-	}

-

-	return traverseNestedParams(r, fallback, keySegments)

-}

-

-func traverseDirectParams(r ResourceParamsProvider, fallback map[string]interface{}, key string) (interface{}, error) {

-	keyStr := strings.ToLower(key)

-	if val, ok := r.Params()[keyStr]; ok {

-		return val, nil

-	}

-

-	if fallback == nil {

-		return nil, nil

-	}

-

-	return fallback[keyStr], nil

-}

-

-func traverseNestedParams(r ResourceParamsProvider, fallback map[string]interface{}, keySegments []string) (interface{}, error) {

-	result := traverseParams(keySegments, r.Params())

-	if result != nil {

-		return result, nil

-	}

-

-	if fallback != nil {

-		result = traverseParams(keySegments, fallback)

-		if result != nil {

-			return result, nil

-		}

-	}

-

-	// Didn't find anything, but also no problems.

-	return nil, nil

-}

-

-func traverseParams(keys []string, m map[string]interface{}) interface{} {

-	// Shift first element off.

-	firstKey, rest := keys[0], keys[1:]

-	result := m[firstKey]

-

-	// No point in continuing here.

-	if result == nil {

-		return result

-	}

-

-	if len(rest) == 0 {

-		// That was the last key.

-		return result

-	}

-

-	// That was not the last key.

-	return traverseParams(rest, cast.ToStringMap(result))

 }

--- a/resources/resource_factories/bundler/bundler.go

+++ b/resources/resource_factories/bundler/bundler.go

@@ -124,9 +124,9 @@

 			return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil

 		}

-		composite, err := c.rs.NewForFs(

-			c.rs.FileCaches.AssetsCache().Fs,

+		composite, err := c.rs.New(

 			resources.ResourceSourceDescriptor{

+				Fs:                 c.rs.FileCaches.AssetsCache().Fs,

 				LazyPublish:        true,

 				OpenReadSeekCloser: concatr,

 				RelTargetFilename:  filepath.Clean(targetPath)})

--- a/resources/resource_factories/create/create.go

+++ b/resources/resource_factories/create/create.go

@@ -40,10 +40,10 @@

 func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {

 	filename = filepath.Clean(filename)

 	return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {

-		return c.rs.NewForFs(fs,

-			resources.ResourceSourceDescriptor{

-				LazyPublish:    true,

-				SourceFilename: filename})

+		return c.rs.New(resources.ResourceSourceDescriptor{

+			Fs:             fs,

+			LazyPublish:    true,

+			SourceFilename: filename})

 	})

 }

@@ -51,9 +51,9 @@

 // FromString creates a new Resource from a string with the given relative target path.

 func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {

 	return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) {

-		return c.rs.NewForFs(

-			c.rs.FileCaches.AssetsCache().Fs,

+		return c.rs.New(

 			resources.ResourceSourceDescriptor{

+				Fs:          c.rs.FileCaches.AssetsCache().Fs,

 				LazyPublish: true,

 				OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {

 					return hugio.NewReadSeekerNoOpCloserFromString(content), nil

--- a/resources/resource_test.go

+++ b/resources/resource_test.go

@@ -21,6 +21,8 @@

 	"testing"

 	"time"

+	"github.com/spf13/afero"

+

 	"github.com/gohugoio/hugo/resources/resource"

 	"github.com/gohugoio/hugo/media"

@@ -61,8 +63,10 @@

 	writeSource(t, spec.Fs, "content/a/b/logo.png", "image")

 	writeSource(t, spec.Fs, "content/a/b/data.json", "json")

-	r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"})

+	bfs := afero.NewBasePathFs(spec.Fs.Source, "content")

+	r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/logo.png"})

+

 	assert.NoError(err)

 	assert.NotNil(r)

 	assert.Equal("image", r.ResourceType())

@@ -69,7 +73,7 @@

 	assert.Equal("/a/b/logo.png", r.RelPermalink())

 	assert.Equal("https://example.com/a/b/logo.png", r.Permalink())

-	r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"})

+	r, err = spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/data.json"})

 	assert.NoError(err)

 	assert.NotNil(r)

@@ -85,8 +89,10 @@

 	spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs")

 	writeSource(t, spec.Fs, "content/a/b/logo.png", "image")

+	bfs := afero.NewBasePathFs(spec.Fs.Source, "content")

-	r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")})

+	fmt.Println()

+	r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: filepath.FromSlash("a/b/logo.png")})

 	assert.NoError(err)

 	assert.NotNil(r)

--- a/resources/resource_transformers/postcss/postcss.go

+++ b/resources/resource_transformers/postcss/postcss.go

@@ -130,7 +130,7 @@

 	if !filepath.IsAbs(configFile) {

 		// We resolve this against the virtual Work filesystem, to allow

 		// this config file to live in one of the themes if needed.

-		fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile)

+		fi, err := t.rs.BaseFs.Work.Stat(configFile)

 		if err != nil {

 			if t.options.Config != "" {

 				// Only fail if the user specificed config file is not found.

@@ -138,7 +138,7 @@

 			}

 			configFile = ""

 		} else {

-			configFile = fi.(hugofs.RealFilenameInfo).RealFilename()

+			configFile = fi.(hugofs.FileMetaInfo).Meta().Filename()

 		}

 	}

--- a/resources/resource_transformers/tocss/scss/client.go

+++ b/resources/resource_transformers/tocss/scss/client.go

@@ -19,6 +19,7 @@

 	"github.com/gohugoio/hugo/hugolib/filesystems"

 	"github.com/gohugoio/hugo/resources"

 	"github.com/gohugoio/hugo/resources/resource"

+	"github.com/spf13/afero"

 	"github.com/mitchellh/mapstructure"

 )

@@ -26,7 +27,7 @@

 type Client struct {

 	rs     *resources.Spec

 	sfs    *filesystems.SourceFilesystem

-	workFs *filesystems.SourceFilesystem

+	workFs afero.Fs

 }

 func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) {

--- a/resources/resource_transformers/tocss/scss/tocss.go

+++ b/resources/resource_transformers/tocss/scss/tocss.go

@@ -55,7 +55,11 @@

 	// Append any workDir relative include paths

 	for _, ip := range options.from.IncludePaths {

-		options.to.IncludePaths = append(options.to.IncludePaths, t.c.workFs.RealDirs(filepath.Clean(ip))...)

+		info, err := t.c.workFs.Stat(filepath.Clean(ip))

+		if err == nil {

+			filename := info.(hugofs.FileMetaInfo).Meta().Filename()

+			options.to.IncludePaths = append(options.to.IncludePaths, filename)

+		}

 	}

 	// To allow for overrides of SCSS files anywhere in the project/theme hierarchy, we need

@@ -74,6 +78,7 @@

 			prevDir = baseDir

 		} else {

 			prevDir = t.c.sfs.MakePathRelative(filepath.Dir(prev))

+

 			if prevDir == "" {

 				// Not a member of this filesystem. Let LibSASS handle it.

 				return "", "", false

@@ -100,8 +105,8 @@

 			filenameToCheck := filepath.Join(basePath, fmt.Sprintf(namePattern, name))

 			fi, err := t.c.sfs.Fs.Stat(filenameToCheck)

 			if err == nil {

-				if fir, ok := fi.(hugofs.RealFilenameInfo); ok {

-					return fir.RealFilename(), "", true

+				if fim, ok := fi.(hugofs.FileMetaInfo); ok {

+					return fim.Meta().Filename(), "", true

 				}

 			}

 		}

--- a/resources/testhelpers_test.go

+++ b/resources/testhelpers_test.go

@@ -4,7 +4,6 @@

 	"path/filepath"

 	"testing"

-	"fmt"

 	"image"

 	"io"

 	"io/ioutil"

@@ -12,6 +11,9 @@

 	"runtime"

 	"strings"

+	"github.com/gohugoio/hugo/langs"

+	"github.com/gohugoio/hugo/modules"

+

 	"github.com/gohugoio/hugo/cache/filecache"

 	"github.com/gohugoio/hugo/helpers"

 	"github.com/gohugoio/hugo/hugofs"

@@ -28,9 +30,8 @@

 	return newTestResourceSpecForBaseURL(assert, "https://example.com/")

 }

-func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {

+func createTestCfg() *viper.Viper {

 	cfg := viper.New()

-	cfg.Set("baseURL", baseURL)

 	cfg.Set("resourceDir", "resources")

 	cfg.Set("contentDir", "content")

 	cfg.Set("dataDir", "data")

@@ -40,6 +41,21 @@

 	cfg.Set("archetypeDir", "archetypes")

 	cfg.Set("publishDir", "public")

+	langs.LoadLanguageSettings(cfg, nil)

+	mod, err := modules.CreateProjectModule(cfg)

+	if err != nil {

+		panic(err)

+	}

+	cfg.Set("allModules", modules.Modules{mod})

+

+	return cfg

+

+}

+

+func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {

+	cfg := createTestCfg()

+	cfg.Set("baseURL", baseURL)

+

 	imagingCfg := map[string]interface{}{

 		"resampleFilter": "linear",

 		"quality":        68,

@@ -71,7 +87,7 @@

 }

 func newTestResourceOsFs(assert *require.Assertions) *Spec {

-	cfg := viper.New()

+	cfg := createTestCfg()

 	cfg.Set("baseURL", "https://example.com")

 	workDir, _ := ioutil.TempDir("", "hugores")

@@ -83,17 +99,10 @@

 	}

 	cfg.Set("workingDir", workDir)

-	cfg.Set("resourceDir", "resources")

-	cfg.Set("contentDir", "content")

-	cfg.Set("dataDir", "data")

-	cfg.Set("i18nDir", "i18n")

-	cfg.Set("layoutDir", "layouts")

-	cfg.Set("assetDir", "assets")

-	cfg.Set("archetypeDir", "archetypes")

-	cfg.Set("publishDir", "public")

 	fs := hugofs.NewFrom(hugofs.Os, cfg)

 	fs.Destination = &afero.MemMapFs{}

+	fs.Source = afero.NewBasePathFs(hugofs.Os, workDir)

 	s, err := helpers.NewPathSpec(fs, cfg)

 	assert.NoError(err)

@@ -126,7 +135,7 @@

 	src, err := os.Open(filepath.FromSlash("testdata/" + name))

 	assert.NoError(err)

-	out, err := helpers.OpenFileForWriting(spec.BaseFs.Content.Fs, name)

+	out, err := helpers.OpenFileForWriting(spec.Fs.Source, name)

 	assert.NoError(err)

 	_, err = io.Copy(out, src)

 	out.Close()

@@ -135,7 +144,7 @@

 	factory := newTargetPaths("/a")

-	r, err := spec.New(ResourceSourceDescriptor{TargetPaths: factory, LazyPublish: true, SourceFilename: name})

+	r, err := spec.New(ResourceSourceDescriptor{Fs: spec.Fs.Source, TargetPaths: factory, LazyPublish: true, SourceFilename: name})

 	assert.NoError(err)

 	return r.(resource.ContentResource)

@@ -144,9 +153,6 @@

 func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {

 	filename = filepath.Clean(filename)

 	f, err := fs.Open(filename)

-	if err != nil {

-		printFs(fs, "", os.Stdout)

-	}

 	assert.NoError(err)

 	defer f.Close()

@@ -169,23 +175,4 @@

 	if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {

 		t.Fatalf("Failed to write file: %s", err)

 	}

-}

-

-func printFs(fs afero.Fs, path string, w io.Writer) {

-	if fs == nil {

-		return

-	}

-	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {

-		if info != nil && !info.IsDir() {

-			s := path

-			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {

-				s = s + "\t" + lang.Lang()

-			}

-			if fp, ok := info.(hugofs.FilePather); ok {

-				s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir()

-			}

-			fmt.Fprintln(w, "    ", s)

-		}

-		return nil

-	})

 }

--- a/resources/transform.go

+++ b/resources/transform.go

@@ -406,6 +406,7 @@

 		}

 		if err := tr.transformation.Transform(tctx); err != nil {

+

 			if err == herrors.ErrFeatureNotAvailable {

 				// This transformation is not available in this

 				// Hugo installation (scss not compiled in, PostCSS not available etc.)

@@ -515,6 +516,9 @@

 		// Copy the file from cache to /public

 		_, src, err := r.cache.fileCache.Get(r.sourceFilename)

+		if src == nil {

+			panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename))

+		}

 		if err == nil {

 			defer src.Close()

--- a/source/fileInfo.go

+++ b/source/fileInfo.go

@@ -14,11 +14,14 @@

 package source

 import (

-	"os"

 	"path/filepath"

 	"strings"

 	"sync"

+	"github.com/gohugoio/hugo/hugofs/files"

+

+	"github.com/pkg/errors"

+

 	"github.com/gohugoio/hugo/common/hugio"

 	"github.com/gohugoio/hugo/hugofs"

@@ -28,8 +31,7 @@

 // fileInfo implements the File interface.

 var (

-	_ File         = (*FileInfo)(nil)

-	_ ReadableFile = (*FileInfo)(nil)

+	_ File = (*FileInfo)(nil)

 )

 // File represents a source file.

@@ -90,15 +92,9 @@

 	// Hugo content files being one of them, considered to be unique.

 	UniqueID() string

-	FileInfo() os.FileInfo

+	FileInfo() hugofs.FileMetaInfo

 }

-// A ReadableFile is a File that is readable.

-type ReadableFile interface {

-	File

-	Open() (hugio.ReadSeekCloser, error)

-}

-

 // FileInfo describes a source file.

 type FileInfo struct {

@@ -107,7 +103,7 @@

 	sp *SourceSpec

-	fi os.FileInfo

+	fi hugofs.FileMetaInfo

 	// Derived from filename

 	ext  string // Extension without any "."

@@ -179,13 +175,14 @@

 }

 // FileInfo returns a file's underlying os.FileInfo.

-func (fi *FileInfo) FileInfo() os.FileInfo { return fi.fi }

+func (fi *FileInfo) FileInfo() hugofs.FileMetaInfo { return fi.fi }

 func (fi *FileInfo) String() string { return fi.BaseFileName() }

 // Open implements ReadableFile.

 func (fi *FileInfo) Open() (hugio.ReadSeekCloser, error) {

-	f, err := fi.sp.SourceFs.Open(fi.Filename())

+	f, err := fi.fi.Meta().Open()

+

 	return f, err

 }

@@ -225,38 +222,45 @@

 	}

 }

-// NewFileInfo returns a new FileInfo structure.

-func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo {

+func (sp *SourceSpec) NewFileInfoFrom(path, filename string) (*FileInfo, error) {

+	meta := hugofs.FileMeta{

+		"filename": filename,

+		"path":     path,

+	}

-	var lang, translationBaseName, relPath string

+	return sp.NewFileInfo(hugofs.NewFileMetaInfo(nil, meta))

+}

-	if fp, ok := fi.(hugofs.FilePather); ok {

-		filename = fp.Filename()

-		baseDir = fp.BaseDir()

-		relPath = fp.Path()

-	}

+func (sp *SourceSpec) NewFileInfo(fi hugofs.FileMetaInfo) (*FileInfo, error) {

-	if fl, ok := fi.(hugofs.LanguageAnnouncer); ok {

-		lang = fl.Lang()

-		translationBaseName = fl.TranslationBaseName()

+	m := fi.Meta()

+

+	filename := m.Filename()

+	relPath := m.Path()

+	isLeafBundle := m.Classifier() == files.ContentClassLeaf

+

+	if relPath == "" || strings.Contains(relPath, "TODO") {

+		return nil, errors.Errorf("no Path provided by %v (%T)", m, m.Fs())

 	}

-	dir, name := filepath.Split(filename)

-	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {

-		dir = dir + helpers.FilePathSeparator

+	if filename == "" || strings.Contains(filename, "TODO") {

+		return nil, errors.Errorf("no Filename provided by %v (%T)", m, m.Fs())

 	}

-	baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator)

-

-	relDir := ""

-	if dir != baseDir {

-		relDir = strings.TrimPrefix(dir, baseDir)

+	relDir := filepath.Dir(relPath)

+	if relDir == "." {

+		relDir = ""

 	}

+	if !strings.HasSuffix(relDir, helpers.FilePathSeparator) {

+		relDir = relDir + helpers.FilePathSeparator

+	}

-	relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator)

+	lang := m.Lang()

+	translationBaseName := m.GetString("translationBaseName")

-	if relPath == "" {

-		relPath = filepath.Join(relDir, name)

+	dir, name := filepath.Split(relPath)

+	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {

+		dir = dir + helpers.FilePathSeparator

 	}

 	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))

@@ -277,14 +281,14 @@

 		lang:                lang,

 		ext:                 ext,

 		dir:                 dir,

-		relDir:              relDir,

-		relPath:             relPath,

+		relDir:              relDir,  // Dir()

+		relPath:             relPath, // Path()

 		name:                name,

-		baseName:            baseName,

+		baseName:            baseName, // BaseFileName()

 		translationBaseName: translationBaseName,

 		isLeafBundle:        isLeafBundle,

 	}

-	return f

+	return f, nil

 }

--- a/source/fileInfo_test.go

+++ b/source/fileInfo_test.go

@@ -15,12 +15,9 @@

 import (

 	"path/filepath"

+	"strings"

 	"testing"

-	"github.com/gohugoio/hugo/helpers"

-

-	"github.com/gohugoio/hugo/hugofs"

-	"github.com/spf13/afero"

 	"github.com/stretchr/testify/require"

 )

@@ -55,56 +52,10 @@

 		}},

 	} {

-		f := s.NewFileInfo(this.base, this.filename, false, nil)

+		path := strings.TrimPrefix(this.filename, this.base)

+		f, err := s.NewFileInfoFrom(path, this.filename)

+		assert.NoError(err)

 		this.assert(f)

 	}

-}

-

-func TestFileInfoLanguage(t *testing.T) {

-	assert := require.New(t)

-	langs := map[string]bool{

-		"sv": true,

-		"en": true,

-	}

-

-	m := afero.NewMemMapFs()

-	lfs := hugofs.NewLanguageFs("sv", langs, m)

-	v := newTestConfig()

-

-	fs := hugofs.NewFrom(m, v)

-

-	ps, err := helpers.NewPathSpec(fs, v)

-	assert.NoError(err)

-	s := SourceSpec{SourceFs: lfs, PathSpec: ps}

-	s.Languages = map[string]interface{}{

-		"en": true,

-	}

-

-	err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777)

-	assert.NoError(err)

-	err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777)

-	assert.NoError(err)

-

-	sv, _ := lfs.Stat("page.md")

-	en, _ := lfs.Stat("page.en.md")

-

-	fiSv := s.NewFileInfo("", "page.md", false, sv)

-	fiEn := s.NewFileInfo("", "page.en.md", false, en)

-

-	assert.Equal("sv", fiSv.Lang())

-	assert.Equal("en", fiEn.Lang())

-

-	// test contentBaseName implementation

-	fi := s.NewFileInfo("", "2018-10-01-contentbasename.md", false, nil)

-	assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())

-

-	fi = s.NewFileInfo("", "2018-10-01-contentbasename.en.md", false, nil)

-	assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())

-

-	fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "index.en.md"), true, nil)

-	assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())

-

-	fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "_index.en.md"), false, nil)

-	assert.Equal("_index", fi.ContentBaseName())

 }

--- a/source/filesystem.go

+++ b/source/filesystem.go

@@ -14,106 +14,100 @@

 package source

 import (

-	"os"

 	"path/filepath"

-	"runtime"

 	"sync"

-	"github.com/gohugoio/hugo/helpers"

-	jww "github.com/spf13/jwalterweatherman"

-	"golang.org/x/text/unicode/norm"

+	"github.com/pkg/errors"

+

+	"github.com/gohugoio/hugo/hugofs"

 )

 // Filesystem represents a source filesystem.

 type Filesystem struct {

-	files     []ReadableFile

-	filesInit sync.Once

+	files        []File

+	filesInit    sync.Once

+	filesInitErr error

 	Base string

+	fi hugofs.FileMetaInfo

+

 	SourceSpec

 }

-// Input describes a source input.

-type Input interface {

-	Files() []ReadableFile

-}

-

 // NewFilesystem returns a new filesytem for a given source spec.

 func (sp SourceSpec) NewFilesystem(base string) *Filesystem {

 	return &Filesystem{SourceSpec: sp, Base: base}

 }

+func (sp SourceSpec) NewFilesystemFromFileMetaInfo(fi hugofs.FileMetaInfo) *Filesystem {

+	return &Filesystem{SourceSpec: sp, fi: fi}

+}

+

 // Files returns a slice of readable files.

-func (f *Filesystem) Files() []ReadableFile {

+func (f *Filesystem) Files() ([]File, error) {

 	f.filesInit.Do(func() {

-		f.captureFiles()

+		err := f.captureFiles()

+		if err != nil {

+			f.filesInitErr = errors.Wrap(err, "capture files")

+		}

 	})

-	return f.files

+	return f.files, f.filesInitErr

 }

 // add populates a file in the Filesystem.files

-func (f *Filesystem) add(name string, fi os.FileInfo) (err error) {

-	var file ReadableFile

+func (f *Filesystem) add(name string, fi hugofs.FileMetaInfo) (err error) {

+	var file File

-	if runtime.GOOS == "darwin" {

-		// When a file system is HFS+, its filepath is in NFD form.

-		name = norm.NFC.String(name)

+	file, err = f.SourceSpec.NewFileInfo(fi)

+	if err != nil {

+		return err

 	}

-	file = f.SourceSpec.NewFileInfo(f.Base, name, false, fi)

 	f.files = append(f.files, file)

 	return err

 }

-func (f *Filesystem) captureFiles() {

-	walker := func(filePath string, fi os.FileInfo, err error) error {

+func (f *Filesystem) captureFiles() error {

+	walker := func(path string, fi hugofs.FileMetaInfo, err error) error {

 		if err != nil {

+			return err

+		}

+

+		if fi.IsDir() {

 			return nil

 		}

-		b, err := f.shouldRead(filePath, fi)

+		meta := fi.Meta()

+		filename := meta.Filename()

+

+		b, err := f.shouldRead(filename, fi)

 		if err != nil {

 			return err

 		}

+

 		if b {

-			f.add(filePath, fi)

+			err = f.add(filename, fi)

 		}

+

 		return err

 	}

-	if f.SourceFs == nil {

-		panic("Must have a fs")

-	}

-	err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker)

+	w := hugofs.NewWalkway(hugofs.WalkwayConfig{

+		Fs:     f.SourceFs,

+		Info:   f.fi,

+		Root:   f.Base,

+		WalkFn: walker,

+	})

-	if err != nil {

-		jww.ERROR.Println(err)

-	}

+	return w.Walk()

 }

-func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {

-	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {

-		link, err := filepath.EvalSymlinks(filename)

-		if err != nil {

-			jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)

-			return false, nil

-		}

-		linkfi, err := f.SourceFs.Stat(link)

-		if err != nil {

-			jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)

-			return false, nil

-		}

+func (f *Filesystem) shouldRead(filename string, fi hugofs.FileMetaInfo) (bool, error) {

-		if !linkfi.Mode().IsRegular() {

-			jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filename)

-		}

-		return false, nil

-	}

-

-	ignore := f.SourceSpec.IgnoreFile(filename)

+	ignore := f.SourceSpec.IgnoreFile(fi.Meta().Filename())

 	if fi.IsDir() {

 		if ignore {

--- a/source/filesystem_test.go

+++ b/source/filesystem_test.go

@@ -14,20 +14,31 @@

 package source

 import (

-	"os"

+	"fmt"

+	"path/filepath"

 	"runtime"

 	"testing"

+	"github.com/gohugoio/hugo/modules"

+

+	"github.com/gohugoio/hugo/langs"

+

+	"github.com/spf13/afero"

+

 	"github.com/gohugoio/hugo/helpers"

 	"github.com/gohugoio/hugo/hugofs"

+	"github.com/stretchr/testify/require"

 	"github.com/spf13/viper"

 )

 func TestEmptySourceFilesystem(t *testing.T) {

+	assert := require.New(t)

 	ss := newTestSourceSpec()

-	src := ss.NewFilesystem("Empty")

-	if len(src.Files()) != 0 {

+	src := ss.NewFilesystem("")

+	files, err := src.Files()

+	assert.NoError(err)

+	if len(files) != 0 {

 		t.Errorf("new filesystem should contain 0 files.")

 	}

 }

@@ -38,6 +49,8 @@

 		return

 	}

+	assert := require.New(t)

+

 	paths := []struct {

 		NFC string

 		NFD string

@@ -47,14 +60,18 @@

 	}

 	ss := newTestSourceSpec()

-	var fi os.FileInfo

+	fi := hugofs.NewFileMetaInfo(nil, hugofs.FileMeta{})

-	for _, path := range paths {

-		src := ss.NewFilesystem("base")

+	for i, path := range paths {

+		base := fmt.Sprintf("base%d", i)

+		assert.NoError(afero.WriteFile(ss.Fs.Source, filepath.Join(base, path.NFD), []byte("some data"), 0777))

+		src := ss.NewFilesystem(base)

 		_ = src.add(path.NFD, fi)

-		f := src.Files()[0]

+		files, err := src.Files()

+		assert.NoError(err)

+		f := files[0]

 		if f.BaseFileName() != path.NFC {

-			t.Fatalf("file name in NFD form should be normalized (%s)", path.NFC)

+			t.Fatalf("file %q name in NFD form should be normalized (%s)", f.BaseFileName(), path.NFC)

 		}

 	}

@@ -70,12 +87,22 @@

 	v.Set("resourceDir", "resources")

 	v.Set("publishDir", "public")

 	v.Set("assetDir", "assets")

+	_, err := langs.LoadLanguageSettings(v, nil)

+	if err != nil {

+		panic(err)

+	}

+	mod, err := modules.CreateProjectModule(v)

+	if err != nil {

+		panic(err)

+	}

+	v.Set("allModules", modules.Modules{mod})

+

 	return v

 }

 func newTestSourceSpec() *SourceSpec {

 	v := newTestConfig()

-	fs := hugofs.NewMem(v)

+	fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(afero.NewMemMapFs()), v)

 	ps, err := helpers.NewPathSpec(fs, v)

 	if err != nil {

 		panic(err)

--- a/source/sourceSpec.go

+++ b/source/sourceSpec.go

@@ -17,6 +17,7 @@

 	"os"

 	"path/filepath"

 	"regexp"

+	"runtime"

 	"github.com/gohugoio/hugo/langs"

 	"github.com/spf13/afero"

@@ -104,6 +105,18 @@

 	for _, re := range s.ignoreFilesRe {

 		if re.MatchString(filename) {

 			return true

+		}

+	}

+

+	if runtime.GOOS == "windows" {

+		// Also check the forward slash variant if different.

+		unixFilename := filepath.ToSlash(filename)

+		if unixFilename != filename {

+			for _, re := range s.ignoreFilesRe {

+				if re.MatchString(unixFilename) {

+					return true

+				}

+			}

 		}

 	}

--- a/tpl/cast/docshelper.go

+++ b/tpl/cast/docshelper.go

@@ -17,7 +17,7 @@

 	"github.com/gohugoio/hugo/common/loggers"

 	"github.com/gohugoio/hugo/deps"

 	"github.com/gohugoio/hugo/docshelper"

-	"github.com/gohugoio/hugo/htesting"

+	"github.com/gohugoio/hugo/resources/page"

 	"github.com/gohugoio/hugo/tpl/internal"

 	"github.com/spf13/viper"

 )

@@ -30,7 +30,7 @@

 			Cfg:                 viper.New(),

 			Log:                 loggers.NewErrorLogger(),

 			BuildStartListeners: &deps.Listeners{},

-			Site:                htesting.NewTestHugoSite(),

+			Site:                page.NewDummyHugoSite(newTestConfig()),

 		}

 		var namespaces internal.TemplateFuncsNamespaces

@@ -46,4 +46,10 @@

 	}

 	docshelper.AddDocProvider("tpl", docsProvider)

+}

+

+func newTestConfig() *viper.Viper {

+	v := viper.New()

+	v.Set("contentDir", "content")

+	return v

 }

--- a/tpl/data/init_test.go

+++ b/tpl/data/init_test.go

@@ -16,6 +16,7 @@

 import (

 	"testing"

+	"github.com/gohugoio/hugo/langs"

 	"github.com/gohugoio/hugo/tpl/internal"

 	"github.com/spf13/viper"

 	"github.com/stretchr/testify/require"

@@ -27,6 +28,7 @@

 	v := viper.New()

 	v.Set("contentDir", "content")

+	langs.LoadLanguageSettings(v, nil)

 	for _, nsf := range internal.TemplateFuncsNamespaceRegistry {

 		ns = nsf(newDeps(v))

--- a/tpl/data/resources_test.go

+++ b/tpl/data/resources_test.go

@@ -23,6 +23,8 @@

 	"testing"

 	"time"

+	"github.com/gohugoio/hugo/modules"

+

 	"github.com/gohugoio/hugo/helpers"

 	"github.com/gohugoio/hugo/cache/filecache"

@@ -87,7 +89,7 @@

 func TestScpGetRemote(t *testing.T) {

 	t.Parallel()

 	fs := new(afero.MemMapFs)

-	cache := filecache.NewCache(fs, 100)

+	cache := filecache.NewCache(fs, 100, "")

 	tests := []struct {

 		path    string

@@ -186,14 +188,19 @@

 	cfg.Set("layoutDir", "layouts")

 	cfg.Set("archetypeDir", "archetypes")

-	l := langs.NewLanguage("en", cfg)

-	l.Set("i18nDir", "i18n")

-	cs, err := helpers.NewContentSpec(l)

+	langs.LoadLanguageSettings(cfg, nil)

+	mod, err := modules.CreateProjectModule(cfg)

 	if err != nil {

 		panic(err)

 	}

+	cfg.Set("allModules", modules.Modules{mod})

-	fs := hugofs.NewMem(l)

+	cs, err := helpers.NewContentSpec(cfg)

+	if err != nil {

+		panic(err)

+	}

+

+	fs := hugofs.NewMem(cfg)

 	logger := loggers.NewErrorLogger()

 	p, err := helpers.NewPathSpec(fs, cfg)

--- a/tpl/hugo/init_test.go

+++ b/tpl/hugo/init_test.go

@@ -16,10 +16,10 @@

 import (

 	"testing"

-	"github.com/gohugoio/hugo/htesting"

-

 	"github.com/gohugoio/hugo/deps"

+	"github.com/gohugoio/hugo/resources/page"

 	"github.com/gohugoio/hugo/tpl/internal"

+	"github.com/spf13/viper"

 	"github.com/stretchr/testify/require"

 )

@@ -26,7 +26,9 @@

 func TestInit(t *testing.T) {

 	var found bool

 	var ns *internal.TemplateFuncsNamespace

-	s := htesting.NewTestHugoSite()

+	v := viper.New()

+	v.Set("contentDir", "content")

+	s := page.NewDummyHugoSite(v)

 	for _, nsf := range internal.TemplateFuncsNamespaceRegistry {

 		ns = nsf(&deps.Deps{Site: s})

--- a/tpl/os/os.go

+++ b/tpl/os/os.go

@@ -18,6 +18,7 @@

 import (

 	"errors"

 	"fmt"

+	"os"

 	_os "os"

 	"github.com/gohugoio/hugo/deps"

@@ -26,23 +27,20 @@

 )

 // New returns a new instance of the os-namespaced template functions.

-func New(deps *deps.Deps) *Namespace {

+func New(d *deps.Deps) *Namespace {

-	// Since Hugo 0.38 we can have multiple content dirs. This can make it hard to

-	// reason about where the file is placed relative to the project root.

-	// To make the {{ readFile .Filename }} variant just work, we create a composite

-	// filesystem that first checks the work dir fs and then the content fs.

 	var rfs afero.Fs

-	if deps.Fs != nil {

-		rfs = deps.Fs.WorkingDir

-		if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil {

-			rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.Content.Fs, deps.Fs.WorkingDir))

+	if d.Fs != nil {

+		rfs = d.Fs.WorkingDir

+		if d.PathSpec != nil && d.PathSpec.BaseFs != nil {

+			rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(d.PathSpec.BaseFs.Content.Fs, d.Fs.WorkingDir))

 		}

+

 	}

 	return &Namespace{

 		readFileFs: rfs,

-		deps:       deps,

+		deps:       d,

 	}

 }

@@ -76,6 +74,9 @@

 			return "", fmt.Errorf("file %q is too big", filename)

 		}

 	} else {

+		if os.IsNotExist(err) {

+			return "", fmt.Errorf("file %q does not exist", filename)

+		}

 		return "", err

 	}

 	b, err := afero.ReadFile(fs, filename)

@@ -94,6 +95,10 @@

 	s, err := cast.ToStringE(i)

 	if err != nil {

 		return "", err

+	}

+

+	if ns.deps.PathSpec != nil {

+		s = ns.deps.PathSpec.RelPathify(s)

 	}

 	return readFile(ns.readFileFs, s)

--- a/tpl/site/init_test.go

+++ b/tpl/site/init_test.go

@@ -16,8 +16,10 @@

 import (

 	"testing"

+	"github.com/spf13/viper"

+

 	"github.com/gohugoio/hugo/deps"

-	"github.com/gohugoio/hugo/htesting"

+	"github.com/gohugoio/hugo/resources/page"

 	"github.com/gohugoio/hugo/tpl/internal"

 	"github.com/stretchr/testify/require"

 )

@@ -25,7 +27,9 @@

 func TestInit(t *testing.T) {

 	var found bool

 	var ns *internal.TemplateFuncsNamespace

-	s := htesting.NewTestHugoSite()

+	v := viper.New()

+	v.Set("contentDir", "content")

+	s := page.NewDummyHugoSite(v)

 	for _, nsf := range internal.TemplateFuncsNamespaceRegistry {

 		ns = nsf(&deps.Deps{Site: s})

--- a/tpl/template.go

+++ b/tpl/template.go

@@ -252,12 +252,15 @@

 	if err != nil {

 		return nil, "", err

 	}

-	f, err := fs.Open(filename)

+	fim := fi.(hugofs.FileMetaInfo)

+	meta := fim.Meta()

+

+	f, err := meta.Open()

 	if err != nil {

 		return nil, "", errors.Wrapf(err, "failed to open template file %q:", filename)

 	}

-	return f, fi.(hugofs.RealFilenameInfo).RealFilename(), nil

+	return f, meta.Filename(), nil

 }

 // ExecuteToString executes the current template and returns the result as a

--- a/tpl/tplimpl/embedded/templates.autogen.go

+++ b/tpl/tplimpl/embedded/templates.autogen.go

@@ -268,8 +268,7 @@

     </li>

     {{ end }}

 </ul>

-{{ end }}

-`},

+{{ end }}`},

 	{`schema.html`, `<meta itemprop="name" content="{{ .Title }}">

 <meta itemprop="description" content="{{ with .Description }}{{ . }}{{ else }}{{if .IsPage}}{{ .Summary }}{{ else }}{{ with .Site.Params.description }}{{ . }}{{ end }}{{ end }}{{ end }}">

--- a/tpl/tplimpl/template.go

+++ b/tpl/tplimpl/template.go

@@ -711,7 +711,7 @@

 func (t *templateHandler) loadTemplates(prefix string) error {

-	walker := func(path string, fi os.FileInfo, err error) error {

+	walker := func(path string, fi hugofs.FileMetaInfo, err error) error {

 		if err != nil || fi.IsDir() {

 			return err

 		}

@@ -928,8 +928,8 @@

 		realFilename := filename

 		if fi, err := fs.Stat(filename); err == nil {

-			if fir, ok := fi.(hugofs.RealFilenameInfo); ok {

-				realFilename = fir.RealFilename()

+			if fim, ok := fi.(hugofs.FileMetaInfo); ok {

+				realFilename = fim.Meta().Filename()

 			}

 		}

--- a/tpl/tplimpl/template_funcs_test.go

+++ b/tpl/tplimpl/template_funcs_test.go

@@ -21,8 +21,10 @@

 	"testing"

 	"time"

-	"github.com/gohugoio/hugo/htesting"

+	"github.com/gohugoio/hugo/modules"

+	"github.com/gohugoio/hugo/resources/page"

+

 	"github.com/gohugoio/hugo/common/hugo"

 	"github.com/gohugoio/hugo/common/loggers"

 	"github.com/gohugoio/hugo/config"

@@ -52,6 +54,14 @@

 	v.Set("assetDir", "assets")

 	v.Set("resourceDir", "resources")

 	v.Set("publishDir", "public")

+

+	langs.LoadLanguageSettings(v, nil)

+	mod, err := modules.CreateProjectModule(v)

+	if err != nil {

+		panic(err)

+	}

+	v.Set("allModules", modules.Modules{mod})

+

 	return v

 }

@@ -59,7 +69,7 @@

 	l := langs.NewLanguage("en", cfg)

 	return deps.DepsCfg{

 		Language:            l,

-		Site:                htesting.NewTestHugoSite(),

+		Site:                page.NewDummyHugoSite(cfg),

 		Cfg:                 cfg,

 		Fs:                  hugofs.NewMem(l),

 		Logger:              logger,

--- a/transform/livereloadinject/livereloadinject_test.go

+++ b/transform/livereloadinject/livereloadinject_test.go

@@ -34,7 +34,7 @@

 	tr := transform.New(New(1313))

 	tr.Apply(out, in)

-	expected := fmt.Sprintf(`<script data-no-instant>document.write('<script src="/livereload.js?port=1313&mindelay=10"></' + 'script>')</script>%s`, bodyEndTag)

+	expected := fmt.Sprintf(`<script data-no-instant>document.write('<script src="/livereload.js?port=1313&mindelay=10&v=2"></' + 'script>')</script>%s`, bodyEndTag)

 	if out.String() != expected {

 		t.Errorf("Expected %s got %s", expected, out.String())

 	}