ref: 74c90553b42284eef503cdb9f81d5dc097d9d2cb
parent: 7e196a82944148ed3f78f334303b452ab2bd4078
author: Steve Francia <[email protected]>
date: Wed Jan 13 06:42:43 EST 2016
Static file incremental sync improvements in tandem with Afero improvements
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -42,6 +42,7 @@
"github.com/spf13/nitro"
"github.com/spf13/viper"
"gopkg.in/fsnotify.v1"
+ "github.com/spf13/afero"
)
var mainSite *hugolib.Site
@@ -434,6 +435,46 @@
return nil
}
+func getStaticSourceFs() afero.Fs {+ source := hugofs.SourceFs
+ themeDir, err := helpers.GetThemeStaticDirPath()
+ staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator
+
+ useTheme := true
+ useStatic := true
+
+ if err != nil {+ jww.WARN.Println(err)
+ useTheme = false
+ } else {+ if _, err := source.Stat(themeDir); os.IsNotExist(err) {+ jww.WARN.Println("Unable to find Theme Static Directory:", themeDir)+ useTheme = false
+ }
+ }
+
+ if _, err := source.Stat(staticDir); os.IsNotExist(err) {+ jww.WARN.Println("Unable to find Static Directory:", staticDir)+ useStatic = false
+ }
+
+ if !useStatic && !useTheme {+ return nil
+ }
+
+ if !useStatic {+ return afero.NewReadOnlyFs(afero.NewBasePathFs(source, themeDir))
+ }
+
+ if !useTheme {+ return afero.NewReadOnlyFs(afero.NewBasePathFs(source, staticDir))
+ }
+
+ base := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, themeDir))
+ overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, staticDir))
+ return afero.NewCopyOnWriteFs(base, overlay)
+}
+
func copyStatic() error { publishDir := helpers.AbsPathify(viper.GetString("PublishDir")) + helpers.FilePathSeparator@@ -442,31 +483,51 @@
publishDir = helpers.FilePathSeparator
}
+ staticSourceFs := getStaticSourceFs()
+
+ if staticSourceFs == nil {+ jww.WARN.Println("No static directories found to sync")+ return nil
+ }
+
syncer := fsync.NewSyncer()
syncer.NoTimes = viper.GetBool("notimes")- syncer.SrcFs = hugofs.SourceFs
+ syncer.SrcFs = staticSourceFs
syncer.DestFs = hugofs.DestinationFS
+ // Now that we are using a unionFs for the static directories
+ // We can effectively clean the publishDir on initial sync
+ syncer.Delete = true
+ jww.INFO.Println("syncing static files to", publishDir)- themeDir, err := helpers.GetThemeStaticDirPath()
- if err != nil {- jww.WARN.Println(err)
- }
-
- // Copy the theme's static directory
- if themeDir != "" {- jww.INFO.Println("syncing from", themeDir, "to", publishDir)- utils.CheckErr(syncer.Sync(publishDir, themeDir), fmt.Sprintf("Error copying static files of theme to %s", publishDir))- }
-
- // Copy the site's own static directory
- staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator
- if _, err := os.Stat(staticDir); err == nil {- jww.INFO.Println("syncing from", staticDir, "to", publishDir)- return syncer.Sync(publishDir, staticDir)
- } else if os.IsNotExist(err) {- jww.WARN.Println("Unable to find Static Directory:", staticDir)- }
+ // because we are using a baseFs (to get the union right). Sync from the root
+ syncer.Sync(publishDir, helpers.FilePathSeparator)
return nil
+//
+// themeDir, err := helpers.GetThemeStaticDirPath()
+// if err != nil {+// jww.WARN.Println(err)
+// }
+//
+// staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator
+// if _, err := os.Stat(staticDir); os.IsNotExist(err) {+// jww.WARN.Println("Unable to find Static Directory:", staticDir)+// }
+//
+// // Copy the theme's static directory
+// if themeDir != "" {+// jww.INFO.Println("syncing from", themeDir, "to", publishDir)+// utils.CheckErr(syncer.Sync(publishDir, themeDir), fmt.Sprintf("Error copying static files of theme to %s", publishDir))+// }
+//
+// // Copy the site's own static directory
+// staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator
+// if _, err := os.Stat(staticDir); err == nil {+// jww.INFO.Println("syncing from", staticDir, "to", publishDir)+// return syncer.Sync(publishDir, staticDir)
+// } else if os.IsNotExist(err) {+// jww.WARN.Println("Unable to find Static Directory:", staticDir)+// }
+// return nil
}
// getDirList provides NewWatcher() with a list of directories to watch for changes.
@@ -597,6 +658,11 @@
continue
}
+ // Sometimes during rm -rf operations a '"": REMOVE' is triggered. Just ignore these
+ if ev.Name == "" {+ continue
+ }
+
// Write and rename operations are often followed by CHMOD.
// There may be valid use cases for rebuilding the site on CHMOD,
// but that will require more complex logic than this simple conditional.
@@ -609,13 +675,22 @@
continue
}
- // add new directory to watch list
- if s, err := os.Stat(ev.Name); err == nil && s.Mode().IsDir() {- if ev.Op&fsnotify.Create == fsnotify.Create {- watcher.Add(ev.Name)
+ walkAdder := func (path string, f os.FileInfo, err error) error {+ if f.IsDir() {+ jww.FEEDBACK.Println("adding created directory to watchlist", path)+ watcher.Add(path)
}
+ return nil
}
+ // recursively add new directories to watch list
+ // When mkdir -p is used, only the top directory triggers an event (at least on OSX)
+ if ev.Op&fsnotify.Create == fsnotify.Create {+ if s, err := hugofs.SourceFs.Stat(ev.Name); err == nil && s.Mode().IsDir() {+ afero.Walk(hugofs.SourceFs, ev.Name, walkAdder)
+ }
+ }
+
isstatic := strings.HasPrefix(ev.Name, helpers.GetStaticDirPath()) || (len(helpers.GetThemesDirPath()) > 0 && strings.HasPrefix(ev.Name, helpers.GetThemesDirPath()))
if isstatic {@@ -627,7 +702,18 @@
}
if len(staticEvents) > 0 {- jww.FEEDBACK.Printf("Static file changed, syncing\n")+ publishDir := helpers.AbsPathify(viper.GetString("PublishDir")) + helpers.FilePathSeparator+
+ // If root, remove the second '/'
+ if publishDir == "//" {+ publishDir = helpers.FilePathSeparator
+ }
+
+ jww.FEEDBACK.Println("\n Static file changes detected")+ jww.FEEDBACK.Println("syncing to", publishDir)+ const layout = "2006-01-02 15:04 -0700"
+ fmt.Println(time.Now().Format(layout))
+
if viper.GetBool("ForceSyncStatic") { jww.FEEDBACK.Printf("Syncing all static files\n")err := copyStatic()
@@ -636,26 +722,38 @@
utils.StopOnErr(err, fmt.Sprintf("Error copying static files to %s", helpers.AbsPathify(viper.GetString("PublishDir"))))}
} else {+ staticSourceFs := getStaticSourceFs()
+
+ if staticSourceFs == nil {+ jww.WARN.Println("No static directories found to sync")+ return
+ }
+
syncer := fsync.NewSyncer()
syncer.NoTimes = viper.GetBool("notimes")- syncer.SrcFs = hugofs.SourceFs
+ syncer.SrcFs = staticSourceFs
syncer.DestFs = hugofs.DestinationFS
- publishDir := helpers.AbsPathify(viper.GetString("PublishDir")) + helpers.FilePathSeparator- if publishDir == "//" || publishDir == helpers.FilePathSeparator {- publishDir = ""
- }
-
- staticDir := helpers.GetStaticDirPath()
- themeStaticDir := helpers.GetThemesDirPath()
-
- jww.FEEDBACK.Printf("Syncing from: \n \tStaticDir: '%s'\n\tThemeStaticDir: '%s'\n", staticDir, themeStaticDir)-
for _, ev := range staticEvents {+ // Due to our approach of layering both directories and the content's rendered output
+ // into one we can't accurately remove a file not in one of the source directories.
+ // If a file is in the local static dir and also in the theme static dir and we remove
+ // it from one of those locations we expect it to still exist in the destination
+ // If a file is generated by the content over a static file we expect it to remain as well.
+ // Because we are never certain if the file was overwritten by the content generation
+ // We can't effectively remove anything.
+ //
+ // This leads to two approaches:
+ // 1. Not overwrite anything
+ // 2. Assume these cases are rare and overwrite anyway. If things get out of sync
+ // a clean sync will be needed.
+ // There is an alternative which is quite heavy. We would have to track every single file
+ // placed into the publishedPath and which pipeline put it there.
+ // We have chosen to take the 2nd approach
fmt.Println(ev)
+
fromPath := ev.Name
- var publishPath string
// If we are here we already know the event took place in a static dir
relPath, err := helpers.MakeStaticPathRelative(fromPath)
@@ -663,28 +761,42 @@
fmt.Println(err)
continue
}
+ fmt.Println("relpath", relPath)- if strings.HasPrefix(fromPath, staticDir) {- publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, staticDir))
- } else if strings.HasPrefix(relPath, themeStaticDir) {- publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, themeStaticDir))
- }
- jww.FEEDBACK.Println("Syncing file", relPath)- // Due to our approach of layering many directories onto one we can't accurately
- // remove file not in one of the source directories.
- // If a file is in the local static dir and also in the theme static dir and we remove
- // it from one of those locations we expect it to still exist in the destination
-
- // if remove or rename ignore
+ // if remove or rename ignore.. as in leave the old file in the publishDir
if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {+ // What about the case where a file in the theme is moved so the local static file can
+ // take it's place.
+ if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) {+ // If file doesn't exist in any static dir, remove it
+ toRemove :=filepath.Join(publishDir, relPath)
+ jww.FEEDBACK.Println("File no longer exists in static dir, removing", toRemove)+ hugofs.DestinationFS.Remove(toRemove)
+ } else if err == nil {+ // If file still exists, sync it
+ jww.FEEDBACK.Println("Syncing", relPath, "to", publishDir)+ syncer.Sync(filepath.Join(publishDir, relPath), relPath)
+ } else {+ jww.ERROR.Println(err)
+ }
+
continue
}
- jww.INFO.Println("syncing from ", fromPath, " to ", publishPath)- if er := syncer.Sync(publishPath, fromPath); er != nil {- jww.ERROR.Printf("Error on syncing file '%s'\n %s\n", relPath, er)- }
+// if strings.HasPrefix(fromPath, staticDir) {+// publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, staticDir))
+// } else if strings.HasPrefix(relPath, themeStaticDir) {+// publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, themeStaticDir))
+// }
+ jww.FEEDBACK.Println("Syncing", relPath, "to", publishDir)+ syncer.Sync(filepath.Join(publishDir, relPath), relPath)
+
+
+// jww.INFO.Println("syncing from ", fromPath, " to ", publishPath)+// if er := syncer.Sync(publishPath, fromPath); er != nil {+// jww.ERROR.Printf("Error on syncing file '%s'\n %s\n", relPath, er)+// }
}
}
@@ -692,7 +804,7 @@
// Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized
// force refresh when more than one file
- if len(staticEvents) == 1 {+ if len(staticEvents) > 0 { for _, ev := range staticEvents {path, _ := helpers.MakeStaticPathRelative(ev.Name)
livereload.RefreshPath(path)
@@ -704,7 +816,7 @@
}
}
- if len(dynamicEvents) >0 {+ if len(dynamicEvents) > 0 { fmt.Print("\nChange detected, rebuilding site\n")const layout = "2006-01-02 15:04 -0700"
fmt.Println(time.Now().Format(layout))