ref: 0c98d8e9ed7013ce722d53ee094f1adc79b90079
parent: 97eb55da8977f764560b849d1d05af7fbfa91526
author: Tatsushi Demachi <[email protected]>
date: Sun Aug 16 08:30:22 EDT 2015
Use LazyFileReader for reading file contents Simple ioutil.ReadFile is used for reading file contents but it reads all of the file contents and copies them into the memory and is run in a single goroutine. It causes much memory consumption at copying media files in content directory to publish directory and it is not good at performance. This improves the both issue by replacing ReadFile with LazyFileReader. It postpones reading the file contents until it is really needed. As the result, actual file read is run in parallelized goroutine. It improves performance especially in a really big site. In addition, if this reader is called from io.Copy, it does not copy the file contents into the memory but just copies them into destination file. It improves much memory consumption issue when the site has many media files. Fix #1181
--- a/source/filesystem.go
+++ b/source/filesystem.go
@@ -14,10 +14,8 @@
package source
import (
- "bytes"
"github.com/spf13/viper"
"io"
- "io/ioutil"
"os"
"path/filepath"
"regexp"
@@ -114,11 +112,11 @@
if isNonProcessablePath(filePath) {return nil
}
- data, err := ioutil.ReadFile(filePath)
+ rd, err := NewLazyFileReader(filePath)
if err != nil {return err
}
- f.add(filePath, bytes.NewBuffer(data))
+ f.add(filePath, rd)
return nil
}