shithub: hugo

Info  •  Files  •  Log  •  Branches

Add HUGO_NUMWORKERMULTIPLIER

And use that to calculate number of workers, if set, else fall back to number of logical CPUs.

Also tweak the relevant related settings to match the new setup.

Also remove the setting of `runtime.GOMAXPROCS` as this has been the default behaviour since Go 1.5.

Fixes #5814

--- /dev/null

+++ b/config/env.go

@@ -1,0 +1,33 @@

+// Copyright 2019 The Hugo Authors. All rights reserved.

+//

+// Licensed under the Apache License, Version 2.0 (the "License");

+// you may not use this file except in compliance with the License.

+// You may obtain a copy of the License at

+// http://www.apache.org/licenses/LICENSE-2.0

+//

+// Unless required by applicable law or agreed to in writing, software

+// distributed under the License is distributed on an "AS IS" BASIS,

+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+// See the License for the specific language governing permissions and

+// limitations under the License.

+

+package config

+

+import (

+	"os"

+	"runtime"

+	"strconv"

+)

+

+// GetNumWorkerMultiplier returns the base value used to calculate the number

+// of workers to use for Hugo's parallel execution.

+// It returns the value in HUGO_NUMWORKERMULTIPLIER OS env variable if set to a

+// positive integer, else the number of logical CPUs.

+func GetNumWorkerMultiplier() int {

+	if gmp := os.Getenv("HUGO_NUMWORKERMULTIPLIER"); gmp != "" {

+		if p, err := strconv.Atoi(gmp); err == nil && p > 0 {

+			return p

+		}

+	}

+	return runtime.NumCPU()

+}

--- a/docs/content/en/getting-started/configuration.md

+++ b/docs/content/en/getting-started/configuration.md

@@ -297,6 +297,11 @@

 ```

 {{% /note %}}

+## Configuration Environment Variables

+

+HUGO_NUMWORKERMULTIPLIER

+: Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used.

+

 ## Configuration Lookup Order

 Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior:

--- a/hugolib/pagebundler.go

+++ b/hugolib/pagebundler.go

@@ -18,8 +18,9 @@

 	"fmt"

 	"math"

 	"path/filepath"

-	"runtime"

+	"github.com/gohugoio/hugo/config"

+

 	_errors "github.com/pkg/errors"

 	"golang.org/x/sync/errgroup"

@@ -73,10 +74,7 @@

 }

 func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor {

-	numWorkers := 12

-	if n := runtime.NumCPU() * 3; n > numWorkers {

-		numWorkers = n

-	}

+	numWorkers := config.GetNumWorkerMultiplier() * 3

 	numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites))))

--- a/hugolib/pagebundler_capture.go

+++ b/hugolib/pagebundler_capture.go

@@ -19,8 +19,9 @@

 	"os"

 	"path"

 	"path/filepath"

-	"runtime"

+	"github.com/gohugoio/hugo/config"

+

 	"github.com/gohugoio/hugo/common/loggers"

 	_errors "github.com/pkg/errors"

@@ -70,10 +71,7 @@

 	contentChanges *contentChangeMap,

 	filenames ...string) *capturer {

-	numWorkers := 4

-	if n := runtime.NumCPU(); n > numWorkers {

-		numWorkers = n

-	}

+	numWorkers := config.GetNumWorkerMultiplier()

 	// TODO(bep) the "index" vs "_index" check/strings should be moved in one place.

 	isBundleHeader := func(filename string) bool {

--- a/hugolib/site.go

+++ b/hugolib/site.go

@@ -1881,15 +1881,6 @@

 	return p

 }

-func getGoMaxProcs() int {

-	if gmp := os.Getenv("GOMAXPROCS"); gmp != "" {

-		if p, err := strconv.Atoi(gmp); err == nil {

-			return p

-		}

-	}

-	return 1

-}

-

 func (s *Site) shouldBuild(p page.Page) bool {

 	return shouldBuild(s.BuildFuture, s.BuildExpired,

 		s.BuildDrafts, p.Draft(), p.PublishDate(), p.ExpiryDate())

--- a/hugolib/site_render.go

+++ b/hugolib/site_render.go

@@ -19,6 +19,8 @@

 	"strings"

 	"sync"

+	"github.com/gohugoio/hugo/config"

+

 	"github.com/gohugoio/hugo/output"

 	"github.com/pkg/errors"

@@ -55,7 +57,7 @@

 // TODO(bep np doc

 func (s *Site) renderPages(ctx *siteRenderContext) error {

-	numWorkers := getGoMaxProcs() * 4

+	numWorkers := config.GetNumWorkerMultiplier()

 	results := make(chan error)

 	pages := make(chan *pageState, numWorkers) // buffered for performance

--- a/main.go

+++ b/main.go

@@ -14,8 +14,6 @@

 package main

 import (

-	"runtime"

-

 	"os"

 	"github.com/gohugoio/hugo/commands"

@@ -22,8 +20,6 @@

 )

 func main() {

-

-	runtime.GOMAXPROCS(runtime.NumCPU())

 	resp := commands.Execute(os.Args[1:])

 	if resp.Err != nil {