shithub: hugo

Info  •  Files  •  Log  •  Branches

Do not create robots.txt by default

Meny people, including me, have a custom robots.txt in static.

Also remove that option from the command line; it doesn't feel
important enough.

Fixes ##2049

--- a/commands/hugo.go

+++ b/commands/hugo.go

@@ -126,7 +126,7 @@

 	buildWatch            bool

 	canonifyURLs          bool

 	cleanDestination      bool

-	disableRobotsTXT      bool

+	enableRobotsTXT       bool

 	disableRSS            bool

 	disableSitemap        bool

 	draft                 bool

@@ -216,7 +216,6 @@

 	cmd.Flags().BoolVarP(&future, "buildFuture", "F", false, "include content with publishdate in the future")

 	cmd.Flags().BoolVar(&disableRSS, "disableRSS", false, "Do not build RSS files")

 	cmd.Flags().BoolVar(&disableSitemap, "disableSitemap", false, "Do not build Sitemap file")

-	cmd.Flags().BoolVar(&disableRobotsTXT, "disableRobotsTXT", false, "Do not build Robots TXT file")

 	cmd.Flags().StringVarP(&source, "source", "s", "", "filesystem path to read files relative from")

 	cmd.Flags().StringVarP(&contentDir, "contentDir", "c", "", "filesystem path to content directory")

 	cmd.Flags().StringVarP(&layoutDir, "layoutDir", "l", "", "filesystem path to layout directory")

@@ -364,8 +363,8 @@

 		if flagChanged(cmdV.Flags(), "disableSitemap") {

 			viper.Set("DisableSitemap", disableSitemap)

 		}

-		if flagChanged(cmdV.Flags(), "disableRobotsTXT") {

-			viper.Set("DisableRobotsTXT", disableRobotsTXT)

+		if flagChanged(cmdV.Flags(), "enableRobotsTXT") {

+			viper.Set("EnableRobotsTXT", enableRobotsTXT)

 		}

 		if flagChanged(cmdV.Flags(), "pluralizeListTitles") {

 			viper.Set("PluralizeListTitles", pluralizeListTitles)

--- a/docs/content/extras/robots-txt.md

+++ b/docs/content/extras/robots-txt.md

@@ -13,13 +13,12 @@

 Hugo can generated a customized [robots.txt](http://www.robotstxt.org/) in the

 [same way as any other templates]({{< ref "templates/go-templates.md" >}}).

-By default, it generates a robots.txt, which allows everything, with the following content:

+To enable it, just set `enableRobotsTXT` option to `true` in the [configuration file]({{< ref "overview/configuration.md" >}}). By default, it generates a robots.txt, which allows everything, with the following content:

 ```http

 User-agent: *

 ```

-To disable it, just set `disableRobotsTXT` option to `false` in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}).

 Hugo will use the template `robots.txt` according to the following list in descending precedence:

--- a/docs/content/overview/configuration.md

+++ b/docs/content/overview/configuration.md

@@ -95,8 +95,8 @@

     disableRSS:                 false

     # Do not build Sitemap file

     disableSitemap:             false

-    # Do not build robots.txt file

-    disableRobotsTXT:           false

+    # Build robots.txt file

+    enableRobotsTXT:           false

     # edit new content with this editor, if provided

     editor:                     ""

     # Enable Emoji emoticons support for page content.

--- a/hugolib/robotstxt_test.go

+++ b/hugolib/robotstxt_test.go

@@ -36,6 +36,7 @@

 	hugofs.InitMemFs()

 	viper.Set("baseurl", "http://auth/bub/")

+	viper.Set("enableRobotsTXT", true)

 	s := &Site{

 		Source: &source.InMemorySource{ByteSource: WEIGHTED_SOURCES},

--- a/hugolib/site.go

+++ b/hugolib/site.go

@@ -1864,7 +1864,7 @@

 }

 func (s *Site) RenderRobotsTXT() error {

-	if viper.GetBool("DisableRobotsTXT") {

+	if !viper.GetBool("EnableRobotsTXT") {

 		return nil

 	}