From fc0e2e93837df623fbf03ee18c33a91dcd9fe23c Mon Sep 17 00:00:00 2001
From: Maksim Nabokikh <maksim.nabokikh@flant.com>
Date: Thu, 23 Feb 2023 18:53:09 +0400
Subject: [PATCH] feat: Add default robots.txt (#2834)

Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>
---
 server/server.go    |  4 +++-
 server/templates.go | 14 ++++++++++----
 web/robots.txt      |  2 ++
 web/web.go          |  2 +-
 4 files changed, 16 insertions(+), 6 deletions(-)
 create mode 100644 web/robots.txt

diff --git a/server/server.go b/server/server.go
index df16e655cf..0aac0a6cd3 100755
--- a/server/server.go
+++ b/server/server.go
@@ -252,7 +252,7 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
 		extra:     c.Web.Extra,
 	}
 
-	static, theme, tmpls, err := loadWebConfig(web)
+	static, theme, robots, tmpls, err := loadWebConfig(web)
 	if err != nil {
 		return nil, fmt.Errorf("server: failed to load web static: %v", err)
 	}
@@ -390,6 +390,8 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
 
 	handlePrefix("/static", static)
 	handlePrefix("/theme", theme)
+	handleFunc("/robots.txt", robots)
+
 	s.mux = r
 
 	s.startKeyRotation(ctx, rotationStrategy, now)
diff --git a/server/templates.go b/server/templates.go
index e6ab3a793c..b77663e1f5 100644
--- a/server/templates.go
+++ b/server/templates.go
@@ -88,7 +88,7 @@ func getFuncMap(c webConfig) (template.FuncMap, error) {
 //	|- themes
 //	|  |- (theme name)
 //	|- templates
-func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error) {
+func loadWebConfig(c webConfig) (http.Handler, http.Handler, http.HandlerFunc, *templates, error) {
 	// fallback to the default theme if the legacy theme name is provided
 	if c.theme == "coreos" || c.theme == "tectonic" {
 		c.theme = ""
@@ -105,18 +105,24 @@ func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error)
 
 	staticFiles, err := fs.Sub(c.webFS, "static")
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("read static dir: %v", err)
+		return nil, nil, nil, nil, fmt.Errorf("read static dir: %v", err)
 	}
 	themeFiles, err := fs.Sub(c.webFS, path.Join("themes", c.theme))
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("read themes dir: %v", err)
+		return nil, nil, nil, nil, fmt.Errorf("read themes dir: %v", err)
+	}
+	robotsContent, err := fs.ReadFile(c.webFS, "robots.txt")
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("read robots.txt dir: %v", err)
 	}
 
 	static := http.FileServer(http.FS(staticFiles))
 	theme := http.FileServer(http.FS(themeFiles))
+	robots := func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, string(robotsContent)) }
 
 	templates, err := loadTemplates(c, "templates")
-	return static, theme, templates, err
+
+	return static, theme, robots, templates, err
 }
 
 // loadTemplates parses the expected templates from the provided directory.
diff --git a/web/robots.txt b/web/robots.txt
new file mode 100644
index 0000000000..1f53798bb4
--- /dev/null
+++ b/web/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/web/web.go b/web/web.go
index c5ff751490..0c7e9873a7 100644
--- a/web/web.go
+++ b/web/web.go
@@ -5,7 +5,7 @@ import (
 	"io/fs"
 )
 
-//go:embed static/* templates/* themes/*
+//go:embed static/* templates/* themes/* robots.txt
 var files embed.FS
 
 // FS returns a filesystem with the default web assets.