robots

Change-Id: Ifba648985db9abe0c29dafb1d4c9d119b38357d2
seankhliao · Nov 16, 2024 · 7e24247 · 7e24247
1 parent 3a3c47a
commit 7e24247
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 4 deletions.
diff --git a/ghdefaults/ghdefaults.go b/ghdefaults/ghdefaults.go
@@ -17,6 +17,11 @@ import (
 	"golang.org/x/oauth2"
 )
 
+func Register(a *App, r yrun.HTTPRegistrar) {
+	r.Pattern("GET", a.host, "/robots.txt", a.robots)
+	r.Pattern("POST", a.host, "/webhook", a.ServeHTTP)
+}
+
 type Config struct {
 	Host          string
 	AppID         int64
@@ -42,10 +47,6 @@ func New(c Config, o yrun.O11y) (*App, error) {
 	}, nil
 }
 
-func Register(a *App, r yrun.HTTPRegistrar) {
-	r.Pattern("POST", a.host, "/webhook", a.ServeHTTP)
-}
-
 var defaultConfig = map[string]github.Repository{
 	"erred": {
 		AllowMergeCommit:    github.Bool(false),
@@ -241,3 +242,12 @@ func (a *App) setDefaults(ctx context.Context, installID int64, owner, repo stri
 
 	return nil
 }
+
+const robotsTxt = `
+User-agent: *
+Disallow: /
+`
+
+func (a *App) robots(rw http.ResponseWriter, r *http.Request) {
+	fmt.Fprintln(rw, robotsTxt)
+}
diff --git a/reqlog/reqlog.go b/reqlog/reqlog.go
@@ -1,13 +1,15 @@
 package reqlog
 
 import (
+	"fmt"
 	"log/slog"
 	"net/http"
 
 	"go.seankhliao.com/mono/yrun"
 )
 
 func Register(a *App, r yrun.HTTPRegistrar) {
+	r.Pattern("GET", a.config.Host, "/robots.txt", a.robots)
 	r.Pattern("", a.config.Host, "/", a.ServeHTTP)
 }
 
@@ -28,6 +30,10 @@ func New(c Config, o yrun.O11y) (*App, error) {
 }
 
 func (a *App) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	rw.Header().Set("x-robots-tag", "none")
+	rw.Header().Set("content-type", "text/plain")
+	fmt.Fprintln(rw, "ok")
+
 	a.o.L.LogAttrs(r.Context(), slog.LevelInfo, "received request",
 		slog.String("http.method", r.Method),
 		slog.String("http.proto", r.Proto),
@@ -39,3 +45,12 @@ func (a *App) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		slog.Any("http.trailers", r.Trailer),
 	)
 }
+
+const robotsTxt = `
+User-agent: *
+Disallow: /
+`
+
+func (a *App) robots(rw http.ResponseWriter, r *http.Request) {
+	fmt.Fprintln(rw, robotsTxt)
+}