some lil fixes for kibou compatibility

internal/api/security/robots.go

@@ -0,0 +1,17 @@
+package security
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+)
+
+const robotsString = `User-agent: *
+Disallow: /
+`
+
+// RobotsGETHandler returns the most restrictive possible robots.txt file in response to a call to /robots.txt.
+// The response instructs bots with *any* user agent not to index the instance at all.
+func (m *Module) RobotsGETHandler(c *gin.Context) {
+	c.String(http.StatusOK, robotsString)
+}

internal/api/security/security.go

@@ -19,12 +19,16 @@
 package security
 
 import (
+	"net/http"
+
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/router"
 )
 
+const robotsPath = "/robots.txt"
+
 // Module implements the ClientAPIModule interface for security middleware
 type Module struct {
 	config *config.Config
@@ -44,5 +48,6 @@ func (m *Module) Route(s router.Router) error {
 	s.AttachMiddleware(m.FlocBlock)
 	s.AttachMiddleware(m.ExtraHeaders)
 	s.AttachMiddleware(m.UserAgentBlock)
+	s.AttachHandler(http.MethodGet, robotsPath, m.RobotsGETHandler)
 	return nil
 }

internal/api/security/useragentblock.go

@@ -23,20 +23,24 @@ import (
 	"strings"
 
 	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
 )
 
-// UserAgentBlock is a middleware that prevents google chrome cohort tracking by
-// writing the Permissions-Policy header after all other parts of the request have been completed.
-// See: https://plausible.io/blog/google-floc
+// UserAgentBlock blocks requests with undesired, empty, or invalid user-agent strings.
 func (m *Module) UserAgentBlock(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func": "UserAgentBlock",
+	})
 
 	ua := c.Request.UserAgent()
 	if ua == "" {
+		l.Debug("aborting request because there's no user-agent set")
 		c.AbortWithStatus(http.StatusTeapot)
 		return
 	}
 
-	if strings.Contains(strings.ToLower(c.Request.UserAgent()), strings.ToLower("friendica")) {
+	if strings.Contains(strings.ToLower(ua), strings.ToLower("friendica")) {
+		l.Debugf("aborting request with user-agent %s because it contains 'friendica'", ua)
 		c.AbortWithStatus(http.StatusTeapot)
 		return
 	}