From d50d08ad2c87650d6c6a2612d2d8459b0f80a3d3 Mon Sep 17 00:00:00 2001
From: Jakob Ketterl <jakob.ketterl@gmx.de>
Date: Wed, 24 Mar 2021 16:08:13 +0100
Subject: [PATCH] add a robots.txt to exclude certain routes for search engines

---
 owrx/controllers/robots.py | 16 ++++++++++++++++
 owrx/http.py               |  2 ++
 2 files changed, 18 insertions(+)
 create mode 100644 owrx/controllers/robots.py

diff --git a/owrx/controllers/robots.py b/owrx/controllers/robots.py
new file mode 100644
index 00000000..3d00a1e2
--- /dev/null
+++ b/owrx/controllers/robots.py
@@ -0,0 +1,16 @@
+from owrx.controllers import Controller
+
+
+class RobotsController(Controller):
+    def indexAction(self):
+        # search engines should not be crawling internal / API routes
+        self.send_response(
+            """User-agent: *
+Disallow: /login
+Disallow: /logout
+Disallow: /pwchange
+Disallow: /settings
+Disallow: /imageupload
+""",
+            content_type="text/plain",
+        )
diff --git a/owrx/http.py b/owrx/http.py
index 51a6837b..9247dea1 100644
--- a/owrx/http.py
+++ b/owrx/http.py
@@ -20,6 +20,7 @@ from owrx.controllers.settings.bookmarks import BookmarksController
 from owrx.controllers.session import SessionController
 from owrx.controllers.profile import ProfileController
 from owrx.controllers.imageupload import ImageUploadController
+from owrx.controllers.robots import RobotsController
 from http.server import BaseHTTPRequestHandler
 from urllib.parse import urlparse, parse_qs
 import re
@@ -105,6 +106,7 @@ class Router(object):
     def __init__(self):
         self.routes = [
             StaticRoute("/", IndexController),
+            StaticRoute("/robots.txt", RobotsController),
             StaticRoute("/status.json", StatusController),
             RegexRoute("^/static/(.+)$", OwrxAssetsController),
             RegexRoute("^/compiled/(.+)$", CompiledAssetsController),