Merge pull request #2261 from alphagov/robots.txt

Serve robots.txt from the admin app

app/main/views/index.py

@@ -26,6 +26,17 @@ def index():
     return render_template('views/signedout.html')
 
 
+@main.route('/robots.txt')
+def robots():
+    return (
+        'User-agent: *\n'
+        'Disallow: /sign-in\n'
+        'Disallow: /support\n'
+        'Disallow: /support/\n'
+        'Disallow: /register\n'
+    ), 200, {'Content-Type': 'text/plain'}
+
+
 @main.route('/error/<int:status_code>')
 def error(status_code):
     if status_code >= 500:

app/navigation.py

@@ -195,6 +195,7 @@ class HeaderNavigation(Navigation):
         'resend_email_verification',
         'resume_service',
         'revoke_api_key',
+        'robots',
         'send_messages',
         'send_notification',
         'send_one_off',
@@ -453,6 +454,7 @@ class MainNavigation(Navigation):
         'resend_email_verification',
         'resume_service',
         'roadmap',
+        'robots',
         'security',
         'send_notification',
         'service_dashboard_updates',
@@ -641,6 +643,7 @@ class CaseworkNavigation(Navigation):
         'resume_service',
         'revoke_api_key',
         'roadmap',
+        'robots',
         'security',
         'send_messages',
         'send_notification',
@@ -863,6 +866,7 @@ class OrgNavigation(Navigation):
         'resume_service',
         'revoke_api_key',
         'roadmap',
+        'robots',
         'security',
         'send_messages',
         'send_notification',