Remove robots.txt

Google’s documentation says:

> robots.txt is not a mechanism for keeping a web page out of Google. To
> keep a web page out of Google, you should use noindex directives

We’ve implemented a noindex directive now, so we don’t need to serve
robots.txt any more.

app/main/views/index.py

@@ -36,17 +36,6 @@ def index():
     )
 
 
-@main.route('/robots.txt')
-def robots():
-    return (
-        'User-agent: *\n'
-        'Disallow: /sign-in\n'
-        'Disallow: /support\n'
-        'Disallow: /support/\n'
-        'Disallow: /register\n'
-    ), 200, {'Content-Type': 'text/plain'}
-
-
 @main.route('/error/<int:status_code>')
 def error(status_code):
     if status_code >= 500:

app/navigation.py

@@ -256,7 +256,6 @@ class HeaderNavigation(Navigation):
         'returned_letters',
         'returned_letters_report',
         'revoke_api_key',
-        'robots',
         'send_messages',
         'send_notification',
         'send_one_off',
@@ -608,7 +607,6 @@ class MainNavigation(Navigation):
         'returned_letters_report',
         'revalidate_email_sent',
         'roadmap',
-        'robots',
         'security',
         'send_notification',
         'send_from_contact_list',
@@ -870,7 +868,6 @@ class CaseworkNavigation(Navigation):
         'revalidate_email_sent',
         'revoke_api_key',
         'roadmap',
-        'robots',
         'security',
         'send_messages',
         'send_notification',
@@ -1163,7 +1160,6 @@ class OrgNavigation(Navigation):
         'revalidate_email_sent',
         'revoke_api_key',
         'roadmap',
-        'robots',
         'security',
         'send_messages',
         'send_notification',

tests/app/main/views/test_index.py

@@ -63,18 +63,8 @@ def test_logged_in_user_redirects_to_choose_account(
     )
 
 
-def test_robots(client):
+def test_robots(client_request):
-    assert url_for('main.robots') == '/robots.txt'
+    client_request.get_url('/robots.txt', _expected_status=404)
-    response = client.get(url_for('main.robots'))
-    assert response.headers['Content-Type'] == 'text/plain'
-    assert response.status_code == 200
-    assert response.get_data(as_text=True) == (
-        'User-agent: *\n'
-        'Disallow: /sign-in\n'
-        'Disallow: /support\n'
-        'Disallow: /support/\n'
-        'Disallow: /register\n'
-    )
 
 
 @pytest.mark.parametrize('endpoint, kwargs', (