Merge pull request #3462 from alphagov/meta-tag-instead-of-robots

Hide pages from search engines using a meta tag instead of robots.txt

app/main/views/feedback.py

@@ -19,11 +19,13 @@ from app.models.feedback import (
     PROBLEM_TICKET_TYPE,
     QUESTION_TICKET_TYPE,
 )
+from app.utils import hide_from_search_engines
 
 bank_holidays = BankHolidays(use_cached_holidays=True)
 
 
 @main.route('/support', methods=['GET', 'POST'])
+@hide_from_search_engines
 def support():
 
     if current_user.is_authenticated:
@@ -50,12 +52,14 @@ def support():
 
 
 @main.route('/support/public')
+@hide_from_search_engines
 def support_public():
     return render_template('views/support/public.html')
 
 
 @main.route('/support/triage', methods=['GET', 'POST'])
 @main.route('/support/triage/<ticket_type:ticket_type>', methods=['GET', 'POST'])
+@hide_from_search_engines
 def triage(ticket_type=PROBLEM_TICKET_TYPE):
     form = Triage()
     if form.validate_on_submit():
@@ -75,6 +79,7 @@ def triage(ticket_type=PROBLEM_TICKET_TYPE):
 
 
 @main.route('/support/<ticket_type:ticket_type>', methods=['GET', 'POST'])
+@hide_from_search_engines
 def feedback(ticket_type):
     form = FeedbackOrProblem()
 
@@ -153,6 +158,7 @@ def feedback(ticket_type):
 
 
 @main.route('/support/escalate', methods=['GET', 'POST'])
+@hide_from_search_engines
 def bat_phone():
 
     if current_user.is_authenticated:
@@ -162,6 +168,7 @@ def bat_phone():
 
 
 @main.route('/support/thanks', methods=['GET', 'POST'])
+@hide_from_search_engines
 def thanks():
     return render_template(
         'views/support/thanks.html',

app/main/views/index.py

@@ -21,7 +21,7 @@ from app.main.views.sub_navigation_dictionaries import (
     using_notify_nav,
 )
 from app.models.feedback import QUESTION_TICKET_TYPE
-from app.utils import get_logo_cdn_domain
+from app.utils import get_logo_cdn_domain, hide_from_search_engines
 
 
 @main.route('/')
@@ -36,17 +36,6 @@ def index():
     )
 
 
-@main.route('/robots.txt')
-def robots():
-    return (
-        'User-agent: *\n'
-        'Disallow: /sign-in\n'
-        'Disallow: /support\n'
-        'Disallow: /support/\n'
-        'Disallow: /register\n'
-    ), 200, {'Content-Type': 'text/plain'}
-
-
 @main.route('/error/<int:status_code>')
 def error(status_code):
     if status_code >= 500:
@@ -253,6 +242,7 @@ def roadmap():
 
 
 @main.route('/features/email')
+@hide_from_search_engines
 def features_email():
     return render_template(
         'views/features/emails.html',

app/main/views/register.py

@@ -11,9 +11,11 @@ from app.main.forms import (
 )
 from app.main.views.verify import activate_user
 from app.models.user import InvitedOrgUser, InvitedUser, User
+from app.utils import hide_from_search_engines
 
 
 @main.route('/register', methods=['GET', 'POST'])
+@hide_from_search_engines
 def register():
     if current_user and current_user.is_authenticated:
         return redirect(url_for('main.show_accounts_or_dashboard'))

app/main/views/sign_in.py

@@ -14,9 +14,11 @@ from app import login_manager
 from app.main import main
 from app.main.forms import LoginForm
 from app.models.user import InvitedUser, User
+from app.utils import hide_from_search_engines
 
 
 @main.route('/sign-in', methods=(['GET', 'POST']))
+@hide_from_search_engines
 def sign_in():
     if current_user and current_user.is_authenticated:
         return redirect(url_for('main.show_accounts_or_dashboard'))

app/navigation.py

@@ -256,7 +256,6 @@ class HeaderNavigation(Navigation):
         'returned_letters',
         'returned_letters_report',
         'revoke_api_key',
-        'robots',
         'send_messages',
         'send_notification',
         'send_one_off',
@@ -608,7 +607,6 @@ class MainNavigation(Navigation):
         'returned_letters_report',
         'revalidate_email_sent',
         'roadmap',
-        'robots',
         'security',
         'send_notification',
         'send_from_contact_list',
@@ -870,7 +868,6 @@ class CaseworkNavigation(Navigation):
         'revalidate_email_sent',
         'revoke_api_key',
         'roadmap',
-        'robots',
         'security',
         'send_messages',
         'send_notification',
@@ -1163,7 +1160,6 @@ class OrgNavigation(Navigation):
         'revalidate_email_sent',
         'revoke_api_key',
         'roadmap',
-        'robots',
         'security',
         'send_messages',
         'send_notification',

app/templates/admin_template.html

@@ -17,6 +17,9 @@
   <style>
       .govuk-header__container { border-color: {{header_colour}} }
   </style>
+  {% if g.hide_from_search_engines %}
+    <meta name="robots" content="noindex" />
+  {% endif %}
   <meta name="google-site-verification" content="niWnSqImOWz6mVQTYqNb5tFK8HaKSB4b3ED4Z9gtUQ0" />
   {% block meta_format_detection %}
     <meta name="format-detection" content="telephone=no">

app/templates/views/support/public.html

@@ -3,7 +3,7 @@
 {% from "components/page-header.html" import page_header %}
 
 {% block per_page_title %}
-  The GOV.UK Notify team cannot give advice to members of the public
+  The GOV.UK Notify service is for people who work in the government
 {% endblock %}
 
 {% block maincolumn_content %}

app/utils.py

@@ -16,7 +16,16 @@ import pyexcel
 import pyexcel_xlsx
 import pytz
 from dateutil import parser
-from flask import abort, current_app, redirect, request, session, url_for
+from flask import (
+    abort,
+    current_app,
+    g,
+    make_response,
+    redirect,
+    request,
+    session,
+    url_for,
+)
 from flask_login import current_user, login_required
 from notifications_utils.field import Field
 from notifications_utils.formatters import (
@@ -767,3 +776,13 @@ def is_less_than_90_days_ago(date_from_db):
     return (datetime.utcnow() - datetime.strptime(
         date_from_db, "%Y-%m-%dT%H:%M:%S.%fZ"
     )).days < 90
+
+
+def hide_from_search_engines(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        g.hide_from_search_engines = True
+        response = make_response(f(*args, **kwargs))
+        response.headers['X-Robots-Tag'] = 'noindex'
+        return response
+    return decorated_function

tests/app/main/views/test_index.py

@@ -3,6 +3,7 @@ from functools import partial
 import pytest
 from bs4 import BeautifulSoup
 from flask import url_for
+from freezegun import freeze_time
 
 from app.main.forms import FieldWithNoneOption
 from tests.conftest import SERVICE_ONE_ID, normalize_spaces, sample_uuid
@@ -62,18 +63,36 @@ def test_logged_in_user_redirects_to_choose_account(
     )
 
 
-def test_robots(client):
-    assert url_for('main.robots') == '/robots.txt'
-    response = client.get(url_for('main.robots'))
-    assert response.headers['Content-Type'] == 'text/plain'
-    assert response.status_code == 200
-    assert response.get_data(as_text=True) == (
-        'User-agent: *\n'
-        'Disallow: /sign-in\n'
-        'Disallow: /support\n'
-        'Disallow: /support/\n'
-        'Disallow: /register\n'
-    )
+def test_robots(client_request):
+    client_request.get_url('/robots.txt', _expected_status=404)
+
+
+@pytest.mark.parametrize('endpoint, kwargs', (
+    ('sign_in', {}),
+    ('support', {}),
+    ('support_public', {}),
+    ('triage', {}),
+    ('feedback', {'ticket_type': 'ask-question-give-feedback'}),
+    ('feedback', {'ticket_type': 'general'}),
+    ('feedback', {'ticket_type': 'report-problem'}),
+    ('bat_phone', {}),
+    ('thanks', {}),
+    ('register', {}),
+    ('features_email', {}),
+    pytest.param('index', {}, marks=pytest.mark.xfail(raises=AssertionError)),
+))
+@freeze_time('2012-12-12 12:12')  # So we don’t go out of business hours
+def test_hiding_pages_from_search_engines(
+    client,
+    mock_get_service_and_organisation_counts,
+    endpoint,
+    kwargs,
+):
+    response = client.get(url_for(f'main.{endpoint}', **kwargs))
+    assert 'X-Robots-Tag' in response.headers
+    assert response.headers['X-Robots-Tag'] == 'noindex'
+    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
+    assert page.select_one('meta[name=robots]')['content'] == 'noindex'
 
 
 @pytest.mark.parametrize('view', [