diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index 65e2dc106..d2494be10 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -19,11 +19,13 @@ from app.models.feedback import ( PROBLEM_TICKET_TYPE, QUESTION_TICKET_TYPE, ) +from app.utils import hide_from_search_engines bank_holidays = BankHolidays(use_cached_holidays=True) @main.route('/support', methods=['GET', 'POST']) +@hide_from_search_engines def support(): if current_user.is_authenticated: @@ -50,12 +52,14 @@ def support(): @main.route('/support/public') +@hide_from_search_engines def support_public(): return render_template('views/support/public.html') @main.route('/support/triage', methods=['GET', 'POST']) @main.route('/support/triage/', methods=['GET', 'POST']) +@hide_from_search_engines def triage(ticket_type=PROBLEM_TICKET_TYPE): form = Triage() if form.validate_on_submit(): @@ -75,6 +79,7 @@ def triage(ticket_type=PROBLEM_TICKET_TYPE): @main.route('/support/', methods=['GET', 'POST']) +@hide_from_search_engines def feedback(ticket_type): form = FeedbackOrProblem() @@ -153,6 +158,7 @@ def feedback(ticket_type): @main.route('/support/escalate', methods=['GET', 'POST']) +@hide_from_search_engines def bat_phone(): if current_user.is_authenticated: @@ -162,6 +168,7 @@ def bat_phone(): @main.route('/support/thanks', methods=['GET', 'POST']) +@hide_from_search_engines def thanks(): return render_template( 'views/support/thanks.html', diff --git a/app/main/views/index.py b/app/main/views/index.py index 4ed477e70..7e59a0347 100644 --- a/app/main/views/index.py +++ b/app/main/views/index.py @@ -21,7 +21,7 @@ from app.main.views.sub_navigation_dictionaries import ( using_notify_nav, ) from app.models.feedback import QUESTION_TICKET_TYPE -from app.utils import get_logo_cdn_domain +from app.utils import get_logo_cdn_domain, hide_from_search_engines @main.route('/') @@ -36,17 +36,6 @@ def index(): ) -@main.route('/robots.txt') -def robots(): - return ( - 'User-agent: *\n' - 'Disallow: /sign-in\n' - 'Disallow: /support\n' - 'Disallow: /support/\n' - 'Disallow: /register\n' - ), 200, {'Content-Type': 'text/plain'} - - @main.route('/error/') def error(status_code): if status_code >= 500: @@ -253,6 +242,7 @@ def roadmap(): @main.route('/features/email') +@hide_from_search_engines def features_email(): return render_template( 'views/features/emails.html', diff --git a/app/main/views/register.py b/app/main/views/register.py index 8134538f0..35693a5ee 100644 --- a/app/main/views/register.py +++ b/app/main/views/register.py @@ -11,9 +11,11 @@ from app.main.forms import ( ) from app.main.views.verify import activate_user from app.models.user import InvitedOrgUser, InvitedUser, User +from app.utils import hide_from_search_engines @main.route('/register', methods=['GET', 'POST']) +@hide_from_search_engines def register(): if current_user and current_user.is_authenticated: return redirect(url_for('main.show_accounts_or_dashboard')) diff --git a/app/main/views/sign_in.py b/app/main/views/sign_in.py index 567bb3f5a..ae15fa751 100644 --- a/app/main/views/sign_in.py +++ b/app/main/views/sign_in.py @@ -14,9 +14,11 @@ from app import login_manager from app.main import main from app.main.forms import LoginForm from app.models.user import InvitedUser, User +from app.utils import hide_from_search_engines @main.route('/sign-in', methods=(['GET', 'POST'])) +@hide_from_search_engines def sign_in(): if current_user and current_user.is_authenticated: return redirect(url_for('main.show_accounts_or_dashboard')) diff --git a/app/navigation.py b/app/navigation.py index b407db0a5..d98fba4cb 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -256,7 +256,6 @@ class HeaderNavigation(Navigation): 'returned_letters', 'returned_letters_report', 'revoke_api_key', - 'robots', 'send_messages', 'send_notification', 'send_one_off', @@ -608,7 +607,6 @@ class MainNavigation(Navigation): 'returned_letters_report', 'revalidate_email_sent', 'roadmap', - 'robots', 'security', 'send_notification', 'send_from_contact_list', @@ -870,7 +868,6 @@ class CaseworkNavigation(Navigation): 'revalidate_email_sent', 'revoke_api_key', 'roadmap', - 'robots', 'security', 'send_messages', 'send_notification', @@ -1163,7 +1160,6 @@ class OrgNavigation(Navigation): 'revalidate_email_sent', 'revoke_api_key', 'roadmap', - 'robots', 'security', 'send_messages', 'send_notification', diff --git a/app/templates/admin_template.html b/app/templates/admin_template.html index 1371f4ff9..ec40d460b 100644 --- a/app/templates/admin_template.html +++ b/app/templates/admin_template.html @@ -17,6 +17,9 @@ + {% if g.hide_from_search_engines %} + + {% endif %} {% block meta_format_detection %} diff --git a/app/templates/views/support/public.html b/app/templates/views/support/public.html index c965e5e72..aca84aece 100644 --- a/app/templates/views/support/public.html +++ b/app/templates/views/support/public.html @@ -3,7 +3,7 @@ {% from "components/page-header.html" import page_header %} {% block per_page_title %} - The GOV.UK Notify team cannot give advice to members of the public + The GOV.UK Notify service is for people who work in the government {% endblock %} {% block maincolumn_content %} diff --git a/app/utils.py b/app/utils.py index 6de6baf3b..01f85de3b 100644 --- a/app/utils.py +++ b/app/utils.py @@ -16,7 +16,16 @@ import pyexcel import pyexcel_xlsx import pytz from dateutil import parser -from flask import abort, current_app, redirect, request, session, url_for +from flask import ( + abort, + current_app, + g, + make_response, + redirect, + request, + session, + url_for, +) from flask_login import current_user, login_required from notifications_utils.field import Field from notifications_utils.formatters import ( @@ -767,3 +776,13 @@ def is_less_than_90_days_ago(date_from_db): return (datetime.utcnow() - datetime.strptime( date_from_db, "%Y-%m-%dT%H:%M:%S.%fZ" )).days < 90 + + +def hide_from_search_engines(f): + @wraps(f) + def decorated_function(*args, **kwargs): + g.hide_from_search_engines = True + response = make_response(f(*args, **kwargs)) + response.headers['X-Robots-Tag'] = 'noindex' + return response + return decorated_function diff --git a/tests/app/main/views/test_index.py b/tests/app/main/views/test_index.py index ea6f4b54a..25bbc69d5 100644 --- a/tests/app/main/views/test_index.py +++ b/tests/app/main/views/test_index.py @@ -3,6 +3,7 @@ from functools import partial import pytest from bs4 import BeautifulSoup from flask import url_for +from freezegun import freeze_time from app.main.forms import FieldWithNoneOption from tests.conftest import SERVICE_ONE_ID, normalize_spaces, sample_uuid @@ -62,18 +63,36 @@ def test_logged_in_user_redirects_to_choose_account( ) -def test_robots(client): - assert url_for('main.robots') == '/robots.txt' - response = client.get(url_for('main.robots')) - assert response.headers['Content-Type'] == 'text/plain' - assert response.status_code == 200 - assert response.get_data(as_text=True) == ( - 'User-agent: *\n' - 'Disallow: /sign-in\n' - 'Disallow: /support\n' - 'Disallow: /support/\n' - 'Disallow: /register\n' - ) +def test_robots(client_request): + client_request.get_url('/robots.txt', _expected_status=404) + + +@pytest.mark.parametrize('endpoint, kwargs', ( + ('sign_in', {}), + ('support', {}), + ('support_public', {}), + ('triage', {}), + ('feedback', {'ticket_type': 'ask-question-give-feedback'}), + ('feedback', {'ticket_type': 'general'}), + ('feedback', {'ticket_type': 'report-problem'}), + ('bat_phone', {}), + ('thanks', {}), + ('register', {}), + ('features_email', {}), + pytest.param('index', {}, marks=pytest.mark.xfail(raises=AssertionError)), +)) +@freeze_time('2012-12-12 12:12') # So we don’t go out of business hours +def test_hiding_pages_from_search_engines( + client, + mock_get_service_and_organisation_counts, + endpoint, + kwargs, +): + response = client.get(url_for(f'main.{endpoint}', **kwargs)) + assert 'X-Robots-Tag' in response.headers + assert response.headers['X-Robots-Tag'] == 'noindex' + page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser') + assert page.select_one('meta[name=robots]')['content'] == 'noindex' @pytest.mark.parametrize('view', [