Send HTTP header as well as inserting meta tag

This will let us use the decorator on endpoints that don’t return HTML.

app/utils.py

@@ -16,7 +16,16 @@ import pyexcel
 import pyexcel_xlsx
 import pytz
 from dateutil import parser
-from flask import abort, current_app, g, redirect, request, session, url_for
+from flask import (
+    abort,
+    current_app,
+    g,
+    make_response,
+    redirect,
+    request,
+    session,
+    url_for,
+)
 from flask_login import current_user, login_required
 from notifications_utils.field import Field
 from notifications_utils.formatters import (
@@ -773,5 +782,7 @@ def hide_from_search_engines(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
         g.hide_from_search_engines = True
-        return f(*args, **kwargs)
+        response = make_response(f(*args, **kwargs))
+        response.headers['X-Robots-Tag'] = 'noindex'
+        return response
     return decorated_function

tests/app/main/views/test_index.py

@@ -82,9 +82,16 @@ def test_robots(client_request):
     pytest.param('index', {}, marks=pytest.mark.xfail(raises=AssertionError)),
 ))
 @freeze_time('2012-12-12 12:12')  # So we don’t go out of business hours
-def test_hiding_pages_from_search_engines(client_request, endpoint, kwargs):
+def test_hiding_pages_from_search_engines(
-    client_request.logout()
+    client,
-    page = client_request.get(f'main.{endpoint}', **kwargs)
+    mock_get_service_and_organisation_counts,
+    endpoint,
+    kwargs,
+):
+    response = client.get(url_for(f'main.{endpoint}', **kwargs))
+    assert 'X-Robots-Tag' in response.headers
+    assert response.headers['X-Robots-Tag'] == 'noindex'
+    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
     assert page.select_one('meta[name=robots]')['content'] == 'noindex'