Use meta tag to tell search engines not to index

Google’s documentation says:

> robots.txt is not a mechanism for keeping a web page out of Google. To
> keep a web page out of Google, you should use noindex directives

A noindex directive means adding the following meta tag to pages that
shouldn’t be indexed:
```html
<meta name="robots" content="noindex" />
```

It’s also possible to set the directive as a HTTP header, but this seems
trickier to achieve on a per-view basis in Flask.

I’ve implemented this as a decorator so it can quickly be added to any
other pages that we decide shouldn’t appear in search results.
This commit is contained in:
Chris Hill-Scott
2020-05-26 17:39:25 +01:00
parent b98f4561fa
commit 92ffe3a78c
7 changed files with 45 additions and 2 deletions

View File

@@ -3,6 +3,7 @@ from functools import partial
import pytest
from bs4 import BeautifulSoup
from flask import url_for
from freezegun import freeze_time
from app.main.forms import FieldWithNoneOption
from tests.conftest import SERVICE_ONE_ID, normalize_spaces, sample_uuid
@@ -76,6 +77,26 @@ def test_robots(client):
)
@pytest.mark.parametrize('endpoint, kwargs', (
('sign_in', {}),
('support', {}),
('support_public', {}),
('triage', {}),
('feedback', {'ticket_type': 'ask-question-give-feedback'}),
('feedback', {'ticket_type': 'general'}),
('feedback', {'ticket_type': 'report-problem'}),
('bat_phone', {}),
('thanks', {}),
('register', {}),
pytest.param('index', {}, marks=pytest.mark.xfail(raises=AssertionError)),
))
@freeze_time('2012-12-12 12:12') # So we dont go out of business hours
def test_hiding_pages_from_search_engines(client_request, endpoint, kwargs):
client_request.logout()
page = client_request.get(f'main.{endpoint}', **kwargs)
assert page.select_one('meta[name=robots]')['content'] == 'noindex'
@pytest.mark.parametrize('view', [
'cookies', 'privacy', 'pricing', 'terms', 'roadmap',
'features', 'documentation', 'security',