Merge pull request #3462 from alphagov/meta-tag-instead-of-robots

Hide pages from search engines using a meta tag instead of robots.txt

app/main/views/feedback.py

@@ -19,11 +19,13 @@ from app.models.feedback import (
     PROBLEM_TICKET_TYPE,
     QUESTION_TICKET_TYPE,
 )
+from app.utils import hide_from_search_engines
 
 bank_holidays = BankHolidays(use_cached_holidays=True)
 
 
 @main.route('/support', methods=['GET', 'POST'])
+@hide_from_search_engines
 def support():
 
     if current_user.is_authenticated:
@@ -50,12 +52,14 @@ def support():
 
 
 @main.route('/support/public')
+@hide_from_search_engines
 def support_public():
     return render_template('views/support/public.html')
 
 
 @main.route('/support/triage', methods=['GET', 'POST'])
 @main.route('/support/triage/<ticket_type:ticket_type>', methods=['GET', 'POST'])
+@hide_from_search_engines
 def triage(ticket_type=PROBLEM_TICKET_TYPE):
     form = Triage()
     if form.validate_on_submit():
@@ -75,6 +79,7 @@ def triage(ticket_type=PROBLEM_TICKET_TYPE):
 
 
 @main.route('/support/<ticket_type:ticket_type>', methods=['GET', 'POST'])
+@hide_from_search_engines
 def feedback(ticket_type):
     form = FeedbackOrProblem()
 
@@ -153,6 +158,7 @@ def feedback(ticket_type):
 
 
 @main.route('/support/escalate', methods=['GET', 'POST'])
+@hide_from_search_engines
 def bat_phone():
 
     if current_user.is_authenticated:
@@ -162,6 +168,7 @@ def bat_phone():
 
 
 @main.route('/support/thanks', methods=['GET', 'POST'])
+@hide_from_search_engines
 def thanks():
     return render_template(
         'views/support/thanks.html',

app/main/views/index.py

@@ -21,7 +21,7 @@ from app.main.views.sub_navigation_dictionaries import (
     using_notify_nav,
 )
 from app.models.feedback import QUESTION_TICKET_TYPE
-from app.utils import get_logo_cdn_domain
+from app.utils import get_logo_cdn_domain, hide_from_search_engines
 
 
 @main.route('/')
@@ -36,17 +36,6 @@ def index():
     )
 
 
-@main.route('/robots.txt')
-def robots():
-    return (
-        'User-agent: *\n'
-        'Disallow: /sign-in\n'
-        'Disallow: /support\n'
-        'Disallow: /support/\n'
-        'Disallow: /register\n'
-    ), 200, {'Content-Type': 'text/plain'}
-
-
 @main.route('/error/<int:status_code>')
 def error(status_code):
     if status_code >= 500:
@@ -253,6 +242,7 @@ def roadmap():
 
 
 @main.route('/features/email')
+@hide_from_search_engines
 def features_email():
     return render_template(
         'views/features/emails.html',

app/main/views/register.py

@@ -11,9 +11,11 @@ from app.main.forms import (
 )
 from app.main.views.verify import activate_user
 from app.models.user import InvitedOrgUser, InvitedUser, User
+from app.utils import hide_from_search_engines
 
 
 @main.route('/register', methods=['GET', 'POST'])
+@hide_from_search_engines
 def register():
     if current_user and current_user.is_authenticated:
         return redirect(url_for('main.show_accounts_or_dashboard'))

app/main/views/sign_in.py

@@ -14,9 +14,11 @@ from app import login_manager
 from app.main import main
 from app.main.forms import LoginForm
 from app.models.user import InvitedUser, User
+from app.utils import hide_from_search_engines
 
 
 @main.route('/sign-in', methods=(['GET', 'POST']))
+@hide_from_search_engines
 def sign_in():
     if current_user and current_user.is_authenticated:
         return redirect(url_for('main.show_accounts_or_dashboard'))