diff --git a/app/main/views/api_keys.py b/app/main/views/api_keys.py
index 90c5feb2a..b4d4b52bb 100644
--- a/app/main/views/api_keys.py
+++ b/app/main/views/api_keys.py
@@ -131,9 +131,9 @@ def revoke_api_key(service_id, key_id):
     key_name = current_service.get_api_key(key_id)['name']
     if request.method == 'GET':
         flash([
-            "Are you sure you want to revoke this API key?",
-            "‘{}’ will no longer let you connect to GOV.UK Notify.".format(key_name)
-        ], 'revoke')
+            "Are you sure you want to revoke ‘{}’?".format(key_name),
+            "You will not be able to use this API key to connect to GOV.UK Notify."
+        ], 'revoke this API key')
         return render_template(
             'views/api/keys.html',
         )
diff --git a/app/main/views/templates.py b/app/main/views/templates.py
index 488cc1446..ded6fd581 100644
--- a/app/main/views/templates.py
+++ b/app/main/views/templates.py
@@ -615,7 +615,7 @@ def delete_service_template(service_id, template_id):
         last_used_notification = template_statistics_client.get_template_statistics_for_template(
             service_id, template['id']
         )
-        message = 'It was last used {} ago'.format(
+        message = 'This template was last used {} ago.'.format(
             'more than seven days' if not last_used_notification else get_human_readable_delta(
                 parse(last_used_notification['created_at']).replace(tzinfo=None),
                 datetime.utcnow()
diff --git a/app/templates/flash_messages.html b/app/templates/flash_messages.html
index a33947d48..55617e7fd 100644
--- a/app/templates/flash_messages.html
+++ b/app/templates/flash_messages.html
@@ -6,7 +6,7 @@
         {{ banner(
           message if message is string else message[0],
           'default' if ((category == 'default') or (category == 'default_with_tick')) else 'dangerous',
-          delete_button="Yes, {}".format(category) if category in ['delete', 'suspend', 'resume', 'remove', 'revoke'] else None,
+          delete_button="Yes, {}".format(category) if category in ['delete', 'suspend', 'resume', 'remove', 'revoke this API key'] else None,
           with_tick=True if category == 'default_with_tick' else False,
           context=message[1] if message is not string
         )}}
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
index 5de6a9c25..f1d1c58a2 100755
--- a/scripts/run_tests.sh
+++ b/scripts/run_tests.sh
@@ -37,5 +37,5 @@ npm test
 display_result $? 3 "Front end code style check"
 
 ## Code coverage
-py.test -n4 --maxfail=10 --cov=app --cov-report=term-missing tests/ --junitxml=test_results.xml --strict
+py.test -n4 --maxfail=10 --cov=app --cov-report=term-missing tests/ --junitxml=test_results.xml --strict -p no:warnings
 display_result $? 4 "Code coverage"
diff --git a/tests/app/main/views/test_api_integration.py b/tests/app/main/views/test_api_integration.py
index f187fec1f..5d8d32208 100644
--- a/tests/app/main/views/test_api_integration.py
+++ b/tests/app/main/views/test_api_integration.py
@@ -308,9 +308,9 @@ def test_should_show_confirm_revoke_api_key(
         _test_page_title=False,
     )
     assert normalize_spaces(page.select('.banner-dangerous')[0].text) == (
-        'Are you sure you want to revoke this API key? '
-        '‘some key name’ will no longer let you connect to GOV.UK Notify. '
-        'Yes, revoke'
+        'Are you sure you want to revoke ‘some key name’? '
+        'You will not be able to use this API key to connect to GOV.UK Notify. '
+        'Yes, revoke this API key'
     )
     assert mock_get_api_keys.call_args_list == [
         call(
diff --git a/tests/app/main/views/test_templates.py b/tests/app/main/views/test_templates.py
index 5f4664143..fcfa62108 100644
--- a/tests/app/main/views/test_templates.py
+++ b/tests/app/main/views/test_templates.py
@@ -1283,7 +1283,7 @@ def test_should_show_delete_template_page_with_time_block(
         )
     assert "Are you sure you want to delete ‘Two week reminder’?" in page.select('.banner-dangerous')[0].text
     assert normalize_spaces(page.select('.banner-dangerous p')[0].text) == (
-        'It was last used 10 minutes ago'
+        'This template was last used 10 minutes ago.'
     )
     assert normalize_spaces(page.select('.sms-message-wrapper')[0].text) == (
         'service one: Template <em>content</em> with & entity'
@@ -1312,7 +1312,7 @@ def test_should_show_delete_template_page_with_time_block_for_empty_notification
         )
     assert "Are you sure you want to delete ‘Two week reminder’?" in page.select('.banner-dangerous')[0].text
     assert normalize_spaces(page.select('.banner-dangerous p')[0].text) == (
-        'It was last used more than seven days ago'
+        'This template was last used more than seven days ago.'
     )
     assert normalize_spaces(page.select('.sms-message-wrapper')[0].text) == (
         'service one: Template <em>content</em> with & entity'