diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py
index 51fdf0f70..8d96a0940 100644
--- a/app/celery/nightly_tasks.py
+++ b/app/celery/nightly_tasks.py
@@ -240,17 +240,16 @@ def letter_raise_alert_if_no_ack_file_for_zip():
                                               subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
         ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))
 
-    message = (
-        "Letter ack file does not contain all zip files sent. "
-        "Missing ack for zip files: {}, "
-        "pdf bucket: {}, subfolder: {}, "
-        "ack bucket: {}"
-    ).format(
-        str(sorted(zip_file_set - ack_file_set)),
-        current_app.config['LETTERS_PDF_BUCKET_NAME'],
-        datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
-        current_app.config['DVLA_RESPONSE_BUCKET_NAME']
-    )
+    message = '\n'.join([
+        "Letter ack file does not contain all zip files sent."
+        "",
+        f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n",  # noqa
+        f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent",  # noqa
+        f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}",
+        "",
+        f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}",
+    ])
+
     # strip empty element before comparison
     ack_file_set.discard('')
     zip_file_set.discard('')
diff --git a/tests/app/celery/test_nightly_tasks.py b/tests/app/celery/test_nightly_tasks.py
index af29ff55e..b65546907 100644
--- a/tests/app/celery/test_nightly_tasks.py
+++ b/tests/app/celery/test_nightly_tasks.py
@@ -384,22 +384,17 @@ def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db):
 
     assert mock_file_list.call_count == 2
 
-    message = "Letter ack file does not contain all zip files sent. " \
-              "Missing ack for zip files: {}, " \
-              "pdf bucket: {}, subfolder: {}, " \
-              "ack bucket: {}".format(str(['NOTIFY.2018-01-11175009', 'NOTIFY.2018-01-11175010']),
-                                      current_app.config['LETTERS_PDF_BUCKET_NAME'],
-                                      datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
-                                      current_app.config['DVLA_RESPONSE_BUCKET_NAME'])
     mock_create_ticket.assert_called_once_with(
         ANY,
         subject="Letter acknowledge error",
-        message=message,
+        message=ANY,
         ticket_type='incident',
         technical_ticket=True,
         ticket_categories=['notify_letters']
     )
     mock_send_ticket_to_zendesk.assert_called_once()
+    assert "['NOTIFY.2018-01-11175009', 'NOTIFY.2018-01-11175010']" in mock_create_ticket.call_args[1]['message']
+    assert '2018-01-11/zips_sent' in mock_create_ticket.call_args[1]['message']
 
 
 @freeze_time('2018-01-11T23:00:00')