Merge branch 'master' into add-http-healthchecks

app/__init__.py

@@ -219,8 +219,6 @@ def init_app(app):
     def record_user_agent():
         statsd_client.incr("user-agent.{}".format(process_user_agent(request.headers.get('User-Agent', None))))
 
-    app.before_request(request_helper.check_proxy_header_before_request)
-
     @app.before_request
     def record_request_details():
         g.start = monotonic()

app/authentication/auth.py

@@ -1,6 +1,7 @@
 from flask import request, _request_ctx_stack, current_app, g
 from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
 from notifications_python_client.errors import TokenDecodeError, TokenExpiredError, TokenIssuerError
+from notifications_utils import request_helper
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
 
@@ -48,6 +49,8 @@ def requires_no_auth():
 
 
 def requires_admin_auth():
+    request_helper.check_proxy_header_before_request()
+
     auth_token = get_auth_token(request)
     client = __get_token_issuer(auth_token)
 
@@ -59,6 +62,8 @@ def requires_admin_auth():
 
 
 def requires_auth():
+    request_helper.check_proxy_header_before_request()
+
     auth_token = get_auth_token(request)
     client = __get_token_issuer(auth_token)
 

app/celery/letters_pdf_tasks.py

@@ -21,6 +21,7 @@ from app.dao.notifications_dao import (
     dao_get_notifications_by_references,
     dao_update_notifications_by_reference,
 )
+from app.errors import VirusScanError
 from app.letters.utils import (
     get_reference_from_filename,
     move_scanned_pdf_to_test_or_live_pdf_bucket,
@@ -162,9 +163,9 @@ def letter_in_created_state(filename):
 
 @notify_celery.task(name='process-virus-scan-passed')
 def process_virus_scan_passed(filename):
-    current_app.logger.info('Virus scan passed: {}'.format(filename))
     reference = get_reference_from_filename(filename)
     notification = dao_get_notification_by_reference(reference)
+    current_app.logger.info('notification id {} Virus scan passed: {}'.format(notification.id, filename))
 
     is_test_key = notification.key_type == KEY_TYPE_TEST
     move_scanned_pdf_to_test_or_live_pdf_bucket(
@@ -179,9 +180,9 @@ def process_virus_scan_passed(filename):
 
 @notify_celery.task(name='process-virus-scan-failed')
 def process_virus_scan_failed(filename):
-    current_app.logger.exception('Virus scan failed: {}'.format(filename))
     move_failed_pdf(filename, ScanErrorType.FAILURE)
     reference = get_reference_from_filename(filename)
+    notification = dao_get_notification_by_reference(reference)
     updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED)
 
     if updated_count != 1:
@@ -191,12 +192,14 @@ def process_virus_scan_failed(filename):
             )
         )
 
+    raise VirusScanError('notification id {} Virus scan failed: {}'.format(notification.id, filename))
+
 
 @notify_celery.task(name='process-virus-scan-error')
 def process_virus_scan_error(filename):
-    current_app.logger.exception('Virus scan error: {}'.format(filename))
     move_failed_pdf(filename, ScanErrorType.ERROR)
     reference = get_reference_from_filename(filename)
+    notification = dao_get_notification_by_reference(reference)
     updated_count = update_letter_pdf_status(reference, NOTIFICATION_TECHNICAL_FAILURE)
 
     if updated_count != 1:
@@ -206,6 +209,8 @@ def process_virus_scan_error(filename):
             )
         )
 
+    raise VirusScanError('notification id {} Virus scan error: {}'.format(notification.id, filename))
+
 
 def update_letter_pdf_status(reference, status):
     return dao_update_notifications_by_reference(

app/celery/reporting_tasks.py

@@ -84,15 +84,18 @@ def create_nightly_billing(day_start=None):
 
         for data in transit_data:
             update_count = FactBilling.query.filter(
-                FactBilling.bst_date == process_day,
+                FactBilling.bst_date == datetime.date(process_day),
                 FactBilling.template_id == data.template_id,
+                FactBilling.service_id == data.service_id,
                 FactBilling.provider == data.sent_by,  # This could be zero - this is a bug that needs to be fixed.
                 FactBilling.rate_multiplier == data.rate_multiplier,
                 FactBilling.notification_type == data.notification_type,
+                FactBilling.international == data.international
             ).update(
                 {"notifications_sent": data.notifications_sent,
                  "billable_units": data.billable_units},
                 synchronize_session=False)
+
             if update_count == 0:
                 billing_record = FactBilling(
                     bst_date=process_day,

app/errors.py

@@ -10,6 +10,12 @@ from jsonschema import ValidationError as JsonSchemaValidationError
 from app.authentication.auth import AuthError
 
 
+class VirusScanError(Exception):
+    def __init__(self, message):
+
+        super().__init__(message)
+
+
 class InvalidRequest(Exception):
     code = None
     fields = []

app/template/rest.py

@@ -200,7 +200,6 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
     template = dao_get_template_by_id(notification.template_id)
 
     if template.is_precompiled_letter:
-
         try:
 
             pdf_file = get_letter_pdf(notification)
@@ -215,9 +214,9 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
         content = base64.b64encode(pdf_file).decode('utf-8')
 
         if file_type == 'png':
-
             try:
-                page_number = page if page else "0"
+                page_number = page if page else "1"
+
                 pdf_page = extract_page_from_pdf(BytesIO(pdf_file), int(page_number) - 1)
                 content = base64.b64encode(pdf_page).decode('utf-8')
             except PdfReadError as e:
@@ -227,12 +226,12 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
                     status_code=500
                 )
 
-            url = '{}/precompiled-preview.png'.format(
-                current_app.config['TEMPLATE_PREVIEW_API_HOST']
+            url = '{}/precompiled-preview.png{}'.format(
+                current_app.config['TEMPLATE_PREVIEW_API_HOST'],
+                '?hide_notify=true' if page_number == '1' else ''
             )
 
             content = _get_png_preview(url, content, notification.id, json=False)
-
     else:
 
         template_for_letter_print = {
@@ -256,7 +255,6 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
             file_type,
             '?page={}'.format(page) if page else ''
         )
-
         content = _get_png_preview(url, data, notification.id, json=True)
 
     return jsonify({"content": content})