more wip

app/celery/letters_pdf_tasks.py

@@ -42,7 +42,6 @@ from app.models import (
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_VALIDATION_FAILED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
-    LETTER_TYPE
 )
 from app.cronitor import cronitor
 
@@ -217,7 +216,7 @@ def group_letters(letter_pdfs):
 def sanitise_letter(self, filename):
     try:
         reference = get_reference_from_filename(filename)
-        notification = dao_get_notification_by_reference(reference=reference, notification_type=LETTER_TYPE)
+        notification = dao_get_notification_by_reference(reference)
 
         current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename))
 
@@ -353,7 +352,7 @@ def _move_invalid_letter_and_update_status(
 def process_virus_scan_failed(filename):
     move_failed_pdf(filename, ScanErrorType.FAILURE)
     reference = get_reference_from_filename(filename)
-    notification = dao_get_notification_by_reference(reference=reference, notification_type=LETTER_TYPE)
+    notification = dao_get_notification_by_reference(reference)
     updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED, billable_units=0)
 
     if updated_count != 1:
@@ -372,7 +371,7 @@ def process_virus_scan_failed(filename):
 def process_virus_scan_error(filename):
     move_failed_pdf(filename, ScanErrorType.ERROR)
     reference = get_reference_from_filename(filename)
-    notification = dao_get_notification_by_reference(reference=reference, notification_type=LETTER_TYPE)
+    notification = dao_get_notification_by_reference(reference)
     updated_count = update_letter_pdf_status(reference, NOTIFICATION_TECHNICAL_FAILURE, billable_units=0)
 
     if updated_count != 1:

app/celery/process_ses_receipts_tasks.py

@@ -10,7 +10,7 @@ from app import notify_celery, statsd_client
 from app.config import QueueNames
 from app.clients.email.aws_ses import get_aws_responses
 from app.dao import notifications_dao
-from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING, EMAIL_TYPE
+from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING
 
 from app.notifications.notifications_ses_callback import (
     determine_notification_bounce_type,
@@ -39,9 +39,7 @@ def process_ses_results(self, response):
         reference = ses_message['mail']['messageId']
 
         try:
-            notification = notifications_dao.dao_get_notification_or_history_by_reference(
-                reference=reference, notification_type=EMAIL_TYPE
-            )
+            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
         except NoResultFound:
             message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
             if datetime.utcnow() - message_time < timedelta(minutes=5):

app/celery/tasks.py

@@ -536,7 +536,7 @@ def update_letter_notification(filename, temporary_failures, update):
 
 
 def check_billable_units(notification_update):
-    notification = dao_get_notification_or_history_by_reference(notification_update.reference, LETTER_TYPE)
+    notification = dao_get_notification_or_history_by_reference(notification_update.reference)
 
     if int(notification_update.page_count) != notification.billable_units:
         msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(

app/dao/notifications_dao.py

@@ -650,29 +650,33 @@ def dao_get_notifications_by_recipient_or_reference(
 
 
 @statsd(namespace="dao")
-def dao_get_notification_by_reference(reference, notification_type):
+def dao_get_notification_by_reference(reference):
     return Notification.query.filter(
-        Notification.reference == reference,
-        Notification.notification_type == notification_type
+        Notification.reference == reference
     ).one()
 
 
 @statsd(namespace="dao")
-def dao_get_notification_or_history_by_reference(reference, notification_type):
+def dao_get_notification_or_history_by_reference(reference):
     try:
         # This try except is necessary because in test keys and research mode does not create notification history.
         # Otherwise we could just search for the NotificationHistory object
         return Notification.query.filter(
-            Notification.reference == reference,
-            Notification.notification_type == notification_type
+            Notification.reference == reference
         ).one()
     except NoResultFound:
         return NotificationHistory.query.filter(
-            NotificationHistory.reference == reference,
-            NotificationHistory.notification_type == notification_type
+            NotificationHistory.reference == reference
         ).one()
 
 
+@statsd(namespace="dao")
+def dao_get_notifications_by_references(references):
+    return Notification.query.filter(
+        Notification.reference.in_(references)
+    ).all()
+
+
 @statsd(namespace="dao")
 def dao_created_scheduled_notification(scheduled_notification):
     db.session.add(scheduled_notification)

app/notifications/notifications_ses_callback.py

@@ -5,7 +5,7 @@ from app.dao.notifications_dao import dao_get_notification_or_history_by_referen
 from app.dao.service_callback_api_dao import (
     get_service_delivery_status_callback_api_for_service, get_service_complaint_callback_api_for_service
 )
-from app.models import Complaint, EMAIL_TYPE
+from app.models import Complaint
 from app.celery.service_callback_tasks import (
     send_delivery_status_to_service,
     send_complaint_to_service,
@@ -33,7 +33,7 @@ def handle_complaint(ses_message):
     except KeyError as e:
         current_app.logger.exception("Complaint from SES failed to get reference from message", e)
         return
-    notification = dao_get_notification_or_history_by_reference(reference, EMAIL_TYPE)
+    notification = dao_get_notification_or_history_by_reference(reference)
     ses_complaint = ses_message.get('complaint', None)
 
     complaint = Complaint(

application.py

@@ -2,9 +2,12 @@
 from __future__ import print_function
 
 from flask import Flask
+import psycogreen.eventlet
 
 from app import create_app
 
+psycogreen.eventlet.patch_psycopg()
+
 application = Flask('app')
 
 create_app(application)

gunicorn_config.py

@@ -6,7 +6,7 @@ import gunicorn
 from gds_metrics.gunicorn import child_exit  # noqa
 
 workers = 4
-worker_class = "eventlet"
+worker_class = "gevent"
 worker_connections = 256
 errorlog = "/home/vcap/logs/gunicorn_error.log"
 bind = "0.0.0.0:{}".format(os.getenv("PORT"))

manifest.yml.j2

@@ -1,6 +1,6 @@
 {%- set app_vars = {
-  'notify-api': {
-    'NOTIFY_APP_NAME': 'api',
+  'notify-api-canary1': {
+    'NOTIFY_APP_NAME': 'api-canary1',
     'disk_quota': '2G',
     'sqlalchemy_pool_size': 30,
     'routes': {
@@ -12,8 +12,8 @@
     'health-check-invocation-timeout': 3,
     'instances': {
       'preview': None,
-      'staging': None,
-      'production': 25
+      'staging': 1,
+      'production': 1
     },
   },
   'notify-api-db-migration': {

requirements-app.txt

@@ -10,13 +10,14 @@ Flask-Migrate==2.5.3
 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
 Flask==1.1.2
 click-datetime==0.2
-eventlet==0.25.2
+gevent==20.6.1
 gunicorn==20.0.4
 iso8601==0.1.12
 itsdangerous==1.1.0
 jsonschema==3.2.0
 marshmallow-sqlalchemy==0.23.0
 marshmallow==2.21.0 # pyup: <3 # v3 throws errors
+psycogreen==1.0.2
 psycopg2-binary==2.8.5
 PyJWT==1.7.1
 SQLAlchemy==1.3.17

requirements.txt

@@ -12,13 +12,14 @@ Flask-Migrate==2.5.3
 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
 Flask==1.1.2
 click-datetime==0.2
-eventlet==0.25.2
+gevent==20.6.1
 gunicorn==20.0.4
 iso8601==0.1.12
 itsdangerous==1.1.0
 jsonschema==3.2.0
 marshmallow-sqlalchemy==0.23.0
 marshmallow==2.21.0 # pyup: <3 # v3 throws errors
+psycogreen==1.0.2
 psycopg2-binary==2.8.5
 PyJWT==1.7.1
 SQLAlchemy==1.3.17
@@ -39,19 +40,18 @@ alembic==1.4.2
 amqp==1.4.9
 anyjson==0.3.3
 attrs==19.3.0
-awscli==1.18.75
+awscli==1.18.80
 bcrypt==3.1.7
 billiard==3.3.0.23
 bleach==3.1.4
 blinker==1.4
 boto==2.49.0
 boto3==1.10.38
-botocore==1.16.25
+botocore==1.17.3
 certifi==2020.4.5.2
 chardet==3.0.4
 click==7.1.2
 colorama==0.4.3
-dnspython==1.16.0
 docutils==0.15.2
 flask-redis==0.4.0
 future==0.18.2
@@ -88,3 +88,5 @@ urllib3==1.25.9
 webencodings==0.5.1
 Werkzeug==1.0.1
 zipp==3.1.0
+zope.event==4.4
+zope.interface==5.1.0

scripts/paas_app_wrapper.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 case $NOTIFY_APP_NAME in
-  api)
+  api|api-canary1)
     unset GUNICORN_CMD_ARGS
     exec scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
     ;;

tests/app/dao/notification_dao/test_notification_dao.py

@@ -28,6 +28,7 @@ from app.dao.notifications_dao import (
     update_notification_status_by_id,
     update_notification_status_by_reference,
     dao_get_notification_by_reference,
+    dao_get_notifications_by_references,
     dao_get_notification_or_history_by_reference,
     notifications_not_yet_sent,
 )
@@ -1612,7 +1613,7 @@ def test_dao_update_notifications_by_reference_updates_history_when_one_of_two_n
 
 def test_dao_get_notification_by_reference_with_one_match_returns_notification(sample_letter_template, notify_db):
     create_notification(template=sample_letter_template, reference='REF1')
-    notification = dao_get_notification_by_reference('REF1', 'letter')
+    notification = dao_get_notification_by_reference('REF1')
 
     assert notification.reference == 'REF1'
 
@@ -1622,25 +1623,30 @@ def test_dao_get_notification_by_reference_with_multiple_matches_raises_error(sa
     create_notification(template=sample_letter_template, reference='REF1')
 
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_by_reference('REF1', 'letter')
+        dao_get_notification_by_reference('REF1')
 
 
 def test_dao_get_notification_by_reference_with_no_matches_raises_error(notify_db):
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_by_reference('REF1', 'email')
+        dao_get_notification_by_reference('REF1')
 
 
-def test_dao_get_notification_by_reference_with_no_matches_for_type_raises_error(sample_email_template):
-    create_notification(template=sample_email_template, reference='REF1')
-    with pytest.raises(SQLAlchemyError):
-        dao_get_notification_by_reference('REF1', 'letter')
+def test_dao_get_notifications_by_references(sample_template):
+    create_notification(template=sample_template, reference='noref')
+    notification_1 = create_notification(template=sample_template, reference='ref')
+    notification_2 = create_notification(template=sample_template, reference='ref')
+
+    notifications = dao_get_notifications_by_references(['ref'])
+    assert len(notifications) == 2
+    assert notifications[0].id in [notification_1.id, notification_2.id]
+    assert notifications[1].id in [notification_1.id, notification_2.id]
 
 
 def test_dao_get_notification_or_history_by_reference_with_one_match_returns_notification(
         sample_letter_template
 ):
     create_notification(template=sample_letter_template, reference='REF1')
-    notification = dao_get_notification_or_history_by_reference('REF1', 'letter')
+    notification = dao_get_notification_or_history_by_reference('REF1')
 
     assert notification.reference == 'REF1'
 
@@ -1652,18 +1658,12 @@ def test_dao_get_notification_or_history_by_reference_with_multiple_matches_rais
     create_notification(template=sample_letter_template, reference='REF1')
 
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_or_history_by_reference('REF1', 'letter')
+        dao_get_notification_or_history_by_reference('REF1')
 
 
-def test_dao_get_notification_or_history_by_reference_with_no_matches_raises_error(sample_letter_template):
-    create_notification(template=sample_letter_template, reference='REF1')
+def test_dao_get_notification_or_history_by_reference_with_no_matches_raises_error(notify_db):
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_or_history_by_reference('REF1', 'email')
-
-
-def test_dao_get_notification_or_history_by_reference_with_no_matches_for_type_raises_error(notify_db):
-    with pytest.raises(SQLAlchemyError):
-        dao_get_notification_or_history_by_reference('REF1', 'email')
+        dao_get_notification_or_history_by_reference('REF1')
 
 
 @pytest.mark.parametrize("notification_type",