Some small changes to letter tasks and adding statsd

- Add statsd to collate-letter-pdfs-to-be-sent
- Add statsd to dao_get_letters_to_be_printed
- rename scheduled task for clarity

app/celery/letters_pdf_tasks.py

@@ -52,7 +52,7 @@ from app.models import (
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_VALIDATION_FAILED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
-    POSTAGE_TYPES)
+)
 from app.cronitor import cronitor
 
 
@@ -120,6 +120,7 @@ def get_letters_pdf(template, contact_block, filename, values):
 
 @notify_celery.task(name='collate-letter-pdfs-to-be-sent')
 @cronitor("collate-letter-pdfs-to-be-sent")
+@statsd(namespace="tasks")
 def collate_letter_pdfs_to_be_sent():
     """
     Finds all letters which are still waiting to be sent to DVLA for printing
@@ -136,55 +137,51 @@ def collate_letter_pdfs_to_be_sent():
         hour=17, minute=30, second=0, microsecond=0
     )
 
-    for postage in POSTAGE_TYPES:
-        letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
+    letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
 
-        for i, letters in enumerate(group_letters(letters_to_print)):
-            filenames = [letter['Key'] for letter in letters]
+    for i, letters in enumerate(group_letters(letters_to_print)):
+        filenames = [letter['Key'] for letter in letters]
 
-            hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
-            # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-            dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
-                date=print_run_deadline.strftime("%Y-%m-%d"),
-                num=i + 1,
-                hash=hash
-            )
-
-            current_app.logger.info(
-                'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
-                    len(filenames),
-                    dvla_filename,
-                    sum(letter['Size'] for letter in letters)
-                )
-            )
-            notify_celery.send_task(
-                name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
-                kwargs={
-                    'filenames_to_zip': filenames,
-                    'upload_filename': dvla_filename
-                },
-                queue=QueueNames.PROCESS_FTP,
-                compression='zlib'
+        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
+            date=print_run_deadline.strftime("%Y-%m-%d"),
+            num=i + 1,
+            hash=hash
+        )
+
+        current_app.logger.info(
+            'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
+                len(filenames),
+                dvla_filename,
+                sum(letter['Size'] for letter in letters)
             )
+        )
+        notify_celery.send_task(
+            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+            kwargs={
+                'filenames_to_zip': filenames,
+                'upload_filename': dvla_filename
+            },
+            queue=QueueNames.PROCESS_FTP,
+            compression='zlib'
+        )
 
 
-def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
-    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
+def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
+    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
 
     letter_pdfs = []
     for letter in letters_awaiting_sending:
-        try:
-            letter_file_name = get_letter_pdf_filename(
-                reference=letter.reference,
-                crown=letter.service.crown,
-                sending_date=letter.created_at,
-                postage=letter.postage
-            )
-            letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
-            letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
-        except BotoClientError as e:
-            current_app.logger.exception(
-                f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}")
+        letter_file_name = get_letter_pdf_filename(
+            reference=letter.reference,
+            crown=letter.service.crown,
+            sending_date=letter.created_at,
+            postage=letter.postage
+        )
+
+        letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
+        letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
 
     return letter_pdfs
 

app/celery/scheduled_tasks.py

@@ -205,7 +205,7 @@ def replay_created_notifications():
 
         current_app.logger.info(msg)
         for letter in letters:
-            create_letters_pdf.apply_async([str(letter.id)], queue=QueueNames.LETTERS)
+            create_letters_pdf.apply_async([letter.id], queue=QueueNames.LETTERS)
 
 
 @notify_celery.task(name='check-precompiled-letter-state')
@@ -232,6 +232,13 @@ def check_precompiled_letter_state():
 @notify_celery.task(name='check-templated-letter-state')
 @statsd(namespace="tasks")
 def check_templated_letter_state():
+    # renaming task for clarity. Next deploy will remove this task.
+    check_letters_created_yesterday_are_sending()
+
+
+@notify_celery.task(name='check-letters-created-yesterday-are-sending')
+@statsd(namespace="tasks")
+def check_letters_created_yesterday_are_sending():
     letters = dao_old_letters_with_created_status()
 
     if len(letters) > 0:

app/dao/notifications_dao.py

@@ -731,7 +731,8 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-def dao_get_letters_to_be_printed(print_run_deadline, postage):
+@statsd(namespace="dao")
+def dao_get_letters_to_be_printed(print_run_deadline):
     """
     Return all letters created before the print run deadline that have not yet been sent
     """
@@ -739,8 +740,7 @@ def dao_get_letters_to_be_printed(print_run_deadline, postage):
         Notification.created_at < convert_bst_to_utc(print_run_deadline),
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.postage == postage
+        Notification.key_type == KEY_TYPE_NORMAL
     ).order_by(
         Notification.created_at
     ).all()

app/dao/service_contact_list_dao.py

@@ -1,25 +0,0 @@
-from app import db
-from app.models import ServiceContactList
-
-
-def dao_get_contact_list_by_id(service_id, contact_list_id):
-    contact_list = ServiceContactList.query.filter_by(
-        service_id=service_id,
-        id=contact_list_id
-    ).one()
-
-    return contact_list
-
-
-def dao_get_contact_lists(service_id):
-    contact_lists = ServiceContactList.query.filter_by(
-        service_id=service_id
-    ).order_by(
-        ServiceContactList.created_at.desc()
-    )
-    return contact_lists.all()
-
-
-def save_service_contact_list(service_contact_list):
-    db.session.add(service_contact_list)
-    db.session.commit()

app/job/rest.py

@@ -141,6 +141,7 @@ def create_job(service_id):
         raise InvalidRequest("Create job is not allowed: service is inactive ", 403)
 
     data = request.get_json()
+
     data.update({
         "service": service_id
     })

app/models.py

@@ -1214,7 +1214,6 @@ class Job(db.Model):
         db.String(255), db.ForeignKey('job_status.name'), index=True, nullable=False, default='pending'
     )
     archived = db.Column(db.Boolean, nullable=False, default=False)
-    contact_list_id = db.Column(UUID(as_uuid=True), db.ForeignKey('service_contact_list.id'), nullable=True)
 
 
 VERIFY_CODE_TYPES = [EMAIL_TYPE, SMS_TYPE]
@@ -2120,31 +2119,3 @@ class ReturnedLetter(db.Model):
     notification_id = db.Column(UUID(as_uuid=True), unique=True, nullable=False)
     created_at = db.Column(db.DateTime, nullable=False)
     updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
-
-
-class ServiceContactList(db.Model):
-    __tablename__ = 'service_contact_list'
-
-    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    original_file_name = db.Column(db.String, nullable=False)
-    row_count = db.Column(db.Integer, nullable=False)
-    template_type = db.Column(template_types, nullable=False)
-    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False)
-    service = db.relationship(Service, backref=db.backref('contact_list'))
-    created_by = db.relationship('User')
-    created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=True)
-    created_at = db.Column(db.DateTime, nullable=False)
-    updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
-
-    def serialize(self):
-        created_at_in_bst = convert_utc_to_bst(self.created_at)
-        contact_list = {
-            "id": str(self.id),
-            "original_file_name": self.original_file_name,
-            "row_count": self.row_count,
-            "template_type": self.template_type,
-            "service_id": str(self.service_id),
-            "created_by": self.created_by.name,
-            "created_at": created_at_in_bst.strftime("%Y-%m-%d %H:%M:%S"),
-        }
-        return contact_list

app/notifications/validators.py

@@ -50,8 +50,7 @@ def check_service_over_daily_message_limit(key_type, service):
 
 def check_rate_limiting(service, api_key):
     check_service_over_api_rate_limit(service, api_key)
-    # Reduce queries to the notifications table
-    # check_service_over_daily_message_limit(api_key.key_type, service)
+    check_service_over_daily_message_limit(api_key.key_type, service)
 
 
 def check_template_is_for_notification_type(notification_type, template_type):

app/schemas.py

@@ -381,7 +381,6 @@ class JobSchema(BaseSchema):
         ServiceSchema, attribute="service", dump_to="service_name", only=["name"], dump_only=True)
 
     template_type = fields.Method('get_template_type', dump_only=True)
-    contact_list_id = field_for(models.Job, 'contact_list_id')
 
     def get_template_type(self, job):
         return job.template.template_type

app/service/rest.py

@@ -36,8 +36,6 @@ from app.dao.returned_letters_dao import (
     fetch_returned_letter_summary,
     fetch_returned_letters,
 )
-from app.dao.service_contact_list_dao import dao_get_contact_lists, save_service_contact_list, \
-    dao_get_contact_list_by_id
 from app.dao.service_data_retention_dao import (
     fetch_service_data_retention,
     fetch_service_data_retention_by_id,
@@ -97,14 +95,12 @@ from app.errors import (
 from app.letters.utils import letter_print_day
 from app.models import (
     KEY_TYPE_NORMAL, LETTER_TYPE, NOTIFICATION_CANCELLED, Permission, Service,
-    EmailBranding, LetterBranding,
-    ServiceContactList
+    EmailBranding, LetterBranding
 )
 from app.notifications.process_notifications import persist_notification, send_notification_to_queue
 from app.schema_validation import validate
 from app.service import statistics
 from app.service.send_pdf_letter_schema import send_pdf_letter_request
-from app.service.service_contact_list_schema import create_service_contact_list_schema
 from app.service.service_data_retention_schema import (
     add_service_data_retention_request,
     update_service_data_retention_request
@@ -1015,33 +1011,3 @@ def get_returned_letters(service_id):
          } for x in results]
 
     return jsonify(sorted(json_results, key=lambda i: i['created_at'], reverse=True))
-
-
-@service_blueprint.route('/<uuid:service_id>/contact-list', methods=['GET'])
-def get_contact_list(service_id):
-    contact_lists = dao_get_contact_lists(service_id)
-
-    return jsonify([x.serialize() for x in contact_lists])
-
-
-@service_blueprint.route('/<uuid:service_id>/contact-list/<uuid:contact_list_id>', methods=['GET'])
-def get_contact_list_by_id(service_id, contact_list_id):
-    contact_list = dao_get_contact_list_by_id(
-        service_id=service_id,
-        contact_list_id=contact_list_id
-    )
-
-    return jsonify(contact_list.serialize())
-
-
-@service_blueprint.route('/<uuid:service_id>/contact-list', methods=['POST'])
-def create_contact_list(service_id):
-    service_contact_list = validate(request.get_json(), create_service_contact_list_schema)
-    service_contact_list['created_by_id'] = service_contact_list.pop('created_by')
-    service_contact_list['created_at'] = datetime.utcnow()
-    service_contact_list['service_id'] = str(service_id)
-    list_to_save = ServiceContactList(**service_contact_list)
-
-    save_service_contact_list(list_to_save)
-
-    return jsonify(list_to_save.serialize()), 201

app/service/service_contact_list_schema.py

@@ -1,16 +0,0 @@
-from app.schema_validation.definitions import uuid
-
-create_service_contact_list_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
-    "description": "POST create service contact list schema",
-    "type": "object",
-    "title": "Create service contact list",
-    "properties": {
-        "id": uuid,
-        "original_file_name": {"type": "string"},
-        "row_count": {"type": "integer"},
-        "template_type": {"enum": ['email', 'sms']},
-        "created_by": uuid
-    },
-    "required": ["id", "original_file_name", "row_count", "template_type", "created_by"]
-}

migrations/versions/0318_service_contact_list.py

@@ -1,42 +0,0 @@
-"""
-
-Revision ID: 0318_service_contact_list
-Revises: 0317_uploads_for_all
-Create Date: 2020-03-12 15:44:30.784031
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-revision = '0318_service_contact_list'
-down_revision = '0317_uploads_for_all'
-
-
-def upgrade():
-    op.create_table(
-        'service_contact_list',
-        sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column('original_file_name', sa.String(), nullable=False),
-        sa.Column('row_count', sa.Integer(), nullable=False),
-        sa.Column('template_type', postgresql.ENUM(name='template_type', create_type=False), nullable=False),
-        sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column('created_by_id', postgresql.UUID(as_uuid=True), nullable=True),
-        sa.Column('created_at', sa.DateTime(), nullable=False),
-        sa.Column('updated_at', sa.DateTime(), nullable=True),
-        sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ),
-        sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
-        sa.PrimaryKeyConstraint('id')
-    )
-    op.create_index(op.f('ix_service_contact_list_created_by_id'), 'service_contact_list', ['created_by_id'], unique=False)
-    op.create_index(op.f('ix_service_contact_list_service_id'), 'service_contact_list', ['service_id'], unique=False)
-    op.add_column('jobs', sa.Column('contact_list_id', postgresql.UUID(as_uuid=True), nullable=True))
-    op.create_foreign_key('jobs_contact_list_id_fkey', 'jobs', 'service_contact_list', ['contact_list_id'], ['id'])
-
-
-def downgrade():
-    op.drop_constraint('jobs_contact_list_id_fkey', 'jobs', type_='foreignkey')
-    op.drop_column('jobs', 'contact_list_id')
-    op.drop_index(op.f('ix_service_contact_list_service_id'), table_name='service_contact_list')
-    op.drop_index(op.f('ix_service_contact_list_created_by_id'), table_name='service_contact_list')
-    op.drop_table('service_contact_list')

tests/app/celery/test_letters_pdf_tasks.py

@@ -315,48 +315,6 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
     ]
 
 
-@freeze_time('2020-02-17 18:00:00')
-def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
-    notify_api, mocker, sample_letter_template
-):
-    create_notification(
-        template=sample_letter_template,
-        status='created',
-        reference='ref0',
-        created_at=(datetime.now() - timedelta(hours=2))
-    )
-
-    create_notification(
-        template=sample_letter_template,
-        status='created',
-        reference='ref1',
-        created_at=(datetime.now() - timedelta(hours=3))
-    )
-    error_response = {
-        'Error': {
-            'Code': 'FileNotFound',
-            'Message': 'some error message from amazon',
-            'Type': 'Sender'
-        }
-    }
-    mock_head_s3_object = mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[
-        {'ContentLength': 2},
-        ClientError(error_response, "File not found")
-    ])
-
-    results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30))
-
-    assert mock_head_s3_object.call_count == 2
-    mock_head_s3_object.assert_has_calls(
-        [
-            call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'),
-            call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'),
-        ]
-    )
-
-    assert results == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}]
-
-
 @pytest.mark.parametrize('time_to_run_task', [
     "2020-02-17 18:00:00",  # after 5:30pm
     "2020-02-18 02:00:00",  # the next day after midnight, before 5:30pm we expect the same results

tests/app/celery/test_scheduled_tasks.py

@@ -15,10 +15,10 @@ from app.celery.scheduled_tasks import (
     send_scheduled_notifications,
     replay_created_notifications,
     check_precompiled_letter_state,
-    check_templated_letter_state,
     check_for_missing_rows_in_completed_jobs,
     check_for_services_with_high_failure_rates_or_sending_to_tv_numbers,
     switch_current_sms_provider_on_slow_delivery,
+    check_letters_created_yesterday_are_sending
 )
 from app.config import QueueNames, TaskNames, Config
 from app.dao.jobs_dao import dao_get_job_by_id
@@ -329,8 +329,8 @@ def test_replay_created_notifications_create_letters_pdf_tasks_for_letters_not_r
 
     replay_created_notifications()
 
-    calls = [call([str(notification_1.id)], queue=QueueNames.LETTERS),
-             call([str(notification_2.id)], queue=QueueNames.LETTERS),
+    calls = [call([notification_1.id], queue=QueueNames.LETTERS),
+             call([notification_2.id], queue=QueueNames.LETTERS),
              ]
     mock_task.assert_has_calls(calls, any_order=True)
 
@@ -385,7 +385,7 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template):
 
 
 @freeze_time("2019-05-30 14:00:00")
-def test_check_templated_letter_state_during_bst(mocker, sample_letter_template):
+def test_check_letters_created_yesterday_are_sending(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
     mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
@@ -396,7 +396,7 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
     create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
     create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))
 
-    check_templated_letter_state()
+    check_letters_created_yesterday_are_sending()
 
     message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
               "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
@@ -410,7 +410,7 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
 
 
 @freeze_time("2019-01-30 14:00:00")
-def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
+def test_check_letters_created_yesterday_are_sending_during_utc(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
     mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
 
@@ -421,7 +421,7 @@ def test_check_templated_letter_state_during_utc(mocker, sample_letter_template)
     create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
     create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))
 
-    check_templated_letter_state()
+    check_letters_created_yesterday_are_sending()
 
     message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
               "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)

tests/app/dao/test_service_contact_list_dao.py

@@ -1,10 +0,0 @@
-from app.dao.service_contact_list_dao import dao_get_contact_lists
-from tests.app.db import create_service_contact_list
-
-
-def test_dao_get_contact_lists(notify_db_session):
-    contact_list = create_service_contact_list()
-    fetched_list = dao_get_contact_lists(contact_list.service_id)
-
-    assert len(fetched_list) == 1
-    assert fetched_list[0] == contact_list

tests/app/db.py

@@ -60,8 +60,7 @@ from app.models import (
     LetterBranding,
     Domain,
     NotificationHistory,
-    ReturnedLetter,
-    ServiceContactList
+    ReturnedLetter
 )
 
 
@@ -966,26 +965,3 @@ def create_returned_letter(service=None, reported_at=None, notification_id=None)
     db.session.add(returned_letter)
     db.session.commit()
     return returned_letter
-
-
-def create_service_contact_list(
-    service=None,
-    original_file_name='EmergencyContactList.xls',
-    row_count=100,
-    template_type='email',
-    created_by_id=None
-):
-    if not service:
-        service = create_service(service_name='service for contact list', user=create_user())
-
-    contact_list = ServiceContactList(
-        service_id=service.id,
-        original_file_name=original_file_name,
-        row_count=row_count,
-        template_type=template_type,
-        created_by_id=created_by_id or service.users[0].id,
-        created_at=datetime.utcnow(),
-    )
-    db.session.add(contact_list)
-    db.session.commit()
-    return contact_list

tests/app/job/test_rest.py

@@ -13,12 +13,7 @@ from app.models import JOB_STATUS_TYPES, JOB_STATUS_PENDING
 
 from tests import create_authorization_header
 from tests.conftest import set_config
-from tests.app.db import (
-    create_ft_notification_status,
-    create_job,
-    create_notification,
-    create_service_contact_list
-)
+from tests.app.db import create_ft_notification_status, create_job, create_notification
 
 
 def test_get_job_with_invalid_service_id_returns404(client, sample_service):
@@ -238,30 +233,6 @@ def test_create_scheduled_job(client, sample_template, mocker, fake_uuid):
     assert resp_json['data']['notification_count'] == 1
 
 
-def test_create_job_with_contact_list_id(client, mocker, sample_template, fake_uuid):
-    mocker.patch('app.celery.tasks.process_job.apply_async')
-    mocker.patch('app.job.rest.get_job_metadata_from_s3', return_value={
-        'template_id': str(sample_template.id)
-    })
-    contact_list = create_service_contact_list()
-    data = {
-        'id': fake_uuid,
-        'valid': 'True',
-        'original_file_name': contact_list.original_file_name,
-        'created_by': str(sample_template.service.users[0].id),
-        'notification_count': 100,
-        'contact_list_id': str(contact_list.id),
-    }
-    response = client.post(
-        f'/service/{sample_template.service_id}/job',
-        data=json.dumps(data),
-        headers=[('Content-Type', 'application/json'), create_authorization_header()])
-    resp_json = response.get_json()
-    assert response.status_code == 201
-    assert resp_json['data']['contact_list_id'] == str(contact_list.id)
-    assert resp_json['data']['original_file_name'] == 'EmergencyContactList.xls'
-
-
 def test_create_job_returns_403_if_service_is_not_active(client, fake_uuid, sample_service, mocker):
     sample_service.active = False
     mock_job_dao = mocker.patch("app.dao.jobs_dao.dao_create_job")
@@ -712,7 +683,6 @@ def test_get_jobs(admin_request, sample_template):
         'template_type': 'sms',
         'template_version': 1,
         'updated_at': None,
-        'contact_list_id': None
     }
 
 

tests/app/notifications/rest/test_send_notification.py

@@ -18,7 +18,7 @@ from app.dao.services_dao import dao_update_service
 from app.dao.api_key_dao import save_model_api_key
 from app.errors import InvalidRequest
 from app.models import Template
-from app.v2.errors import RateLimitError
+from app.v2.errors import RateLimitError, TooManyRequestsError
 
 from tests import create_authorization_header
 from tests.app.db import (
@@ -404,6 +404,69 @@ def test_should_allow_valid_email_notification(notify_api, sample_email_template
             assert response_data['template_version'] == sample_email_template.version
 
 
+@freeze_time("2016-01-01 12:00:00.061258")
+def test_should_block_api_call_if_over_day_limit_for_live_service(
+        notify_db_session,
+        notify_api,
+        mocker):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            mocker.patch(
+                'app.notifications.validators.check_service_over_daily_message_limit',
+                side_effect=TooManyRequestsError(1)
+            )
+            mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+            service = create_service(message_limit=1)
+            email_template = create_template(service, template_type=EMAIL_TYPE)
+            create_notification(template=email_template)
+
+            data = {
+                'to': 'ok@ok.com',
+                'template': str(email_template.id)
+            }
+
+            auth_header = create_authorization_header(service_id=service.id)
+
+            response = client.post(
+                path='/notifications/email',
+                data=json.dumps(data),
+                headers=[('Content-Type', 'application/json'), auth_header])
+            json.loads(response.get_data(as_text=True))
+            assert response.status_code == 429
+
+
+@freeze_time("2016-01-01 12:00:00.061258")
+def test_should_block_api_call_if_over_day_limit_for_restricted_service(
+        notify_db_session,
+        notify_api,
+        mocker):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+            mocker.patch(
+                'app.notifications.validators.check_service_over_daily_message_limit',
+                side_effect=TooManyRequestsError(1)
+            )
+            service = create_service(restricted=True, message_limit=1)
+            email_template = create_template(service, template_type=EMAIL_TYPE)
+            create_notification(template=email_template)
+
+            data = {
+                'to': 'ok@ok.com',
+                'template': str(email_template.id)
+            }
+
+            auth_header = create_authorization_header(service_id=service.id)
+
+            response = client.post(
+                path='/notifications/email',
+                data=json.dumps(data),
+                headers=[('Content-Type', 'application/json'), auth_header])
+            json.loads(response.get_data(as_text=True))
+
+            assert response.status_code == 429
+
+
 @pytest.mark.parametrize('restricted', [True, False])
 @freeze_time("2016-01-01 12:00:00.061258")
 def test_should_allow_api_call_if_under_day_limit_regardless_of_type(

tests/app/service/test_service_contact_list_rest.py

@@ -1,115 +0,0 @@
-import uuid
-
-from app.models import ServiceContactList
-from tests.app.db import create_service_contact_list, create_service
-
-
-def test_create_service_contact_list(sample_service, admin_request):
-    data = {
-        "id": str(uuid.uuid4()),
-        "row_count": 100,
-        "original_file_name": "staff_emergency_list.xls",
-        "template_type": 'email',
-        "created_by": str(sample_service.users[0].id)
-    }
-
-    response = admin_request.post(
-        'service.create_contact_list',
-        _data=data,
-        service_id=sample_service.id,
-        _expected_status=201
-    )
-
-    assert response['id'] == data['id']
-    assert response['original_file_name'] == 'staff_emergency_list.xls'
-    assert response['row_count'] == 100
-    assert response['template_type'] == 'email'
-    assert response['service_id'] == str(sample_service.id)
-    assert response['created_at']
-
-    db_results = ServiceContactList.query.all()
-    assert len(db_results) == 1
-    assert str(db_results[0].id) == data['id']
-
-
-def test_create_service_contact_list_cannot_save_type_letter(sample_service, admin_request):
-    data = {
-        "id": str(uuid.uuid4()),
-        "row_count": 100,
-        "original_file_name": "staff_emergency_list.xls",
-        "template_type": 'letter',
-        "created_by": str(sample_service.users[0].id)
-    }
-
-    response = admin_request.post(
-        'service.create_contact_list',
-        _data=data,
-        service_id=sample_service.id,
-        _expected_status=400
-    )
-    assert response['errors'][0]['message'] == "template_type letter is not one of [email, sms]"
-
-
-def test_get_contact_list(admin_request, notify_db_session):
-    contact_list = create_service_contact_list()
-
-    response = admin_request.get(
-        'service.get_contact_list',
-        service_id=contact_list.service_id
-    )
-
-    assert len(response) == 1
-    assert response[0] == contact_list.serialize()
-
-
-def test_get_contact_list_returns_for_service(admin_request, notify_db_session):
-    service_1 = create_service(service_name='Service under test')
-    service_2 = create_service(service_name='Service should return results')
-
-    expected_list_1 = create_service_contact_list(service=service_1)
-    expected_list_2 = create_service_contact_list(service=service_1)
-    # not included in results
-    create_service_contact_list(service=service_2)
-
-    response = admin_request.get(
-        'service.get_contact_list',
-        service_id=service_1.id
-    )
-
-    assert len(response) == 2
-    assert response[0] == expected_list_2.serialize()
-    assert response[1] == expected_list_1.serialize()
-
-
-def test_dao_get_contact_list_by_id(admin_request, sample_service):
-    service_1 = create_service(service_name='Service under test')
-
-    expected_list_1 = create_service_contact_list(service=service_1)
-    create_service_contact_list(service=service_1)
-
-    response = admin_request.get(
-        'service.get_contact_list_by_id',
-        service_id=service_1.id,
-        contact_list_id=expected_list_1.id
-    )
-
-    assert response == expected_list_1.serialize()
-
-
-def test_dao_get_contact_list_by_id_does_not_return_if_contact_list_id_for_another_service(
-    admin_request, sample_service
-):
-    service_1 = create_service(service_name='Service requesting list')
-    service_2 = create_service(service_name='Service that owns the list')
-
-    create_service_contact_list(service=service_1)
-    list_2 = create_service_contact_list(service=service_2)
-
-    response = admin_request.get(
-        'service.get_contact_list_by_id',
-        service_id=service_1.id,
-        contact_list_id=list_2.id,
-        _expected_status=404
-    )
-
-    assert response['message'] == "No result found"