diff --git a/app/__init__.py b/app/__init__.py index b4a470ac0..1f94c697e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -4,7 +4,7 @@ import string import uuid from flask import _request_ctx_stack, request, g, jsonify -from flask_sqlalchemy import SQLAlchemy +from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy from flask_marshmallow import Marshmallow from flask_migrate import Migrate from time import monotonic @@ -27,6 +27,19 @@ from app.encryption import Encryption DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" DATE_FORMAT = "%Y-%m-%d" + +class SQLAlchemy(_SQLAlchemy): + """We need to subclass SQLAlchemy in order to override create_engine options""" + + def apply_driver_hacks(self, app, info, options): + super().apply_driver_hacks(app, info, options) + if 'connect_args' not in options: + options['connect_args'] = {} + options['connect_args']["options"] = "-c statement_timeout={}".format( + int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000 + ) + + db = SQLAlchemy() migrate = Migrate() ma = Marshmallow() diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index a40f2bd91..a57be4044 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -188,7 +188,12 @@ def process_virus_scan_passed(self, filename): scan_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename) old_pdf = scan_pdf_object.get()['Body'].read() - billable_units = _get_page_count(notification, old_pdf) + try: + billable_units = _get_page_count(notification, old_pdf) + except PdfReadError: + _move_invalid_letter_and_update_status(notification.reference, filename, scan_pdf_object) + return + new_pdf = _sanitise_precompiled_pdf(self, notification, old_pdf) # TODO: Remove this once CYSP update their template to not cross over the margins @@ -198,12 +203,7 @@ def process_virus_scan_passed(self, filename): if not new_pdf: current_app.logger.info('Invalid precompiled pdf received {} ({})'.format(notification.id, filename)) - - notification.status = NOTIFICATION_VALIDATION_FAILED - dao_update_notification(notification) - - move_scan_to_invalid_pdf_bucket(filename) - scan_pdf_object.delete() + _move_invalid_letter_and_update_status(notification.reference, filename, scan_pdf_object) return else: current_app.logger.info( @@ -233,14 +233,19 @@ def _get_page_count(notification, old_pdf): return billable_units except PdfReadError as e: current_app.logger.exception(msg='Invalid PDF received for notification_id: {}'.format(notification.id)) - update_letter_pdf_status( - reference=notification.reference, - status=NOTIFICATION_VALIDATION_FAILED, - billable_units=0 - ) raise e +def _move_invalid_letter_and_update_status(notification_reference, filename, scan_pdf_object): + move_scan_to_invalid_pdf_bucket(filename) + scan_pdf_object.delete() + + update_letter_pdf_status( + reference=notification_reference, + status=NOTIFICATION_VALIDATION_FAILED, + billable_units=0) + + def _upload_pdf_to_test_or_live_pdf_bucket(pdf_data, filename, is_test_letter): target_bucket_config = 'TEST_LETTERS_BUCKET_NAME' if is_test_letter else 'LETTERS_PDF_BUCKET_NAME' target_bucket_name = current_app.config[target_bucket_config] @@ -261,7 +266,9 @@ def _sanitise_precompiled_pdf(self, notification, precompiled_pdf): current_app.config['TEMPLATE_PREVIEW_API_HOST'] ), data=precompiled_pdf, - headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])} + headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY']), + 'Service-ID': str(notification.service_id), + 'Notification-ID': str(notification.id)} ) resp.raise_for_status() return resp.content diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py new file mode 100644 index 000000000..e452befd1 --- /dev/null +++ b/app/celery/nightly_tasks.py @@ -0,0 +1,342 @@ +from datetime import ( + datetime, + timedelta +) + +import pytz +from flask import current_app +from notifications_utils.statsd_decorators import statsd +from sqlalchemy import func +from sqlalchemy.exc import SQLAlchemyError + +from app import notify_celery, performance_platform_client, zendesk_client +from app.aws import s3 +from app.celery.service_callback_tasks import ( + send_delivery_status_to_service, + create_delivery_status_callback_data, +) +from app.config import QueueNames +from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago +from app.dao.jobs_dao import ( + dao_get_jobs_older_than_data_retention, + dao_archive_job +) +from app.dao.notifications_dao import ( + dao_timeout_notifications, + delete_notifications_created_more_than_a_week_ago_by_type, +) +from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service +from app.exceptions import NotificationTechnicalFailureException +from app.models import ( + Notification, + NOTIFICATION_SENDING, + EMAIL_TYPE, + SMS_TYPE, + LETTER_TYPE, + KEY_TYPE_NORMAL +) +from app.performance_platform import total_sent_notifications, processing_time +from app.cronitor import cronitor + + +@notify_celery.task(name="remove_sms_email_jobs") +@cronitor("remove_sms_email_jobs") +@statsd(namespace="tasks") +def remove_sms_email_csv_files(): + _remove_csv_files([EMAIL_TYPE, SMS_TYPE]) + + +@notify_celery.task(name="remove_letter_jobs") +@cronitor("remove_letter_jobs") +@statsd(namespace="tasks") +def remove_letter_csv_files(): + _remove_csv_files([LETTER_TYPE]) + + +def _remove_csv_files(job_types): + jobs = dao_get_jobs_older_than_data_retention(notification_types=job_types) + for job in jobs: + s3.remove_job_from_s3(job.service_id, job.id) + dao_archive_job(job) + current_app.logger.info("Job ID {} has been removed from s3.".format(job.id)) + + +@notify_celery.task(name="delete-sms-notifications") +@cronitor("delete-sms-notifications") +@statsd(namespace="tasks") +def delete_sms_notifications_older_than_seven_days(): + try: + start = datetime.utcnow() + deleted = delete_notifications_created_more_than_a_week_ago_by_type('sms') + current_app.logger.info( + "Delete {} job started {} finished {} deleted {} sms notifications".format( + 'sms', + start, + datetime.utcnow(), + deleted + ) + ) + except SQLAlchemyError: + current_app.logger.exception("Failed to delete sms notifications") + raise + + +@notify_celery.task(name="delete-email-notifications") +@cronitor("delete-email-notifications") +@statsd(namespace="tasks") +def delete_email_notifications_older_than_seven_days(): + try: + start = datetime.utcnow() + deleted = delete_notifications_created_more_than_a_week_ago_by_type('email') + current_app.logger.info( + "Delete {} job started {} finished {} deleted {} email notifications".format( + 'email', + start, + datetime.utcnow(), + deleted + ) + ) + except SQLAlchemyError: + current_app.logger.exception("Failed to delete email notifications") + raise + + +@notify_celery.task(name="delete-letter-notifications") +@cronitor("delete-letter-notifications") +@statsd(namespace="tasks") +def delete_letter_notifications_older_than_seven_days(): + try: + start = datetime.utcnow() + deleted = delete_notifications_created_more_than_a_week_ago_by_type('letter') + current_app.logger.info( + "Delete {} job started {} finished {} deleted {} letter notifications".format( + 'letter', + start, + datetime.utcnow(), + deleted + ) + ) + except SQLAlchemyError: + current_app.logger.exception("Failed to delete letter notifications") + raise + + +@notify_celery.task(name='timeout-sending-notifications') +@cronitor('timeout-sending-notifications') +@statsd(namespace="tasks") +def timeout_notifications(): + technical_failure_notifications, temporary_failure_notifications = \ + dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')) + + notifications = technical_failure_notifications + temporary_failure_notifications + for notification in notifications: + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api) + send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification], + queue=QueueNames.CALLBACKS) + + current_app.logger.info( + "Timeout period reached for {} notifications, status has been updated.".format(len(notifications))) + if technical_failure_notifications: + message = "{} notifications have been updated to technical-failure because they " \ + "have timed out and are still in created.Notification ids: {}".format( + len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications]) + raise NotificationTechnicalFailureException(message) + + +@notify_celery.task(name='send-daily-performance-platform-stats') +@cronitor('send-daily-performance-platform-stats') +@statsd(namespace="tasks") +def send_daily_performance_platform_stats(): + if performance_platform_client.active: + yesterday = datetime.utcnow() - timedelta(days=1) + send_total_sent_notifications_to_performance_platform(yesterday) + processing_time.send_processing_time_to_performance_platform() + + +def send_total_sent_notifications_to_performance_platform(day): + count_dict = total_sent_notifications.get_total_sent_notifications_for_day(day) + email_sent_count = count_dict.get('email').get('count') + sms_sent_count = count_dict.get('sms').get('count') + letter_sent_count = count_dict.get('letter').get('count') + start_date = count_dict.get('start_date') + + current_app.logger.info( + "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters" + .format(start_date, email_sent_count, sms_sent_count, letter_sent_count) + ) + + total_sent_notifications.send_total_notifications_sent_for_day_stats( + start_date, + 'sms', + sms_sent_count + ) + + total_sent_notifications.send_total_notifications_sent_for_day_stats( + start_date, + 'email', + email_sent_count + ) + + total_sent_notifications.send_total_notifications_sent_for_day_stats( + start_date, + 'letter', + letter_sent_count + ) + + +@notify_celery.task(name="delete-inbound-sms") +@cronitor("delete-inbound-sms") +@statsd(namespace="tasks") +def delete_inbound_sms_older_than_seven_days(): + try: + start = datetime.utcnow() + deleted = delete_inbound_sms_created_more_than_a_week_ago() + current_app.logger.info( + "Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format( + start, + datetime.utcnow(), + deleted + ) + ) + except SQLAlchemyError: + current_app.logger.exception("Failed to delete inbound sms notifications") + raise + + +@notify_celery.task(name="remove_transformed_dvla_files") +@cronitor("remove_transformed_dvla_files") +@statsd(namespace="tasks") +def remove_transformed_dvla_files(): + jobs = dao_get_jobs_older_than_data_retention(notification_types=[LETTER_TYPE]) + for job in jobs: + s3.remove_transformed_dvla_file(job.id) + current_app.logger.info("Transformed dvla file for job {} has been removed from s3.".format(job.id)) + + +# TODO: remove me, i'm not being run by anything +@notify_celery.task(name="delete_dvla_response_files") +@statsd(namespace="tasks") +def delete_dvla_response_files_older_than_seven_days(): + try: + start = datetime.utcnow() + bucket_objects = s3.get_s3_bucket_objects( + current_app.config['DVLA_RESPONSE_BUCKET_NAME'], + 'root/dispatch' + ) + older_than_seven_days = s3.filter_s3_bucket_objects_within_date_range(bucket_objects) + + for f in older_than_seven_days: + s3.remove_s3_object(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], f['Key']) + + current_app.logger.info( + "Delete dvla response files started {} finished {} deleted {} files".format( + start, + datetime.utcnow(), + len(older_than_seven_days) + ) + ) + except SQLAlchemyError: + current_app.logger.exception("Failed to delete dvla response files") + raise + + +@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending") +@cronitor("raise-alert-if-letter-notifications-still-sending") +@statsd(namespace="tasks") +def raise_alert_if_letter_notifications_still_sending(): + today = datetime.utcnow().date() + + # Do nothing on the weekend + if today.isoweekday() in [6, 7]: + return + + if today.isoweekday() in [1, 2]: + offset_days = 4 + else: + offset_days = 2 + still_sending = Notification.query.filter( + Notification.notification_type == LETTER_TYPE, + Notification.status == NOTIFICATION_SENDING, + Notification.key_type == KEY_TYPE_NORMAL, + func.date(Notification.sent_at) <= today - timedelta(days=offset_days) + ).count() + + if still_sending: + message = "There are {} letters in the 'sending' state from {}".format( + still_sending, + (today - timedelta(days=offset_days)).strftime('%A %d %B') + ) + # Only send alerts in production + if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: + zendesk_client.create_ticket( + subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']), + message=message, + ticket_type=zendesk_client.TYPE_INCIDENT + ) + else: + current_app.logger.info(message) + + +@notify_celery.task(name='raise-alert-if-no-letter-ack-file') +@cronitor('raise-alert-if-no-letter-ack-file') +@statsd(namespace="tasks") +def letter_raise_alert_if_no_ack_file_for_zip(): + # get a list of zip files since yesterday + zip_file_set = set() + + for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], + subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', + suffix='.TXT'): + subname = key.split('/')[-1] # strip subfolder in name + zip_file_set.add(subname.upper().rstrip('.TXT')) + + # get acknowledgement file + ack_file_set = set() + + yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format + + for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], + subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday): + ack_file_set.add(key) + + today_str = datetime.utcnow().strftime('%Y%m%d') + + ack_content_set = set() + for key in ack_file_set: + if today_str in key: + content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key) + for zip_file in content.split('\n'): # each line + s = zip_file.split('|') + ack_content_set.add(s[0].upper()) + + message = ( + "Letter ack file does not contain all zip files sent. " + "Missing ack for zip files: {}, " + "pdf bucket: {}, subfolder: {}, " + "ack bucket: {}" + ).format( + str(sorted(zip_file_set - ack_content_set)), + current_app.config['LETTERS_PDF_BUCKET_NAME'], + datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', + current_app.config['DVLA_RESPONSE_BUCKET_NAME'] + ) + # strip empty element before comparison + ack_content_set.discard('') + zip_file_set.discard('') + + if len(zip_file_set - ack_content_set) > 0: + if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: + zendesk_client.create_ticket( + subject="Letter acknowledge error", + message=message, + ticket_type=zendesk_client.TYPE_INCIDENT + ) + current_app.logger.error(message) + + if len(ack_content_set - zip_file_set) > 0: + current_app.logger.info( + "letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set) + ) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 4d14c0e64..80c5d1bc0 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -4,6 +4,7 @@ from flask import current_app from notifications_utils.statsd_decorators import statsd from app import notify_celery +from app.cronitor import cronitor from app.dao.fact_billing_dao import ( fetch_billing_data_for_day, update_fact_billing @@ -12,6 +13,7 @@ from app.dao.fact_notification_status_dao import fetch_notification_status_for_d @notify_celery.task(name="create-nightly-billing") +@cronitor("create-nightly-billing") @statsd(namespace="tasks") def create_nightly_billing(day_start=None): # day_start is a datetime.date() object. e.g. @@ -34,6 +36,7 @@ def create_nightly_billing(day_start=None): @notify_celery.task(name="create-nightly-notification-status") +@cronitor("create-nightly-notification-status") @statsd(namespace="tasks") def create_nightly_notification_status(day_start=None): # day_start is a datetime.date() object. e.g. diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 9ff9cb956..af072d91b 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -3,33 +3,20 @@ from datetime import ( timedelta ) -import pytz from flask import current_app from notifications_utils.statsd_decorators import statsd -from sqlalchemy import and_, func +from sqlalchemy import and_ from sqlalchemy.exc import SQLAlchemyError from app import notify_celery -from app import performance_platform_client, zendesk_client -from app.aws import s3 -from app.celery.service_callback_tasks import ( - send_delivery_status_to_service, - create_delivery_status_callback_data, -) from app.celery.tasks import process_job from app.config import QueueNames, TaskNames -from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago -from app.dao.jobs_dao import ( - dao_set_scheduled_jobs_to_pending, - dao_get_jobs_older_than_limited_by -) +from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending from app.dao.jobs_dao import dao_update_job from app.dao.notifications_dao import ( - dao_timeout_notifications, is_delivery_slow_for_provider, - delete_notifications_created_more_than_a_week_ago_by_type, dao_get_scheduled_notifications, set_scheduled_notification_to_processed, notifications_not_yet_sent @@ -38,38 +25,18 @@ from app.dao.provider_details_dao import ( get_current_provider, dao_toggle_sms_provider ) -from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service -from app.dao.services_dao import ( - dao_fetch_monthly_historical_stats_by_template -) -from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago -from app.exceptions import NotificationTechnicalFailureException from app.models import ( Job, - Notification, - NOTIFICATION_SENDING, - LETTER_TYPE, JOB_STATUS_IN_PROGRESS, JOB_STATUS_ERROR, SMS_TYPE, EMAIL_TYPE, - KEY_TYPE_NORMAL ) from app.notifications.process_notifications import send_notification_to_queue -from app.performance_platform import total_sent_notifications, processing_time from app.v2.errors import JobIncompleteError -@notify_celery.task(name="remove_csv_files") -@statsd(namespace="tasks") -def remove_csv_files(job_types): - jobs = dao_get_jobs_older_than_limited_by(job_types=job_types) - for job in jobs: - s3.remove_job_from_s3(job.service_id, job.id) - current_app.logger.info("Job ID {} has been removed from s3.".format(job.id)) - - @notify_celery.task(name="run-scheduled-jobs") @statsd(namespace="tasks") def run_scheduled_jobs(): @@ -111,63 +78,6 @@ def delete_verify_codes(): raise -@notify_celery.task(name="delete-sms-notifications") -@statsd(namespace="tasks") -def delete_sms_notifications_older_than_seven_days(): - try: - start = datetime.utcnow() - deleted = delete_notifications_created_more_than_a_week_ago_by_type('sms') - current_app.logger.info( - "Delete {} job started {} finished {} deleted {} sms notifications".format( - 'sms', - start, - datetime.utcnow(), - deleted - ) - ) - except SQLAlchemyError: - current_app.logger.exception("Failed to delete sms notifications") - raise - - -@notify_celery.task(name="delete-email-notifications") -@statsd(namespace="tasks") -def delete_email_notifications_older_than_seven_days(): - try: - start = datetime.utcnow() - deleted = delete_notifications_created_more_than_a_week_ago_by_type('email') - current_app.logger.info( - "Delete {} job started {} finished {} deleted {} email notifications".format( - 'email', - start, - datetime.utcnow(), - deleted - ) - ) - except SQLAlchemyError: - current_app.logger.exception("Failed to delete sms notifications") - raise - - -@notify_celery.task(name="delete-letter-notifications") -@statsd(namespace="tasks") -def delete_letter_notifications_older_than_seven_days(): - try: - start = datetime.utcnow() - deleted = delete_notifications_created_more_than_a_week_ago_by_type('letter') - current_app.logger.info( - "Delete {} job started {} finished {} deleted {} letter notifications".format( - 'letter', - start, - datetime.utcnow(), - deleted - ) - ) - except SQLAlchemyError: - current_app.logger.exception("Failed to delete sms notifications") - raise - - @notify_celery.task(name="delete-invitations") @statsd(namespace="tasks") def delete_invitations(): @@ -183,70 +93,6 @@ def delete_invitations(): raise -@notify_celery.task(name='timeout-sending-notifications') -@statsd(namespace="tasks") -def timeout_notifications(): - technical_failure_notifications, temporary_failure_notifications = \ - dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')) - - notifications = technical_failure_notifications + temporary_failure_notifications - for notification in notifications: - # queue callback task only if the service_callback_api exists - service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) - if service_callback_api: - encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api) - send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification], - queue=QueueNames.CALLBACKS) - - current_app.logger.info( - "Timeout period reached for {} notifications, status has been updated.".format(len(notifications))) - if technical_failure_notifications: - message = "{} notifications have been updated to technical-failure because they " \ - "have timed out and are still in created.Notification ids: {}".format( - len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications]) - raise NotificationTechnicalFailureException(message) - - -@notify_celery.task(name='send-daily-performance-platform-stats') -@statsd(namespace="tasks") -def send_daily_performance_platform_stats(): - if performance_platform_client.active: - yesterday = datetime.utcnow() - timedelta(days=1) - send_total_sent_notifications_to_performance_platform(yesterday) - processing_time.send_processing_time_to_performance_platform() - - -def send_total_sent_notifications_to_performance_platform(day): - count_dict = total_sent_notifications.get_total_sent_notifications_for_day(day) - email_sent_count = count_dict.get('email').get('count') - sms_sent_count = count_dict.get('sms').get('count') - letter_sent_count = count_dict.get('letter').get('count') - start_date = count_dict.get('start_date') - - current_app.logger.info( - "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters" - .format(start_date, email_sent_count, sms_sent_count, letter_sent_count) - ) - - total_sent_notifications.send_total_notifications_sent_for_day_stats( - start_date, - 'sms', - sms_sent_count - ) - - total_sent_notifications.send_total_notifications_sent_for_day_stats( - start_date, - 'email', - email_sent_count - ) - - total_sent_notifications.send_total_notifications_sent_for_day_stats( - start_date, - 'letter', - letter_sent_count - ) - - @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery') @statsd(namespace="tasks") def switch_current_sms_provider_on_slow_delivery(): @@ -254,117 +100,25 @@ def switch_current_sms_provider_on_slow_delivery(): Switch providers if there are at least two slow delivery notifications (more than four minutes) in the last ten minutes. Search from the time we last switched to the current provider. """ - functional_test_provider_service_id = current_app.config.get('FUNCTIONAL_TEST_PROVIDER_SERVICE_ID') - functional_test_provider_template_id = current_app.config.get('FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID') - - if functional_test_provider_service_id and functional_test_provider_template_id: - current_provider = get_current_provider('sms') - slow_delivery_notifications = is_delivery_slow_for_provider( - provider=current_provider.identifier, - threshold=2, - sent_at=max(datetime.utcnow() - timedelta(minutes=10), current_provider.updated_at), - delivery_time=timedelta(minutes=4), - service_id=functional_test_provider_service_id, - template_id=functional_test_provider_template_id - ) - - if slow_delivery_notifications: - current_app.logger.warning( - 'Slow delivery notifications detected for provider {}'.format( - current_provider.identifier - ) - ) - - dao_toggle_sms_provider(current_provider.identifier) - - -@notify_celery.task(name="delete-inbound-sms") -@statsd(namespace="tasks") -def delete_inbound_sms_older_than_seven_days(): - try: - start = datetime.utcnow() - deleted = delete_inbound_sms_created_more_than_a_week_ago() - current_app.logger.info( - "Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format( - start, - datetime.utcnow(), - deleted - ) - ) - except SQLAlchemyError: - current_app.logger.exception("Failed to delete inbound sms notifications") - raise - - -@notify_celery.task(name="remove_transformed_dvla_files") -@statsd(namespace="tasks") -def remove_transformed_dvla_files(): - jobs = dao_get_jobs_older_than_limited_by(job_types=[LETTER_TYPE]) - for job in jobs: - s3.remove_transformed_dvla_file(job.id) - current_app.logger.info("Transformed dvla file for job {} has been removed from s3.".format(job.id)) - - -@notify_celery.task(name="delete_dvla_response_files") -@statsd(namespace="tasks") -def delete_dvla_response_files_older_than_seven_days(): - try: - start = datetime.utcnow() - bucket_objects = s3.get_s3_bucket_objects( - current_app.config['DVLA_RESPONSE_BUCKET_NAME'], - 'root/dispatch' - ) - older_than_seven_days = s3.filter_s3_bucket_objects_within_date_range(bucket_objects) - - for f in older_than_seven_days: - s3.remove_s3_object(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], f['Key']) - - current_app.logger.info( - "Delete dvla response files started {} finished {} deleted {} files".format( - start, - datetime.utcnow(), - len(older_than_seven_days) - ) - ) - except SQLAlchemyError: - current_app.logger.exception("Failed to delete dvla response files") - raise - - -@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending") -@statsd(namespace="tasks") -def raise_alert_if_letter_notifications_still_sending(): - today = datetime.utcnow().date() - - # Do nothing on the weekend - if today.isoweekday() in [6, 7]: + current_provider = get_current_provider('sms') + if current_provider.updated_at > datetime.utcnow() - timedelta(minutes=10): + current_app.logger.info("Slow delivery notifications provider switched less than 10 minutes ago.") return + slow_delivery_notifications = is_delivery_slow_for_provider( + provider=current_provider.identifier, + threshold=0.1, + created_at=datetime.utcnow() - timedelta(minutes=10), + delivery_time=timedelta(minutes=4), + ) - if today.isoweekday() in [1, 2]: - offset_days = 4 - else: - offset_days = 2 - still_sending = Notification.query.filter( - Notification.notification_type == LETTER_TYPE, - Notification.status == NOTIFICATION_SENDING, - Notification.key_type == KEY_TYPE_NORMAL, - func.date(Notification.sent_at) <= today - timedelta(days=offset_days) - ).count() - - if still_sending: - message = "There are {} letters in the 'sending' state from {}".format( - still_sending, - (today - timedelta(days=offset_days)).strftime('%A %d %B') - ) - # Only send alerts in production - if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: - zendesk_client.create_ticket( - subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']), - message=message, - ticket_type=zendesk_client.TYPE_INCIDENT + if slow_delivery_notifications: + current_app.logger.warning( + 'Slow delivery notifications detected for provider {}'.format( + current_provider.identifier ) - else: - current_app.logger.info(message) + ) + + dao_toggle_sms_provider(current_provider.identifier) @notify_celery.task(name='check-job-status') @@ -406,82 +160,6 @@ def check_job_status(): raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids)) -@notify_celery.task(name='daily-stats-template-usage-by-month') -@statsd(namespace="tasks") -def daily_stats_template_usage_by_month(): - results = dao_fetch_monthly_historical_stats_by_template() - - for result in results: - if result.template_id: - insert_or_update_stats_for_template( - result.template_id, - result.month, - result.year, - result.count - ) - - -@notify_celery.task(name='raise-alert-if-no-letter-ack-file') -@statsd(namespace="tasks") -def letter_raise_alert_if_no_ack_file_for_zip(): - # get a list of zip files since yesterday - zip_file_set = set() - - for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], - subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', - suffix='.TXT'): - subname = key.split('/')[-1] # strip subfolder in name - zip_file_set.add(subname.upper().rstrip('.TXT')) - - # get acknowledgement file - ack_file_set = set() - - yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format - - for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], - subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday): - ack_file_set.add(key) - - today_str = datetime.utcnow().strftime('%Y%m%d') - - ack_content_set = set() - for key in ack_file_set: - if today_str in key: - content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key) - for zip_file in content.split('\n'): # each line - s = zip_file.split('|') - ack_content_set.add(s[0].upper()) - - message = ( - "Letter ack file does not contain all zip files sent. " - "Missing ack for zip files: {}, " - "pdf bucket: {}, subfolder: {}, " - "ack bucket: {}" - ).format( - str(sorted(zip_file_set - ack_content_set)), - current_app.config['LETTERS_PDF_BUCKET_NAME'], - datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', - current_app.config['DVLA_RESPONSE_BUCKET_NAME'] - ) - # strip empty element before comparison - ack_content_set.discard('') - zip_file_set.discard('') - - if len(zip_file_set - ack_content_set) > 0: - if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: - zendesk_client.create_ticket( - subject="Letter acknowledge error", - message=message, - ticket_type=zendesk_client.TYPE_INCIDENT - ) - current_app.logger.error(message) - - if len(ack_content_set - zip_file_set) > 0: - current_app.logger.info( - "letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set) - ) - - @notify_celery.task(name='replay-created-notifications') @statsd(namespace="tasks") def replay_created_notifications(): @@ -493,9 +171,10 @@ def replay_created_notifications(): notification_type ) - current_app.logger.info("Sending {} {} notifications " - "to the delivery queue because the notification " - "status was created.".format(len(notifications_to_resend), notification_type)) + if len(notifications_to_resend) > 0: + current_app.logger.info("Sending {} {} notifications " + "to the delivery queue because the notification " + "status was created.".format(len(notifications_to_resend), notification_type)) for n in notifications_to_resend: send_notification_to_queue(notification=n, research_mode=n.service.research_mode) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 96d26fe7d..1948c0964 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -307,6 +307,7 @@ def save_letter( saved_notification = persist_notification( template_id=notification['template'], template_version=notification['template_version'], + template_postage=template.postage, recipient=recipient, service=service, personalisation=notification['personalisation'], diff --git a/app/commands.py b/app/commands.py index 918a25242..c977f333b 100644 --- a/app/commands.py +++ b/app/commands.py @@ -1,4 +1,3 @@ -import sys import functools import uuid from datetime import datetime, timedelta @@ -9,11 +8,10 @@ import flask from click_datetime import Datetime as click_dt from flask import current_app, json from sqlalchemy.orm.exc import NoResultFound -from sqlalchemy import func from notifications_utils.statsd_decorators import statsd -from app import db, DATETIME_FORMAT, encryption, redis_store -from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform +from app import db, DATETIME_FORMAT, encryption +from app.celery.nightly_tasks import send_total_sent_notifications_to_performance_platform from app.celery.service_callback_tasks import send_delivery_status_to_service from app.celery.letters_pdf_tasks import create_letters_pdf from app.config import QueueNames @@ -34,11 +32,7 @@ from app.dao.services_dao import ( from app.dao.users_dao import delete_model_user, delete_user_verify_codes from app.models import PROVIDERS, User, Notification from app.performance_platform.processing_time import send_processing_time_for_start_and_end -from app.utils import ( - cache_key_for_service_template_usage_per_day, - get_london_midnight_in_utc, - get_midnight_for_day_before, -) +from app.utils import get_london_midnight_in_utc, get_midnight_for_day_before @click.group(name='command', help='Additional commands') @@ -430,50 +424,6 @@ def migrate_data_to_ft_billing(start_date, end_date): current_app.logger.info('Total inserted/updated records = {}'.format(total_updated)) -@notify_command() -@click.option('-s', '--service_id', required=True, type=click.UUID) -@click.option('-d', '--day', required=True, type=click_dt(format='%Y-%m-%d')) -def populate_redis_template_usage(service_id, day): - """ - Recalculate and replace the stats in redis for a day. - To be used if redis data is lost for some reason. - """ - if not current_app.config['REDIS_ENABLED']: - current_app.logger.error('Cannot populate redis template usage - redis not enabled') - sys.exit(1) - - # the day variable is set by click to be midnight of that day - start_time = get_london_midnight_in_utc(day) - end_time = get_london_midnight_in_utc(day + timedelta(days=1)) - - usage = { - str(row.template_id): row.count - for row in db.session.query( - Notification.template_id, - func.count().label('count') - ).filter( - Notification.service_id == service_id, - Notification.created_at >= start_time, - Notification.created_at < end_time - ).group_by( - Notification.template_id - ) - } - current_app.logger.info('Populating usage dict for service {} day {}: {}'.format( - service_id, - day, - usage.items()) - ) - if usage: - key = cache_key_for_service_template_usage_per_day(service_id, day) - redis_store.set_hash_and_expire( - key, - usage, - current_app.config['EXPIRE_CACHE_EIGHT_DAYS'], - raise_exception=True - ) - - @notify_command(name='rebuild-ft-billing-for-day') @click.option('-s', '--service_id', required=False, type=click.UUID) @click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True, @@ -654,3 +604,33 @@ def populate_notification_postage(start_date): total_updated += result.rowcount current_app.logger.info('Total inserted/updated records = {}'.format(total_updated)) + + +@notify_command(name='archive-jobs-created-between-dates') +@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) +@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) +@statsd(namespace="tasks") +def update_jobs_archived_flag(start_date, end_date): + current_app.logger.info('Archiving jobs created between {} to {}'.format(start_date, end_date)) + + process_date = start_date + total_updated = 0 + + while process_date < end_date: + start_time = datetime.utcnow() + sql = """update + jobs set archived = true + where + created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' + at time zone 'UTC' + and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'""" + + result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)}) + db.session.commit() + current_app.logger.info('jobs: --- Completed took {}ms. Archived {} jobs for {}'.format( + datetime.now() - start_time, result.rowcount, process_date)) + + process_date += timedelta(days=1) + + total_updated += result.rowcount + current_app.logger.info('Total archived jobs = {}'.format(total_updated)) diff --git a/app/config.py b/app/config.py index 1ba7d6edc..7ce5cfb0f 100644 --- a/app/config.py +++ b/app/config.py @@ -5,10 +5,6 @@ import json from celery.schedules import crontab from kombu import Exchange, Queue -from app.models import ( - EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, -) - if os.environ.get('VCAP_SERVICES'): # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate # standard environment variables from it @@ -108,6 +104,10 @@ class Config(object): DEBUG = False NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH') + # Cronitor + CRONITOR_ENABLED = False + CRONITOR_KEYS = json.loads(os.environ.get('CRONITOR_KEYS', '{}')) + ########################### # Default config values ### ########################### @@ -117,12 +117,12 @@ class Config(object): AWS_REGION = 'eu-west-1' INVITATION_EXPIRATION_DAYS = 2 NOTIFY_APP_NAME = 'api' - SQLALCHEMY_COMMIT_ON_TEARDOWN = False - SQLALCHEMY_RECORD_QUERIES = True - SQLALCHEMY_TRACK_MODIFICATIONS = True + SQLALCHEMY_RECORD_QUERIES = False + SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_POOL_SIZE = int(os.environ.get('SQLALCHEMY_POOL_SIZE', 5)) SQLALCHEMY_POOL_TIMEOUT = 30 SQLALCHEMY_POOL_RECYCLE = 300 + SQLALCHEMY_STATEMENT_TIMEOUT = 1200 PAGE_SIZE = 50 API_PAGE_SIZE = 250 TEST_MESSAGE_FILENAME = 'Test message' @@ -157,8 +157,14 @@ class Config(object): CELERY_TIMEZONE = 'Europe/London' CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' - CELERY_IMPORTS = ('app.celery.tasks', 'app.celery.scheduled_tasks', 'app.celery.reporting_tasks') + CELERY_IMPORTS = ( + 'app.celery.tasks', + 'app.celery.scheduled_tasks', + 'app.celery.reporting_tasks', + 'app.celery.nightly_tasks', + ) CELERYBEAT_SCHEDULE = { + # app/celery/scheduled_tasks.py 'run-scheduled-jobs': { 'task': 'run-scheduled-jobs', 'schedule': crontab(minute=1), @@ -189,17 +195,12 @@ class Config(object): 'schedule': crontab(minute='0, 15, 30, 45'), 'options': {'queue': QueueNames.PERIODIC} }, - # nightly tasks: + # app/celery/nightly_tasks.py 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', 'schedule': crontab(hour=0, minute=5), 'options': {'queue': QueueNames.PERIODIC} }, - 'daily-stats-template-usage-by-month': { - 'task': 'daily-stats-template-usage-by-month', - 'schedule': crontab(hour=0, minute=10), - 'options': {'queue': QueueNames.PERIODIC} - }, 'create-nightly-billing': { 'task': 'create-nightly-billing', 'schedule': crontab(hour=0, minute=15), @@ -236,23 +237,22 @@ class Config(object): 'schedule': crontab(hour=2, minute=0), 'options': {'queue': QueueNames.PERIODIC} }, - 'remove_sms_email_jobs': { - 'task': 'remove_csv_files', - 'schedule': crontab(hour=4, minute=0), - 'options': {'queue': QueueNames.PERIODIC}, - 'kwargs': {'job_types': [EMAIL_TYPE, SMS_TYPE]} - }, - 'remove_letter_jobs': { - 'task': 'remove_csv_files', - 'schedule': crontab(hour=4, minute=20), - 'options': {'queue': QueueNames.PERIODIC}, - 'kwargs': {'job_types': [LETTER_TYPE]} - }, 'remove_transformed_dvla_files': { 'task': 'remove_transformed_dvla_files', - 'schedule': crontab(hour=4, minute=40), + 'schedule': crontab(hour=3, minute=40), 'options': {'queue': QueueNames.PERIODIC} }, + 'remove_sms_email_jobs': { + 'task': 'remove_sms_email_jobs', + 'schedule': crontab(hour=4, minute=0), + 'options': {'queue': QueueNames.PERIODIC}, + }, + 'remove_letter_jobs': { + 'task': 'remove_letter_jobs', + 'schedule': crontab(hour=4, minute=20), # this has to run AFTER remove_transformed_dvla_files + # since we mark jobs as archived + 'options': {'queue': QueueNames.PERIODIC}, + }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=16, minute=30), @@ -290,9 +290,6 @@ class Config(object): SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222') - FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = None - FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = None - DVLA_BUCKETS = { 'job': '{}-dvla-file-per-job'.format(os.getenv('NOTIFY_ENVIRONMENT')), 'notification': '{}-dvla-letter-api-files'.format(os.getenv('NOTIFY_ENVIRONMENT')) @@ -438,12 +435,12 @@ class Live(Config): INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' STATSD_ENABLED = True FROM_NUMBER = 'GOVUK' - FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = '6c1d81bb-dae2-4ee9-80b0-89a4aae9f649' - FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = 'ba9e1789-a804-40b8-871f-cc60d4c1286f' PERFORMANCE_PLATFORM_ENABLED = True API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True + CRONITOR_ENABLED = True + class CloudFoundryConfig(Config): pass diff --git a/app/cronitor.py b/app/cronitor.py new file mode 100644 index 000000000..83a12f61f --- /dev/null +++ b/app/cronitor.py @@ -0,0 +1,52 @@ +import requests +from functools import wraps +from flask import current_app + + +def cronitor(task_name): + # check if task_name is in config + def decorator(func): + def ping_cronitor(command): + if not current_app.config['CRONITOR_ENABLED']: + return + + task_slug = current_app.config['CRONITOR_KEYS'].get(task_name) + if not task_slug: + current_app.logger.error( + 'Cronitor enabled but task_name {} not found in environment'.format(task_name) + ) + return + + if command not in {'run', 'complete', 'fail'}: + raise ValueError('command {} not a valid cronitor command'.format(command)) + + try: + resp = requests.get( + 'https://cronitor.link/{}/{}'.format(task_slug, command), + # cronitor limits msg to 1000 characters + params={ + 'host': current_app.config['API_HOST_NAME'], + } + ) + resp.raise_for_status() + except requests.RequestException as e: + current_app.logger.warning('Cronitor API failed for task {} due to {}'.format( + task_name, + repr(e) + )) + + @wraps(func) + def inner_decorator(*args, **kwargs): + ping_cronitor('run') + try: + ret = func(*args, **kwargs) + status = 'complete' + return ret + except Exception: + status = 'fail' + raise + finally: + ping_cronitor(status) + + return inner_decorator + return decorator diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index f80f175d8..c880830c8 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -4,12 +4,15 @@ from flask import current_app from notifications_utils.timezones import convert_bst_to_utc from sqlalchemy import func from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.sql.expression import literal +from sqlalchemy.sql.expression import literal, extract from sqlalchemy.types import DateTime, Integer from app import db -from app.models import Notification, NotificationHistory, FactNotificationStatus, KEY_TYPE_TEST -from app.utils import get_london_midnight_in_utc, midnight_n_days_ago +from app.models import ( + Notification, NotificationHistory, FactNotificationStatus, KEY_TYPE_TEST, Service, Template, + NOTIFICATION_CANCELLED +) +from app.utils import get_london_midnight_in_utc, midnight_n_days_ago, get_london_month_from_utc_column def fetch_notification_status_for_day(process_day, service_id=None): @@ -104,12 +107,13 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id): ).all() -def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, limit_days=7): +def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7): start_date = midnight_n_days_ago(limit_days) now = datetime.utcnow() stats_for_7_days = db.session.query( FactNotificationStatus.notification_type.label('notification_type'), FactNotificationStatus.notification_status.label('status'), + *([FactNotificationStatus.template_id.label('template_id')] if by_template else []), FactNotificationStatus.notification_count.label('count') ).filter( FactNotificationStatus.service_id == service_id, @@ -120,6 +124,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_ stats_for_today = db.session.query( Notification.notification_type.cast(db.Text), Notification.status, + *([Notification.template_id] if by_template else []), func.count().label('count') ).filter( Notification.created_at >= get_london_midnight_in_utc(now), @@ -127,14 +132,265 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_ Notification.key_type != KEY_TYPE_TEST ).group_by( Notification.notification_type, + *([Notification.template_id] if by_template else []), Notification.status ) + all_stats_table = stats_for_7_days.union_all(stats_for_today).subquery() - return db.session.query( + + query = db.session.query( + *([ + Template.name.label("template_name"), + Template.is_precompiled_letter, + all_stats_table.c.template_id + ] if by_template else []), all_stats_table.c.notification_type, all_stats_table.c.status, func.cast(func.sum(all_stats_table.c.count), Integer).label('count'), - ).group_by( + ) + + if by_template: + query = query.filter(all_stats_table.c.template_id == Template.id) + + return query.group_by( + *([Template.name, Template.is_precompiled_letter, all_stats_table.c.template_id] if by_template else []), all_stats_table.c.notification_type, all_stats_table.c.status, ).all() + + +def fetch_notification_status_totals_for_all_services(start_date, end_date): + stats = db.session.query( + FactNotificationStatus.notification_type.label('notification_type'), + FactNotificationStatus.notification_status.label('status'), + FactNotificationStatus.key_type.label('key_type'), + func.sum(FactNotificationStatus.notification_count).label('count') + ).filter( + FactNotificationStatus.bst_date >= start_date, + FactNotificationStatus.bst_date <= end_date + ).group_by( + FactNotificationStatus.notification_type, + FactNotificationStatus.notification_status, + FactNotificationStatus.key_type, + ) + today = get_london_midnight_in_utc(datetime.utcnow()) + if start_date <= today.date() <= end_date: + stats_for_today = db.session.query( + Notification.notification_type.cast(db.Text).label('notification_type'), + Notification.status, + Notification.key_type, + func.count().label('count') + ).filter( + Notification.created_at >= today + ).group_by( + Notification.notification_type.cast(db.Text), + Notification.status, + Notification.key_type, + ) + all_stats_table = stats.union_all(stats_for_today).subquery() + query = db.session.query( + all_stats_table.c.notification_type, + all_stats_table.c.status, + all_stats_table.c.key_type, + func.cast(func.sum(all_stats_table.c.count), Integer).label('count'), + ).group_by( + all_stats_table.c.notification_type, + all_stats_table.c.status, + all_stats_table.c.key_type, + ).order_by( + all_stats_table.c.notification_type + ) + else: + query = stats.order_by( + FactNotificationStatus.notification_type + ) + return query.all() + + +def fetch_notification_statuses_for_job(job_id): + return db.session.query( + FactNotificationStatus.notification_status.label('status'), + func.sum(FactNotificationStatus.notification_count).label('count'), + ).filter( + FactNotificationStatus.job_id == job_id, + ).group_by( + FactNotificationStatus.notification_status + ).all() + + +def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True): + stats = db.session.query( + FactNotificationStatus.service_id.label('service_id'), + Service.name.label('name'), + Service.restricted.label('restricted'), + Service.research_mode.label('research_mode'), + Service.active.label('active'), + Service.created_at.label('created_at'), + FactNotificationStatus.notification_type.label('notification_type'), + FactNotificationStatus.notification_status.label('status'), + func.sum(FactNotificationStatus.notification_count).label('count') + ).filter( + FactNotificationStatus.bst_date >= start_date, + FactNotificationStatus.bst_date <= end_date, + FactNotificationStatus.service_id == Service.id, + ).group_by( + FactNotificationStatus.service_id.label('service_id'), + Service.name, + Service.restricted, + Service.research_mode, + Service.active, + Service.created_at, + FactNotificationStatus.notification_type, + FactNotificationStatus.notification_status, + ).order_by( + FactNotificationStatus.service_id, + FactNotificationStatus.notification_type + ) + if not include_from_test_key: + stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST) + + if start_date <= datetime.utcnow().date() <= end_date: + today = get_london_midnight_in_utc(datetime.utcnow()) + subquery = db.session.query( + Notification.notification_type.cast(db.Text).label('notification_type'), + Notification.status.label('status'), + Notification.service_id.label('service_id'), + func.count(Notification.id).label('count') + ).filter( + Notification.created_at >= today + ).group_by( + Notification.notification_type, + Notification.status, + Notification.service_id + ) + if not include_from_test_key: + subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST) + subquery = subquery.subquery() + + stats_for_today = db.session.query( + Service.id.label('service_id'), + Service.name.label('name'), + Service.restricted.label('restricted'), + Service.research_mode.label('research_mode'), + Service.active.label('active'), + Service.created_at.label('created_at'), + subquery.c.notification_type.label('notification_type'), + subquery.c.status.label('status'), + subquery.c.count.label('count') + ).outerjoin( + subquery, + subquery.c.service_id == Service.id + ) + + all_stats_table = stats.union_all(stats_for_today).subquery() + query = db.session.query( + all_stats_table.c.service_id, + all_stats_table.c.name, + all_stats_table.c.restricted, + all_stats_table.c.research_mode, + all_stats_table.c.active, + all_stats_table.c.created_at, + all_stats_table.c.notification_type, + all_stats_table.c.status, + func.cast(func.sum(all_stats_table.c.count), Integer).label('count'), + ).group_by( + all_stats_table.c.service_id, + all_stats_table.c.name, + all_stats_table.c.restricted, + all_stats_table.c.research_mode, + all_stats_table.c.active, + all_stats_table.c.created_at, + all_stats_table.c.notification_type, + all_stats_table.c.status, + ).order_by( + all_stats_table.c.name, + all_stats_table.c.notification_type, + all_stats_table.c.status + ) + else: + query = stats + return query.all() + + +def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): + # services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service + stats = db.session.query( + FactNotificationStatus.template_id.label('template_id'), + Template.name.label('name'), + Template.template_type.label('template_type'), + Template.is_precompiled_letter.label('is_precompiled_letter'), + extract('month', FactNotificationStatus.bst_date).label('month'), + extract('year', FactNotificationStatus.bst_date).label('year'), + func.sum(FactNotificationStatus.notification_count).label('count') + ).join( + Template, FactNotificationStatus.template_id == Template.id + ).filter( + FactNotificationStatus.service_id == service_id, + FactNotificationStatus.bst_date >= start_date, + FactNotificationStatus.bst_date <= end_date, + FactNotificationStatus.key_type != KEY_TYPE_TEST, + FactNotificationStatus.notification_status != NOTIFICATION_CANCELLED, + ).group_by( + FactNotificationStatus.template_id, + Template.name, + Template.template_type, + Template.is_precompiled_letter, + extract('month', FactNotificationStatus.bst_date).label('month'), + extract('year', FactNotificationStatus.bst_date).label('year'), + ).order_by( + extract('year', FactNotificationStatus.bst_date), + extract('month', FactNotificationStatus.bst_date), + Template.name + ) + + if start_date <= datetime.utcnow() <= end_date: + today = get_london_midnight_in_utc(datetime.utcnow()) + month = get_london_month_from_utc_column(Notification.created_at) + + stats_for_today = db.session.query( + Notification.template_id.label('template_id'), + Template.name.label('name'), + Template.template_type.label('template_type'), + Template.is_precompiled_letter.label('is_precompiled_letter'), + extract('month', month).label('month'), + extract('year', month).label('year'), + func.count().label('count') + ).join( + Template, Notification.template_id == Template.id, + ).filter( + Notification.created_at >= today, + Notification.service_id == service_id, + Notification.key_type != KEY_TYPE_TEST, + Notification.status != NOTIFICATION_CANCELLED + ).group_by( + Notification.template_id, + Template.hidden, + Template.name, + Template.template_type, + month + ) + + all_stats_table = stats.union_all(stats_for_today).subquery() + query = db.session.query( + all_stats_table.c.template_id, + all_stats_table.c.name, + all_stats_table.c.is_precompiled_letter, + all_stats_table.c.template_type, + func.cast(all_stats_table.c.month, Integer).label('month'), + func.cast(all_stats_table.c.year, Integer).label('year'), + func.cast(func.sum(all_stats_table.c.count), Integer).label('count'), + ).group_by( + all_stats_table.c.template_id, + all_stats_table.c.name, + all_stats_table.c.is_precompiled_letter, + all_stats_table.c.template_type, + all_stats_table.c.month, + all_stats_table.c.year, + ).order_by( + all_stats_table.c.year, + all_stats_table.c.month, + all_stats_table.c.name + ) + else: + query = stats + return query.all() diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index 6e2297302..e22c1c709 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -16,27 +16,23 @@ from app.models import ( JOB_STATUS_PENDING, JOB_STATUS_SCHEDULED, LETTER_TYPE, - NotificationHistory, + Notification, Template, + ServiceDataRetention ) from app.variables import LETTER_TEST_API_FILENAME @statsd(namespace="dao") def dao_get_notification_outcomes_for_job(service_id, job_id): - query = db.session.query( - func.count(NotificationHistory.status).label('count'), - NotificationHistory.status - ) - - return query.filter( - NotificationHistory.service_id == service_id + return db.session.query( + func.count(Notification.status).label('count'), + Notification.status ).filter( - NotificationHistory.job_id == job_id + Notification.service_id == service_id, + Notification.job_id == job_id ).group_by( - NotificationHistory.status - ).order_by( - asc(NotificationHistory.status) + Notification.status ).all() @@ -66,6 +62,12 @@ def dao_get_job_by_id(job_id): return Job.query.filter_by(id=job_id).one() +def dao_archive_job(job): + job.archived = True + db.session.add(job) + db.session.commit() + + def dao_set_scheduled_jobs_to_pending(): """ Sets all past scheduled jobs to pending, and then returns them for further processing. @@ -115,15 +117,35 @@ def dao_update_job(job): db.session.commit() -def dao_get_jobs_older_than_limited_by(job_types, older_than=7, limit_days=2): - end_date = datetime.utcnow() - timedelta(days=older_than) - start_date = end_date - timedelta(days=limit_days) +def dao_get_jobs_older_than_data_retention(notification_types): + flexible_data_retention = ServiceDataRetention.query.filter( + ServiceDataRetention.notification_type.in_(notification_types) + ).all() + jobs = [] + today = datetime.utcnow().date() + for f in flexible_data_retention: + end_date = today - timedelta(days=f.days_of_retention) - return Job.query.join(Template).filter( - Job.created_at < end_date, - Job.created_at >= start_date, - Template.template_type.in_(job_types) - ).order_by(desc(Job.created_at)).all() + jobs.extend(Job.query.join(Template).filter( + Job.created_at < end_date, + Job.archived == False, # noqa + Template.template_type == f.notification_type, + Job.service_id == f.service_id + ).order_by(desc(Job.created_at)).all()) + + end_date = today - timedelta(days=7) + for notification_type in notification_types: + services_with_data_retention = [ + x.service_id for x in flexible_data_retention if x.notification_type == notification_type + ] + jobs.extend(Job.query.join(Template).filter( + Job.created_at < end_date, + Job.archived == False, # noqa + Template.template_type == notification_type, + Job.service_id.notin_(services_with_data_retention) + ).order_by(desc(Job.created_at)).all()) + + return jobs def dao_get_all_letter_jobs(): diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py deleted file mode 100644 index 7105f5b3e..000000000 --- a/app/dao/notification_usage_dao.py +++ /dev/null @@ -1,180 +0,0 @@ -from datetime import datetime, timedelta - -from notifications_utils.statsd_decorators import statsd -from sqlalchemy import Float, Integer, and_ -from sqlalchemy import func, case, cast -from sqlalchemy import literal_column - -from app import db -from app.dao.date_util import get_financial_year -from app.models import ( - NotificationHistory, - Rate, - NOTIFICATION_STATUS_TYPES_BILLABLE, - KEY_TYPE_TEST, - SMS_TYPE, - EMAIL_TYPE, - LETTER_TYPE, - LetterRate, - Service -) -from app.utils import get_london_month_from_utc_column - - -@statsd(namespace="dao") -def get_billing_data_for_month(service_id, start_date, end_date, notification_type): - results = [] - - if notification_type == EMAIL_TYPE: - return billing_data_per_month_query(0, service_id, start_date, end_date, EMAIL_TYPE) - - elif notification_type == SMS_TYPE: - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - - if not rates: - return [] - - # so the start end date in the query are the valid from the rate, not the month - # - this is going to take some thought - for r, n in zip(rates, rates[1:]): - results.extend( - billing_data_per_month_query( - r.rate, service_id, max(r.valid_from, start_date), - min(n.valid_from, end_date), SMS_TYPE - ) - ) - results.extend( - billing_data_per_month_query( - rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), - end_date, SMS_TYPE - ) - ) - elif notification_type == LETTER_TYPE: - results.extend(billing_letter_data_per_month_query(service_id, start_date, end_date)) - - return results - - -@statsd(namespace="dao") -def get_monthly_billing_data(service_id, year): - start_date, end_date = get_financial_year(year) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - - if not rates: - return [] - - result = [] - for r, n in zip(rates, rates[1:]): - result.extend(billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from, SMS_TYPE)) - result.extend(billing_data_per_month_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date, SMS_TYPE)) - - return [(datetime.strftime(x[0], "%B"), x[1], x[2], x[3], x[4], x[5]) for x in result] - - -def billing_data_filter(notification_type, start_date, end_date, service_id): - return [ - NotificationHistory.notification_type == notification_type, - NotificationHistory.created_at.between(start_date, end_date), - NotificationHistory.service_id == service_id, - NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE), - NotificationHistory.key_type != KEY_TYPE_TEST - ] - - -def get_rates_for_daterange(start_date, end_date, notification_type): - rates = Rate.query.filter(Rate.notification_type == notification_type).order_by(Rate.valid_from).all() - - if not rates: - return [] - - results = [] - for current_rate, current_rate_expiry_date in zip(rates, rates[1:]): - if is_between(current_rate.valid_from, start_date, end_date) or \ - is_between(current_rate_expiry_date.valid_from - timedelta(microseconds=1), start_date, end_date): - results.append(current_rate) - - if is_between(rates[-1].valid_from, start_date, end_date): - results.append(rates[-1]) - - if not results: - for x in reversed(rates): - if start_date >= x.valid_from: - results.append(x) - break - - return results - - -def is_between(date, start_date, end_date): - return start_date <= date <= end_date - - -@statsd(namespace="dao") -def billing_data_per_month_query(rate, service_id, start_date, end_date, notification_type): - month = get_london_month_from_utc_column(NotificationHistory.created_at) - if notification_type == SMS_TYPE: - filter_subq = func.sum(NotificationHistory.billable_units).label('billing_units') - elif notification_type == EMAIL_TYPE: - filter_subq = func.count(NotificationHistory.billable_units).label('billing_units') - - results = db.session.query( - month.label('month'), - filter_subq, - rate_multiplier().label('rate_multiplier'), - NotificationHistory.international, - NotificationHistory.notification_type, - cast(rate, Float()).label('rate') - ).filter( - *billing_data_filter(notification_type, start_date, end_date, service_id) - ).group_by( - NotificationHistory.notification_type, - month, - NotificationHistory.rate_multiplier, - NotificationHistory.international - ).order_by( - month, - rate_multiplier() - ) - return results.all() - - -def rate_multiplier(): - return cast(case([ - (NotificationHistory.rate_multiplier == None, literal_column("'1'")), # noqa - (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier), # noqa - ]), Integer()) - - -@statsd(namespace="dao") -def billing_letter_data_per_month_query(service_id, start_date, end_date): - month = get_london_month_from_utc_column(NotificationHistory.created_at) - crown = Service.query.get(service_id).crown - results = db.session.query( - month.label('month'), - func.count(NotificationHistory.billable_units).label('billing_units'), - rate_multiplier().label('rate_multiplier'), - NotificationHistory.international, - NotificationHistory.notification_type, - cast(LetterRate.rate, Float()).label('rate') - ).join( - LetterRate, - and_(NotificationHistory.created_at >= LetterRate.start_date, - (LetterRate.end_date == None) | # noqa - (LetterRate.end_date > NotificationHistory.created_at)) - ).filter( - LetterRate.sheet_count == NotificationHistory.billable_units, - LetterRate.crown == crown, - LetterRate.post_class == 'second', - NotificationHistory.created_at < end_date, - *billing_data_filter(LETTER_TYPE, start_date, end_date, service_id) - ).group_by( - NotificationHistory.notification_type, - month, - NotificationHistory.rate_multiplier, - NotificationHistory.international, - LetterRate.rate - ).order_by( - month, - rate_multiplier() - ) - return results.all() diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 564ab6acf..15c9e997a 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -7,82 +7,47 @@ from datetime import ( from boto.exception import BotoClientError from flask import current_app - +from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES from notifications_utils.recipients import ( validate_and_format_email_address, InvalidEmailError, try_validate_and_format_phone_number ) from notifications_utils.statsd_decorators import statsd -from werkzeug.datastructures import MultiDict -from sqlalchemy import (desc, func, or_, asc) -from sqlalchemy.orm import joinedload -from sqlalchemy.sql.expression import case -from sqlalchemy.sql import functions -from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES from notifications_utils.timezones import convert_utc_to_bst +from sqlalchemy import (desc, func, asc) +from sqlalchemy.orm import joinedload +from sqlalchemy.sql import functions +from sqlalchemy.sql.expression import case +from werkzeug.datastructures import MultiDict from app import db, create_uuid from app.aws.s3 import remove_s3_object, get_s3_bucket_objects -from app.letters.utils import LETTERS_PDF_FILE_LOCATION_STRUCTURE -from app.utils import midnight_n_days_ago, escape_special_characters +from app.dao.dao_utils import transactional from app.errors import InvalidRequest +from app.letters.utils import LETTERS_PDF_FILE_LOCATION_STRUCTURE from app.models import ( Notification, NotificationHistory, ScheduledNotification, - Template, - TemplateHistory, KEY_TYPE_TEST, LETTER_TYPE, NOTIFICATION_CREATED, NOTIFICATION_DELIVERED, NOTIFICATION_SENDING, NOTIFICATION_PENDING, + NOTIFICATION_PENDING_VIRUS_CHECK, NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_TEMPORARY_FAILURE, NOTIFICATION_PERMANENT_FAILURE, NOTIFICATION_SENT, SMS_TYPE, EMAIL_TYPE, - ServiceDataRetention + ServiceDataRetention, + Service ) - -from app.dao.dao_utils import transactional from app.utils import get_london_midnight_in_utc - - -@statsd(namespace="dao") -def dao_get_template_usage(service_id, day): - start = get_london_midnight_in_utc(day) - end = get_london_midnight_in_utc(day + timedelta(days=1)) - - notifications_aggregate_query = db.session.query( - func.count().label('count'), - Notification.template_id - ).filter( - Notification.created_at >= start, - Notification.created_at < end, - Notification.service_id == service_id, - Notification.key_type != KEY_TYPE_TEST, - ).group_by( - Notification.template_id - ).subquery() - - query = db.session.query( - Template.id, - Template.name, - Template.template_type, - Template.is_precompiled_letter, - func.coalesce(notifications_aggregate_query.c.count, 0).label('count') - ).outerjoin( - notifications_aggregate_query, - notifications_aggregate_query.c.template_id == Template.id - ).filter( - Template.service_id == service_id - ).order_by(Template.name) - - return query.all() +from app.utils import midnight_n_days_ago, escape_special_characters @statsd(namespace="dao") @@ -145,16 +110,23 @@ def _update_notification_status(notification, status): @statsd(namespace="dao") @transactional def update_notification_status_by_id(notification_id, status, sent_by=None): - notification = Notification.query.with_lockmode("update").filter( - Notification.id == notification_id, - or_( - Notification.status == NOTIFICATION_CREATED, - Notification.status == NOTIFICATION_SENDING, - Notification.status == NOTIFICATION_PENDING, - Notification.status == NOTIFICATION_SENT - )).first() + notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first() if not notification: + current_app.logger.error('notification not found for id {} (update to status {})'.format( + notification_id, + status + )) + return None + + if notification.status not in { + NOTIFICATION_CREATED, + NOTIFICATION_SENDING, + NOTIFICATION_PENDING, + NOTIFICATION_SENT, + NOTIFICATION_PENDING_VIRUS_CHECK + }: + _duplicate_update_warning(notification, status) return None if notification.international and not country_records_delivery(notification.phone_prefix): @@ -170,15 +142,18 @@ def update_notification_status_by_id(notification_id, status, sent_by=None): @statsd(namespace="dao") @transactional def update_notification_status_by_reference(reference, status): - notification = Notification.query.filter( - Notification.reference == reference, - or_( - Notification.status == NOTIFICATION_SENDING, - Notification.status == NOTIFICATION_PENDING, - Notification.status == NOTIFICATION_SENT - )).first() + # this is used to update letters and emails + notification = Notification.query.filter(Notification.reference == reference).first() - if not notification or notification.status == NOTIFICATION_SENT: + if not notification: + current_app.logger.error('notification not found for reference {} (update to {})'.format(reference, status)) + return None + + if notification.status not in { + NOTIFICATION_SENDING, + NOTIFICATION_PENDING + }: + _duplicate_update_warning(notification, status) return None return _update_notification_status( @@ -225,11 +200,15 @@ def get_notification_with_personalisation(service_id, notification_id, key_type) @statsd(namespace="dao") -def get_notification_by_id(notification_id, _raise=False): - if _raise: - return Notification.query.filter_by(id=notification_id).one() - else: - return Notification.query.filter_by(id=notification_id).first() +def get_notification_by_id(notification_id, service_id=None, _raise=False): + filters = [Notification.id == notification_id] + + if service_id: + filters.append(Notification.service_id == service_id) + + query = Notification.query.filter(*filters) + + return query.one() if _raise else query.first() def get_notifications(filter_dict=None): @@ -242,6 +221,7 @@ def get_notifications_for_service( filter_dict=None, page=1, page_size=None, + count_pages=True, limit_days=None, key_type=None, personalisation=False, @@ -287,7 +267,8 @@ def get_notifications_for_service( return query.order_by(desc(Notification.created_at)).paginate( page=page, - per_page=page_size + per_page=page_size, + count=count_pages ) @@ -306,41 +287,90 @@ def _filter_query(query, filter_dict=None): # filter by template template_types = multidict.getlist('template_type') if template_types: - query = query.join(TemplateHistory).filter(TemplateHistory.template_type.in_(template_types)) + query = query.filter(Notification.notification_type.in_(template_types)) return query @statsd(namespace="dao") -@transactional -def delete_notifications_created_more_than_a_week_ago_by_type(notification_type): +def delete_notifications_created_more_than_a_week_ago_by_type(notification_type, qry_limit=10000): + current_app.logger.info( + 'Deleting {} notifications for services with flexible data retention'.format(notification_type)) + flexible_data_retention = ServiceDataRetention.query.filter( ServiceDataRetention.notification_type == notification_type ).all() deleted = 0 for f in flexible_data_retention: - days_of_retention = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=f.days_of_retention) - query = db.session.query(Notification).filter( - func.date(Notification.created_at) < days_of_retention, - Notification.notification_type == f.notification_type, Notification.service_id == f.service_id) - if notification_type == LETTER_TYPE: - _delete_letters_from_s3(query) - deleted += query.delete(synchronize_session='fetch') + days_of_retention = get_london_midnight_in_utc( + convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=f.days_of_retention) - seven_days_ago = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7) + if notification_type == LETTER_TYPE: + _delete_letters_from_s3( + notification_type, f.service_id, days_of_retention, qry_limit + ) + + current_app.logger.info( + "Deleting {} notifications for service id: {}".format(notification_type, f.service_id)) + deleted += _delete_notifications( + deleted, notification_type, days_of_retention, f.service_id, qry_limit + ) + + current_app.logger.info( + 'Deleting {} notifications for services without flexible data retention'.format(notification_type)) + + seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=7) services_with_data_retention = [x.service_id for x in flexible_data_retention] - query = db.session.query(Notification).filter(func.date(Notification.created_at) < seven_days_ago, - Notification.notification_type == notification_type, - Notification.service_id.notin_( - services_with_data_retention)) - if notification_type == LETTER_TYPE: - _delete_letters_from_s3(query=query) - deleted += query.delete(synchronize_session='fetch') + service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all() + + for service_id in service_ids_to_purge: + if notification_type == LETTER_TYPE: + _delete_letters_from_s3( + notification_type, service_id, seven_days_ago, qry_limit + ) + + deleted += _delete_notifications( + deleted, notification_type, seven_days_ago, service_id, qry_limit + ) + + current_app.logger.info('Finished deleting {} notifications'.format(notification_type)) + return deleted -def _delete_letters_from_s3(query): - letters_to_delete_from_s3 = query.all() +def _delete_notifications( + deleted, notification_type, date_to_delete_from, service_id, query_limit): + + subquery = db.session.query( + Notification.id + ).filter( + Notification.notification_type == notification_type, + Notification.service_id == service_id, + Notification.created_at < date_to_delete_from + ).limit(query_limit).subquery() + + number_deleted = db.session.query(Notification).filter( + Notification.id.in_(subquery)).delete(synchronize_session='fetch') + deleted += number_deleted + db.session.commit() + while number_deleted > 0: + number_deleted = db.session.query(Notification).filter( + Notification.id.in_(subquery)).delete(synchronize_session='fetch') + deleted += number_deleted + db.session.commit() + return deleted + + +def _delete_letters_from_s3( + notification_type, service_id, date_to_delete_from, query_limit +): + letters_to_delete_from_s3 = db.session.query( + Notification + ).filter( + Notification.notification_type == notification_type, + Notification.created_at < date_to_delete_from, + Notification.service_id == service_id + ).limit(query_limit).all() for letter in letters_to_delete_from_s3: bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME'] if letter.sent_at: @@ -437,22 +467,39 @@ def get_total_sent_notifications_in_date_range(start_date, end_date, notificatio def is_delivery_slow_for_provider( - sent_at, + created_at, provider, threshold, delivery_time, - service_id, - template_id ): - count = db.session.query(Notification).filter( - Notification.service_id == service_id, - Notification.template_id == template_id, - Notification.sent_at >= sent_at, - Notification.status == NOTIFICATION_DELIVERED, + count = db.session.query( + case( + [( + Notification.status == NOTIFICATION_DELIVERED, + (Notification.updated_at - Notification.sent_at) >= delivery_time + )], + else_=(datetime.utcnow() - Notification.sent_at) >= delivery_time + ).label("slow"), func.count() + + ).filter( + Notification.created_at >= created_at, + Notification.sent_at.isnot(None), + Notification.status.in_([NOTIFICATION_DELIVERED, NOTIFICATION_SENDING]), Notification.sent_by == provider, - (Notification.updated_at - Notification.sent_at) >= delivery_time, - ).count() - return count >= threshold + Notification.key_type != KEY_TYPE_TEST + ).group_by("slow").all() + + counts = {c[0]: c[1] for c in count} + total_notifications = sum(counts.values()) + slow_notifications = counts.get(True, 0) + + if total_notifications: + current_app.logger.info("Slow delivery notifications count: {} out of {}. Ratio {}".format( + slow_notifications, total_notifications, slow_notifications / total_notifications + )) + return slow_notifications / total_notifications >= threshold + else: + return False @statsd(namespace="dao") @@ -625,29 +672,17 @@ def guess_notification_type(search_term): return SMS_TYPE -@statsd(namespace='dao') -def fetch_aggregate_stats_by_date_range_for_all_services(start_date, end_date): - start_date = get_london_midnight_in_utc(start_date) - end_date = get_london_midnight_in_utc(end_date + timedelta(days=1)) - table = NotificationHistory - - if start_date >= datetime.utcnow() - timedelta(days=7): - table = Notification - - query = db.session.query( - table.notification_type, - table.status, - table.key_type, - func.count(table.id).label('count') - ).filter( - table.created_at >= start_date, - table.created_at < end_date - ).group_by( - table.notification_type, - table.key_type, - table.status - ).order_by( - table.notification_type, +def _duplicate_update_warning(notification, status): + current_app.logger.info( + ( + 'Duplicate callback received. Notification id {id} received a status update to {new_status}' + '{time_diff} after being set to {old_status}. {type} sent by {sent_by}' + ).format( + id=notification.id, + old_status=notification.status, + new_status=status, + time_diff=datetime.utcnow() - (notification.updated_at or notification.created_at), + type=notification.notification_type, + sent_by=notification.sent_by + ) ) - - return query.all() diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 40918f7e7..52536c116 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,8 +1,8 @@ import uuid -from datetime import date, datetime, timedelta, time +from datetime import date, datetime, timedelta from notifications_utils.statsd_decorators import statsd -from sqlalchemy import asc, func, extract +from sqlalchemy import asc, func from sqlalchemy.orm import joinedload from flask import current_app @@ -11,9 +11,7 @@ from app.dao.dao_utils import ( transactional, version_class ) -from app.dao.date_util import get_financial_year from app.dao.service_sms_sender_dao import insert_service_sms_sender -from app.dao.stats_template_usage_by_month_dao import dao_get_template_usage_stats_by_service from app.models import ( AnnualBilling, ApiKey, @@ -37,7 +35,7 @@ from app.models import ( SMS_TYPE, LETTER_TYPE, ) -from app.utils import get_london_month_from_utc_column, get_london_midnight_in_utc, midnight_n_days_ago +from app.utils import get_london_midnight_in_utc, midnight_n_days_ago DEFAULT_SERVICE_PERMISSIONS = [ SMS_TYPE, @@ -335,51 +333,6 @@ def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_act return query.all() -@statsd(namespace='dao') -def fetch_stats_by_date_range_for_all_services(start_date, end_date, include_from_test_key=True, only_active=True): - start_date = get_london_midnight_in_utc(start_date) - end_date = get_london_midnight_in_utc(end_date + timedelta(days=1)) - table = NotificationHistory - - if start_date >= datetime.utcnow() - timedelta(days=7): - table = Notification - subquery = db.session.query( - table.notification_type, - table.status, - table.service_id, - func.count(table.id).label('count') - ).filter( - table.created_at >= start_date, - table.created_at < end_date - ).group_by( - table.notification_type, - table.status, - table.service_id - ) - if not include_from_test_key: - subquery = subquery.filter(table.key_type != KEY_TYPE_TEST) - subquery = subquery.subquery() - - query = db.session.query( - Service.id.label('service_id'), - Service.name, - Service.restricted, - Service.research_mode, - Service.active, - Service.created_at, - subquery.c.notification_type, - subquery.c.status, - subquery.c.count - ).outerjoin( - subquery, - subquery.c.service_id == Service.id - ).order_by(Service.id) - if only_active: - query = query.filter(Service.active) - - return query.all() - - @transactional @version_class(Service) @version_class(ApiKey) @@ -411,98 +364,3 @@ def dao_fetch_active_users_for_service(service_id): ) return query.all() - - -@statsd(namespace="dao") -def dao_fetch_monthly_historical_stats_by_template(): - month = get_london_month_from_utc_column(NotificationHistory.created_at) - year = func.date_trunc("year", NotificationHistory.created_at) - end_date = datetime.combine(date.today(), time.min) - - return db.session.query( - NotificationHistory.template_id, - extract('month', month).label('month'), - extract('year', year).label('year'), - func.count().label('count') - ).filter( - NotificationHistory.created_at < end_date - ).group_by( - NotificationHistory.template_id, - month, - year - ).order_by( - year, - month - ).all() - - -@statsd(namespace="dao") -def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year): - - results = dao_get_template_usage_stats_by_service(service_id, year) - - stats = [] - for result in results: - stat = type("", (), {})() - stat.template_id = result.template_id - stat.template_type = result.template_type - stat.name = str(result.name) - stat.month = result.month - stat.year = result.year - stat.count = result.count - stat.is_precompiled_letter = result.is_precompiled_letter - stats.append(stat) - - month = get_london_month_from_utc_column(Notification.created_at) - year_func = func.date_trunc("year", Notification.created_at) - start_date = datetime.combine(date.today(), time.min) - - fy_start, fy_end = get_financial_year(year) - - if fy_start < datetime.now() < fy_end: - today_results = db.session.query( - Notification.template_id, - Template.is_precompiled_letter, - Template.name, - Template.template_type, - extract('month', month).label('month'), - extract('year', year_func).label('year'), - func.count().label('count') - ).join( - Template, Notification.template_id == Template.id, - ).filter( - Notification.created_at >= start_date, - Notification.service_id == service_id, - # we don't want to include test keys - Notification.key_type != KEY_TYPE_TEST - ).group_by( - Notification.template_id, - Template.hidden, - Template.name, - Template.template_type, - month, - year_func - ).order_by( - Notification.template_id - ).all() - - for today_result in today_results: - add_to_stats = True - for stat in stats: - if today_result.template_id == stat.template_id and today_result.month == stat.month \ - and today_result.year == stat.year: - stat.count = stat.count + today_result.count - add_to_stats = False - - if add_to_stats: - new_stat = type("StatsTemplateUsageByMonth", (), {})() - new_stat.template_id = today_result.template_id - new_stat.template_type = today_result.template_type - new_stat.name = today_result.name - new_stat.month = int(today_result.month) - new_stat.year = int(today_result.year) - new_stat.count = today_result.count - new_stat.is_precompiled_letter = today_result.is_precompiled_letter - stats.append(new_stat) - - return stats diff --git a/app/dao/stats_template_usage_by_month_dao.py b/app/dao/stats_template_usage_by_month_dao.py deleted file mode 100644 index 541ab7193..000000000 --- a/app/dao/stats_template_usage_by_month_dao.py +++ /dev/null @@ -1,60 +0,0 @@ -from notifications_utils.statsd_decorators import statsd -from sqlalchemy import or_, and_, desc - -from app import db -from app.dao.dao_utils import transactional -from app.models import StatsTemplateUsageByMonth, Template - - -@transactional -@statsd(namespace="dao") -def insert_or_update_stats_for_template(template_id, month, year, count): - result = db.session.query( - StatsTemplateUsageByMonth - ).filter( - StatsTemplateUsageByMonth.template_id == template_id, - StatsTemplateUsageByMonth.month == month, - StatsTemplateUsageByMonth.year == year - ).update( - { - 'count': count - } - ) - if result == 0: - monthly_stats = StatsTemplateUsageByMonth( - template_id=template_id, - month=month, - year=year, - count=count - ) - - db.session.add(monthly_stats) - - -@statsd(namespace="dao") -def dao_get_template_usage_stats_by_service(service_id, year): - return db.session.query( - StatsTemplateUsageByMonth.template_id, - Template.name, - Template.template_type, - Template.is_precompiled_letter, - StatsTemplateUsageByMonth.month, - StatsTemplateUsageByMonth.year, - StatsTemplateUsageByMonth.count - ).join( - Template, StatsTemplateUsageByMonth.template_id == Template.id - ).filter( - Template.service_id == service_id - ).filter( - or_( - and_( - StatsTemplateUsageByMonth.month.in_([4, 5, 6, 7, 8, 9, 10, 11, 12]), - StatsTemplateUsageByMonth.year == year - ), and_( - StatsTemplateUsageByMonth.month.in_([1, 2, 3]), - StatsTemplateUsageByMonth.year == year + 1 - ) - ) - ).order_by( - desc(StatsTemplateUsageByMonth.month) - ).all() diff --git a/app/dao/templates_dao.py b/app/dao/templates_dao.py index e5e93199f..66cbe865a 100644 --- a/app/dao/templates_dao.py +++ b/app/dao/templates_dao.py @@ -129,18 +129,3 @@ def dao_get_template_versions(service_id, template_id): ).order_by( desc(TemplateHistory.version) ).all() - - -def dao_get_multiple_template_details(template_ids): - query = db.session.query( - Template.id, - Template.template_type, - Template.name, - Template.is_precompiled_letter - ).filter( - Template.id.in_(template_ids) - ).order_by( - Template.name - ) - - return query.all() diff --git a/app/job/rest.py b/app/job/rest.py index 6e64e045c..20f37efed 100644 --- a/app/job/rest.py +++ b/app/job/rest.py @@ -1,3 +1,4 @@ +import dateutil from flask import ( Blueprint, jsonify, @@ -13,6 +14,7 @@ from app.dao.jobs_dao import ( dao_get_jobs_by_service_id, dao_get_future_scheduled_job_by_id_and_service_id, dao_get_notification_outcomes_for_job) +from app.dao.fact_notification_status_dao import fetch_notification_statuses_for_job from app.dao.services_dao import dao_fetch_service_by_id from app.dao.templates_dao import dao_get_template_by_id from app.dao.notifications_dao import get_notifications_for_job @@ -24,7 +26,7 @@ from app.schemas import ( ) from app.celery.tasks import process_job from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANCELLED, LETTER_TYPE -from app.utils import pagination_links +from app.utils import pagination_links, midnight_n_days_ago from app.config import QueueNames from app.errors import ( register_errors, @@ -171,8 +173,18 @@ def get_paginated_jobs(service_id, limit_days, statuses, page): ) data = job_schema.dump(pagination.items, many=True).data for job_data in data: - statistics = dao_get_notification_outcomes_for_job(service_id, job_data['id']) - job_data['statistics'] = [{'status': statistic[1], 'count': statistic[0]} for statistic in statistics] + start = job_data['processing_started'] + start = dateutil.parser.parse(start).replace(tzinfo=None) if start else None + + if start is None: + statistics = [] + elif start.replace(tzinfo=None) < midnight_n_days_ago(3): + # ft_notification_status table + statistics = fetch_notification_statuses_for_job(job_data['id']) + else: + # notifications table + statistics = dao_get_notification_outcomes_for_job(service_id, job_data['id']) + job_data['statistics'] = [{'status': statistic.status, 'count': statistic.count} for statistic in statistics] return { 'data': data, diff --git a/app/letters/utils.py b/app/letters/utils.py index 0bf489435..c7e558244 100644 --- a/app/letters/utils.py +++ b/app/letters/utils.py @@ -162,3 +162,16 @@ def _move_s3_object(source_bucket, source_filename, target_bucket, target_filena current_app.logger.info("Moved letter PDF: {}/{} to {}/{}".format( source_bucket, source_filename, target_bucket, target_filename)) + + +def letter_print_day(created_at): + bst_print_datetime = convert_utc_to_bst(created_at) + timedelta(hours=6, minutes=30) + bst_print_date = bst_print_datetime.date() + + current_bst_date = convert_utc_to_bst(datetime.utcnow()).date() + + if bst_print_date >= current_bst_date: + return 'today' + else: + print_date = bst_print_datetime.strftime('%d %B').lstrip('0') + return 'on {}'.format(print_date) diff --git a/app/models.py b/app/models.py index e2d86679a..b4ef0b8d3 100644 --- a/app/models.py +++ b/app/models.py @@ -1,5 +1,4 @@ import itertools -import time import uuid import datetime from flask import url_for, current_app @@ -258,6 +257,7 @@ LETTERS_AS_PDF = 'letters_as_pdf' PRECOMPILED_LETTER = 'precompiled_letter' UPLOAD_DOCUMENT = 'upload_document' EDIT_FOLDERS = 'edit_folders' +CHOOSE_POSTAGE = 'choose_postage' SERVICE_PERMISSION_TYPES = [ EMAIL_TYPE, @@ -271,6 +271,7 @@ SERVICE_PERMISSION_TYPES = [ PRECOMPILED_LETTER, UPLOAD_DOCUMENT, EDIT_FOLDERS, + CHOOSE_POSTAGE ] @@ -763,6 +764,15 @@ class TemplateBase(db.Model): archived = db.Column(db.Boolean, nullable=False, default=False) hidden = db.Column(db.Boolean, nullable=False, default=False) subject = db.Column(db.Text) + postage = db.Column(db.String, nullable=True) + CheckConstraint(""" + CASE WHEN template_type = 'letter' THEN + postage in ('first', 'second') OR + postage is null + ELSE + postage is null + END + """) @declared_attr def service_id(cls): @@ -862,6 +872,7 @@ class TemplateBase(db.Model): } for key in self._as_utils_template().placeholders }, + "postage": self.postage, } return serialized @@ -1067,6 +1078,7 @@ class Job(db.Model): job_status = db.Column( db.String(255), db.ForeignKey('job_status.name'), index=True, nullable=False, default='pending' ) + archived = db.Column(db.Boolean, nullable=False, default=False) VERIFY_CODE_TYPES = [EMAIL_TYPE, SMS_TYPE] @@ -1134,6 +1146,7 @@ NOTIFICATION_STATUS_TYPES_COMPLETED = [ NOTIFICATION_TEMPORARY_FAILURE, NOTIFICATION_PERMANENT_FAILURE, NOTIFICATION_RETURNED_LETTER, + NOTIFICATION_CANCELLED, ] NOTIFICATION_STATUS_SUCCESS = [ @@ -1411,6 +1424,12 @@ class Notification(db.Model): else: return None + def get_created_by_email_address(self): + if self.created_by: + return self.created_by.email_address + else: + return None + def serialize_for_csv(self): created_at_in_bst = convert_utc_to_bst(self.created_at) serialized = { @@ -1420,8 +1439,9 @@ class Notification(db.Model): "template_type": self.template.template_type, "job_name": self.job.original_file_name if self.job else '', "status": self.formatted_status, - "created_at": time.strftime('%A %d %B %Y at %H:%M', created_at_in_bst.timetuple()), + "created_at": created_at_in_bst.strftime("%Y-%m-%d %H:%M:%S"), "created_by_name": self.get_created_by_name(), + "created_by_email_address": self.get_created_by_email_address(), } return serialized @@ -1814,48 +1834,6 @@ class AuthType(db.Model): name = db.Column(db.String, primary_key=True) -class StatsTemplateUsageByMonth(db.Model): - __tablename__ = "stats_template_usage_by_month" - - template_id = db.Column( - UUID(as_uuid=True), - db.ForeignKey('templates.id'), - unique=False, - index=True, - nullable=False, - primary_key=True - ) - month = db.Column( - db.Integer, - nullable=False, - index=True, - unique=False, - primary_key=True, - default=datetime.datetime.month - ) - year = db.Column( - db.Integer, - nullable=False, - index=True, - unique=False, - primary_key=True, - default=datetime.datetime.year - ) - count = db.Column( - db.Integer, - nullable=False, - default=0 - ) - - def serialize(self): - return { - 'template_id': str(self.template_id), - 'month': self.month, - 'year': self.year, - 'count': self.count - } - - class DailySortedLetter(db.Model): __tablename__ = "daily_sorted_letter" diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index e90cfeced..d75abcd4e 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -61,9 +61,6 @@ def process_ses_response(ses_request): notification_status ) if not notification: - warning = "SES callback failed: notification either not found or already updated " \ - "from sending. Status {} for notification reference {}".format(notification_status, reference) - current_app.logger.warning(warning) return if not aws_response_dict['success']: diff --git a/app/notifications/process_client_response.py b/app/notifications/process_client_response.py index d45c3cc92..3de60c587 100644 --- a/app/notifications/process_client_response.py +++ b/app/notifications/process_client_response.py @@ -81,10 +81,6 @@ def _process_for_status(notification_status, client_name, provider_reference): sent_by=client_name.lower() ) if not notification: - current_app.logger.warning("{} callback failed: notification {} either not found or already updated " - "from sending. Status {}".format(client_name, - provider_reference, - notification_status)) return statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status)) diff --git a/app/notifications/process_letter_notifications.py b/app/notifications/process_letter_notifications.py index 984ad257e..06d1127bf 100644 --- a/app/notifications/process_letter_notifications.py +++ b/app/notifications/process_letter_notifications.py @@ -7,6 +7,7 @@ def create_letter_notification(letter_data, template, api_key, status, reply_to_ notification = persist_notification( template_id=template.id, template_version=template.version, + template_postage=template.postage, # we only accept addresses_with_underscores from the API (from CSV we also accept dashes, spaces etc) recipient=letter_data['personalisation']['address_line_1'], service=template.service, @@ -20,6 +21,7 @@ def create_letter_notification(letter_data, template, api_key, status, reply_to_ client_reference=letter_data.get('reference'), status=status, reply_to_text=reply_to_text, - billable_units=billable_units + billable_units=billable_units, + postage=letter_data.get('postage') ) return notification diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index 6483ac766..99665ce78 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -9,7 +9,7 @@ from notifications_utils.recipients import ( validate_and_format_phone_number, format_email_address ) -from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst +from notifications_utils.timezones import convert_bst_to_utc from app import redis_store from app.celery import provider_tasks @@ -23,7 +23,8 @@ from app.models import ( LETTER_TYPE, NOTIFICATION_CREATED, Notification, - ScheduledNotification + ScheduledNotification, + CHOOSE_POSTAGE ) from app.dao.notifications_dao import ( dao_create_notification, @@ -32,11 +33,7 @@ from app.dao.notifications_dao import ( ) from app.v2.errors import BadRequestError -from app.utils import ( - cache_key_for_service_template_counter, - cache_key_for_service_template_usage_per_day, - get_template_instance, -) +from app.utils import get_template_instance def create_content_for_notification(template, personalisation): @@ -72,7 +69,9 @@ def persist_notification( created_by_id=None, status=NOTIFICATION_CREATED, reply_to_text=None, - billable_units=None + billable_units=None, + postage=None, + template_postage=None ): notification_created_at = created_at or datetime.utcnow() if not notification_id: @@ -109,7 +108,13 @@ def persist_notification( elif notification_type == EMAIL_TYPE: notification.normalised_to = format_email_address(notification.to) elif notification_type == LETTER_TYPE: - notification.postage = service.postage + if postage: + notification.postage = postage + else: + if service.has_permission(CHOOSE_POSTAGE) and template_postage: + notification.postage = template_postage + else: + notification.postage = service.postage # if simulated create a Notification model to return but do not persist the Notification to the dB if not simulated: @@ -117,10 +122,6 @@ def persist_notification( if key_type != KEY_TYPE_TEST: if redis_store.get(redis.daily_limit_cache_key(service.id)): redis_store.incr(redis.daily_limit_cache_key(service.id)) - if redis_store.get_all_from_hash(cache_key_for_service_template_counter(service.id)): - redis_store.increment_hash_value(cache_key_for_service_template_counter(service.id), template_id) - - increment_template_usage_cache(service.id, template_id, notification_created_at) current_app.logger.info( "{} {} created at {}".format(notification_type, notification_id, notification_created_at) @@ -128,15 +129,6 @@ def persist_notification( return notification -def increment_template_usage_cache(service_id, template_id, created_at): - key = cache_key_for_service_template_usage_per_day(service_id, convert_utc_to_bst(created_at)) - redis_store.increment_hash_value(key, template_id) - # set key to expire in eight days - we don't know if we've just created the key or not, so must assume that we - # have and reset the expiry. Eight days is longer than any notification is in the notifications table, so we'll - # always capture the full week's numbers - redis_store.expire(key, current_app.config['EXPIRE_CACHE_EIGHT_DAYS']) - - def send_notification_to_queue(notification, research_mode, queue=None): if research_mode or notification.key_type == KEY_TYPE_TEST: queue = QueueNames.RESEARCH_MODE diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 04286688a..aa4be0ea9 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -124,6 +124,7 @@ def send_notification(notification_type): simulated = simulated_recipient(notification_form['to'], notification_type) notification_model = persist_notification(template_id=template.id, template_version=template.version, + template_postage=template.postage, recipient=request.get_json()['to'], service=authenticated_service, personalisation=notification_form.get('personalisation', None), diff --git a/app/platform_stats/rest.py b/app/platform_stats/rest.py index 54e94cc47..efe936f84 100644 --- a/app/platform_stats/rest.py +++ b/app/platform_stats/rest.py @@ -2,7 +2,7 @@ from datetime import datetime from flask import Blueprint, jsonify, request -from app.dao.notifications_dao import fetch_aggregate_stats_by_date_range_for_all_services +from app.dao.fact_notification_status_dao import fetch_notification_status_totals_for_all_services from app.errors import register_errors from app.platform_stats.platform_stats_schema import platform_stats_request from app.service.statistics import format_admin_stats @@ -23,7 +23,7 @@ def get_platform_stats(): start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date() end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date() - data = fetch_aggregate_stats_by_date_range_for_all_services(start_date=start_date, end_date=end_date) + data = fetch_notification_status_totals_for_all_services(start_date=start_date, end_date=end_date) stats = format_admin_stats(data) return jsonify(stats) diff --git a/app/schema_validation/__init__.py b/app/schema_validation/__init__.py index 382d9229c..98e67a50a 100644 --- a/app/schema_validation/__init__.py +++ b/app/schema_validation/__init__.py @@ -8,41 +8,54 @@ from notifications_utils.recipients import (validate_phone_number, validate_emai InvalidEmailError) +format_checker = FormatChecker() + + +@format_checker.checks("validate_uuid", raises=Exception) +def validate_uuid(instance): + if isinstance(instance, str): + UUID(instance) + return True + + +@format_checker.checks('phone_number', raises=InvalidPhoneError) +def validate_schema_phone_number(instance): + if isinstance(instance, str): + validate_phone_number(instance, international=True) + return True + + +@format_checker.checks('email_address', raises=InvalidEmailError) +def validate_schema_email_address(instance): + if isinstance(instance, str): + validate_email_address(instance) + return True + + +@format_checker.checks('postage', raises=ValidationError) +def validate_schema_postage(instance): + if isinstance(instance, str): + if instance not in ["first", "second"]: + raise ValidationError("invalid. It must be either first or second.") + return True + + +@format_checker.checks('datetime_within_next_day', raises=ValidationError) +def validate_schema_date_with_hour(instance): + if isinstance(instance, str): + try: + dt = iso8601.parse_date(instance).replace(tzinfo=None) + if dt < datetime.utcnow(): + raise ValidationError("datetime can not be in the past") + if dt > datetime.utcnow() + timedelta(hours=24): + raise ValidationError("datetime can only be 24 hours in the future") + except ParseError: + raise ValidationError("datetime format is invalid. It must be a valid ISO8601 date time format, " + "https://en.wikipedia.org/wiki/ISO_8601") + return True + + def validate(json_to_validate, schema): - format_checker = FormatChecker() - - @format_checker.checks("validate_uuid", raises=Exception) - def validate_uuid(instance): - if isinstance(instance, str): - UUID(instance) - return True - - @format_checker.checks('phone_number', raises=InvalidPhoneError) - def validate_schema_phone_number(instance): - if isinstance(instance, str): - validate_phone_number(instance, international=True) - return True - - @format_checker.checks('email_address', raises=InvalidEmailError) - def validate_schema_email_address(instance): - if isinstance(instance, str): - validate_email_address(instance) - return True - - @format_checker.checks('datetime_within_next_day', raises=ValidationError) - def validate_schema_date_with_hour(instance): - if isinstance(instance, str): - try: - dt = iso8601.parse_date(instance).replace(tzinfo=None) - if dt < datetime.utcnow(): - raise ValidationError("datetime can not be in the past") - if dt > datetime.utcnow() + timedelta(hours=24): - raise ValidationError("datetime can only be 24 hours in the future") - except ParseError: - raise ValidationError("datetime format is invalid. It must be a valid ISO8601 date time format, " - "https://en.wikipedia.org/wiki/ISO_8601") - return True - validator = Draft7Validator(schema, format_checker=format_checker) errors = list(validator.iter_errors(json_to_validate)) if errors.__len__() > 0: diff --git a/app/schemas.py b/app/schemas.py index 460b52240..0ebc42dc4 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -596,6 +596,7 @@ class NotificationsFilterSchema(ma.Schema): format_for_csv = fields.String() to = fields.String() include_one_off = fields.Boolean(required=False) + count_pages = fields.Boolean(required=False) @pre_load def handle_multidict(self, in_data): diff --git a/app/service/rest.py b/app/service/rest.py index ce0b37048..279d00ed5 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -7,6 +7,8 @@ from flask import ( current_app, Blueprint ) +from notifications_utils.letter_timings import letter_can_be_cancelled +from notifications_utils.timezones import convert_utc_to_bst from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound @@ -21,7 +23,8 @@ from app.dao.api_key_dao import ( from app.dao.fact_notification_status_dao import ( fetch_notification_status_for_service_by_month, fetch_notification_status_for_service_for_day, - fetch_notification_status_for_service_for_today_and_7_previous_days + fetch_notification_status_for_service_for_today_and_7_previous_days, + fetch_stats_for_all_services_by_date_range, fetch_monthly_template_usage_for_service ) from app.dao.inbound_numbers_dao import dao_allocate_number_for_service from app.dao.organisation_dao import dao_get_organisation_by_service_id @@ -46,7 +49,6 @@ from app.dao.services_dao import ( dao_create_service, dao_fetch_all_services, dao_fetch_all_services_by_user, - dao_fetch_monthly_historical_usage_by_template_for_service, dao_fetch_service_by_id, dao_fetch_todays_stats_for_service, dao_fetch_todays_stats_for_all_services, @@ -54,7 +56,6 @@ from app.dao.services_dao import ( dao_remove_user_from_service, dao_suspend_service, dao_update_service, - fetch_stats_by_date_range_for_all_services ) from app.dao.service_whitelist_dao import ( dao_fetch_service_whitelist, @@ -80,7 +81,8 @@ from app.errors import ( InvalidRequest, register_errors ) -from app.models import Service, EmailBranding +from app.letters.utils import letter_print_day +from app.models import LETTER_TYPE, NOTIFICATION_CANCELLED, Service, EmailBranding from app.schema_validation import validate from app.service import statistics from app.service.service_data_retention_schema import ( @@ -103,7 +105,7 @@ from app.schemas import ( notifications_filter_schema, detailed_service_schema ) -from app.utils import pagination_links, convert_utc_to_bst +from app.utils import pagination_links service_blueprint = Blueprint('service', __name__) @@ -342,16 +344,20 @@ def get_all_notifications_for_service(service_id): include_from_test_key = data.get('include_from_test_key', False) include_one_off = data.get('include_one_off', True) + count_pages = data.get('count_pages', True) + pagination = notifications_dao.get_notifications_for_service( service_id, filter_dict=data, page=page, page_size=page_size, + count_pages=count_pages, limit_days=limit_days, include_jobs=include_jobs, include_from_test_key=include_from_test_key, include_one_off=include_one_off ) + kwargs = request.args.to_dict() kwargs['service_id'] = service_id @@ -384,6 +390,31 @@ def get_notification_for_service(service_id, notification_id): ), 200 +@service_blueprint.route('//notifications//cancel', methods=['POST']) +def cancel_notification_for_service(service_id, notification_id): + notification = notifications_dao.get_notification_by_id(notification_id, service_id) + + if not notification: + raise InvalidRequest('Notification not found', status_code=404) + elif notification.notification_type != LETTER_TYPE: + raise InvalidRequest('Notification cannot be cancelled - only letters can be cancelled', status_code=400) + elif not letter_can_be_cancelled(notification.status, notification.created_at): + print_day = letter_print_day(notification.created_at) + + raise InvalidRequest( + "It’s too late to cancel this letter. Printing started {} at 5.30pm".format(print_day), + status_code=400) + + updated_notification = notifications_dao.update_notification_status_by_id( + notification_id, + NOTIFICATION_CANCELLED, + ) + + return jsonify( + notification_with_template_schema.dump(updated_notification).data + ), 200 + + def search_for_notification_by_to_field(service_id, search_term, statuses, notification_type): results = notifications_dao.dao_get_notifications_by_to_field( service_id=service_id, @@ -444,17 +475,14 @@ def get_detailed_services(start_date, end_date, only_active=False, include_from_ only_active=only_active) else: - stats = fetch_stats_by_date_range_for_all_services(start_date=start_date, + stats = fetch_stats_for_all_services_by_date_range(start_date=start_date, end_date=end_date, include_from_test_key=include_from_test_key, - only_active=only_active) + ) results = [] for service_id, rows in itertools.groupby(stats, lambda x: x.service_id): rows = list(rows) - if rows[0].count is None: - s = statistics.create_zeroed_stats_dicts() - else: - s = statistics.format_statistics(rows) + s = statistics.format_statistics(rows) results.append({ 'id': str(rows[0].service_id), 'name': rows[0].name, @@ -551,11 +579,12 @@ def resume_service(service_id): @service_blueprint.route('//notifications/templates_usage/monthly', methods=['GET']) def get_monthly_template_usage(service_id): try: - data = dao_fetch_monthly_historical_usage_by_template_for_service( - service_id, - int(request.args.get('year', 'NaN')) + start_date, end_date = get_financial_year(int(request.args.get('year', 'NaN'))) + data = fetch_monthly_template_usage_for_service( + start_date=start_date, + end_date=end_date, + service_id=service_id ) - stats = list() for i in data: stats.append( diff --git a/app/service/send_notification.py b/app/service/send_notification.py index b80baf770..26307b8c3 100644 --- a/app/service/send_notification.py +++ b/app/service/send_notification.py @@ -1,5 +1,6 @@ from sqlalchemy.orm.exc import NoResultFound +from app import create_random_identifier from app.config import QueueNames from app.dao.notifications_dao import _update_notification_status from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id @@ -37,6 +38,12 @@ def validate_created_by(service, created_by_id): raise BadRequestError(message=message) +def create_one_off_reference(template_type): + if template_type == LETTER_TYPE: + return create_random_identifier() + return None + + def send_one_off_notification(service_id, post_data): service = dao_fetch_service_by_id(service_id) template = dao_get_template_by_id_and_service_id( @@ -70,6 +77,7 @@ def send_one_off_notification(service_id, post_data): notification = persist_notification( template_id=template.id, template_version=template.version, + template_postage=template.postage, recipient=post_data['to'], service=service, personalisation=personalisation, @@ -77,7 +85,8 @@ def send_one_off_notification(service_id, post_data): api_key_id=None, key_type=KEY_TYPE_NORMAL, created_by_id=post_data['created_by'], - reply_to_text=reply_to + reply_to_text=reply_to, + reference=create_one_off_reference(template.template_type), ) queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None diff --git a/app/service/statistics.py b/app/service/statistics.py index 3d22b20d6..5c6accaa5 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -13,7 +13,10 @@ def format_statistics(statistics): # so we can return emails/sms * created, sent, and failed counts = create_zeroed_stats_dicts() for row in statistics: - _update_statuses_from_row(counts[row.notification_type], row) + # any row could be null, if the service either has no notifications in the notifications table, + # or no historical data in the ft_notification_status table. + if row.notification_type: + _update_statuses_from_row(counts[row.notification_type], row) return counts @@ -81,7 +84,8 @@ def create_zeroed_stats_dicts(): def _update_statuses_from_row(update_dict, row): - update_dict['requested'] += row.count + if row.status != 'cancelled': + update_dict['requested'] += row.count if row.status in ('delivered', 'sent'): update_dict['delivered'] += row.count elif row.status in ( diff --git a/app/template/rest.py b/app/template/rest.py index 8cbb4125f..79b1d7d8b 100644 --- a/app/template/rest.py +++ b/app/template/rest.py @@ -31,7 +31,7 @@ from app.errors import ( InvalidRequest ) from app.letters.utils import get_letter_pdf -from app.models import SMS_TYPE, Template +from app.models import SMS_TYPE, Template, CHOOSE_POSTAGE from app.notifications.validators import service_has_permission, check_reply_to from app.schema_validation import validate from app.schemas import (template_schema, template_history_schema) @@ -78,6 +78,12 @@ def create_template(service_id): errors = {'template_type': [message]} raise InvalidRequest(errors, 403) + if new_template.postage: + if not service_has_permission(CHOOSE_POSTAGE, fetched_service.permissions): + message = "Setting postage on templates is not enabled for this service." + errors = {'template_postage': [message]} + raise InvalidRequest(errors, 403) + new_template.service = fetched_service over_limit = _content_count_greater_than_limit(new_template.content, new_template.template_type) @@ -110,6 +116,12 @@ def update_template(service_id, template_id): if data.get('redact_personalisation') is True: return redact_template(fetched_template, data) + if data.get('postage'): + if not service_has_permission(CHOOSE_POSTAGE, fetched_template.service.permissions): + message = "Setting postage on templates is not enabled for this service." + errors = {'template_postage': [message]} + raise InvalidRequest(errors, 403) + if "reply_to" in data: check_reply_to(service_id, data.get("reply_to"), fetched_template.template_type) updated = dao_update_template_reply_to(template_id=template_id, reply_to=data.get("reply_to")) @@ -191,7 +203,7 @@ def get_template_versions(service_id, template_id): def _template_has_not_changed(current_data, updated_template): return all( current_data[key] == updated_template[key] - for key in ('name', 'content', 'subject', 'archived', 'process_type') + for key in ('name', 'content', 'subject', 'archived', 'process_type', 'postage') ) diff --git a/app/template/template_schemas.py b/app/template/template_schemas.py index f63ad4575..9f38262a5 100644 --- a/app/template/template_schemas.py +++ b/app/template/template_schemas.py @@ -17,7 +17,8 @@ post_create_template_schema = { "content": {"type": "string"}, "subject": {"type": "string"}, "created_by": uuid, - "parent_folder_id": uuid + "parent_folder_id": uuid, + "postage": {"type": "string"}, }, "if": { "properties": { diff --git a/app/template_folder/rest.py b/app/template_folder/rest.py index f2f79c521..010bc1a66 100644 --- a/app/template_folder/rest.py +++ b/app/template_folder/rest.py @@ -141,11 +141,8 @@ def move_to_template_folder(service_id, target_template_folder_id=None): def _validate_folder_move(target_template_folder, target_template_folder_id, template_folder, template_folder_id): if str(target_template_folder_id) == str(template_folder_id): - msg = 'Could not move folder to itself' + msg = 'You cannot move a folder to itself' raise InvalidRequest(msg, status_code=400) if target_template_folder and template_folder.is_parent_of(target_template_folder): - msg = 'Could not move to folder: {} is an ancestor of target folder {}'.format( - template_folder_id, - target_template_folder_id - ) + msg = 'You cannot move a folder to one of its subfolders' raise InvalidRequest(msg, status_code=400) diff --git a/app/template_statistics/rest.py b/app/template_statistics/rest.py index 1c0f3b27d..fc179b49a 100644 --- a/app/template_statistics/rest.py +++ b/app/template_statistics/rest.py @@ -1,24 +1,10 @@ -from flask import ( - Blueprint, - jsonify, - request, - current_app -) - -from app import redis_store -from app.dao.notifications_dao import ( - dao_get_template_usage, - dao_get_last_template_usage -) -from app.dao.templates_dao import ( - dao_get_multiple_template_details, - dao_get_template_by_id_and_service_id -) +from flask import Blueprint, jsonify, request +from app.dao.notifications_dao import dao_get_last_template_usage +from app.dao.templates_dao import dao_get_template_by_id_and_service_id +from app.dao.fact_notification_status_dao import fetch_notification_status_for_service_for_today_and_7_previous_days from app.schemas import notification_with_template_schema -from app.utils import cache_key_for_service_template_usage_per_day, last_n_days from app.errors import register_errors, InvalidRequest -from collections import Counter template_statistics = Blueprint('template_statistics', __name__, @@ -39,8 +25,21 @@ def get_template_statistics_for_service_by_day(service_id): if whole_days < 0 or whole_days > 7: raise InvalidRequest({'whole_days': ['whole_days must be between 0 and 7']}, status_code=400) + data = fetch_notification_status_for_service_for_today_and_7_previous_days( + service_id, by_template=True, limit_days=whole_days + ) - return jsonify(data=_get_template_statistics_for_last_n_days(service_id, whole_days)) + return jsonify(data=[ + { + 'count': row.count, + 'template_id': str(row.template_id), + 'template_name': row.template_name, + 'template_type': row.notification_type, + 'is_precompiled_letter': row.is_precompiled_letter, + 'status': row.status + } + for row in data + ]) @template_statistics.route('/') @@ -53,50 +52,3 @@ def get_template_statistics_for_template_id(service_id, template_id): data = notification_with_template_schema.dump(notification).data return jsonify(data=data) - - -def _get_template_statistics_for_last_n_days(service_id, whole_days): - template_stats_by_id = Counter() - - # 0 whole_days = last 1 days (ie since midnight today) = today. - # 7 whole days = last 8 days (ie since midnight this day last week) = a week and a bit - for day in last_n_days(whole_days + 1): - # "{SERVICE_ID}-template-usage-{YYYY-MM-DD}" - key = cache_key_for_service_template_usage_per_day(service_id, day) - stats = redis_store.get_all_from_hash(key) - if stats: - stats = { - k.decode('utf-8'): int(v) for k, v in stats.items() - } - else: - # key didn't exist (or redis was down) - lets populate from DB. - stats = { - str(row.id): row.count for row in dao_get_template_usage(service_id, day=day) - } - # if there is data in db, but not in redis - lets put it in redis so we don't have to do - # this calc again next time. If there isn't any data, we can't put it in redis. - # Zero length hashes aren't a thing in redis. (There'll only be no data if the service has no templates) - # Nothing is stored if redis is down. - if stats: - redis_store.set_hash_and_expire( - key, - stats, - current_app.config['EXPIRE_CACHE_EIGHT_DAYS'] - ) - template_stats_by_id += Counter(stats) - - # attach count from stats to name/type/etc from database - template_details = dao_get_multiple_template_details(template_stats_by_id.keys()) - return [ - { - 'count': template_stats_by_id[str(template.id)], - 'template_id': str(template.id), - 'template_name': template.name, - 'template_type': template.template_type, - 'is_precompiled_letter': template.is_precompiled_letter - } - for template in template_details - # we don't want to return templates with no count to the front-end, - # but they're returned from the DB and might be put in redis like that (if there was no data that day) - if template_stats_by_id[str(template.id)] != 0 - ] diff --git a/app/user/rest.py b/app/user/rest.py index 0bca01a4c..65097b6d4 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -449,7 +449,8 @@ def get_orgs_and_services(user): 'services': [ { 'id': service.id, - 'name': service.name + 'name': service.name, + 'restricted': service.restricted, } for service in org.services if service.active and service in user.services @@ -460,7 +461,8 @@ def get_orgs_and_services(user): 'services_without_organisations': [ { 'id': service.id, - 'name': service.name + 'name': service.name, + 'restricted': service.restricted, } for service in user.services if ( service.active and diff --git a/app/utils.py b/app/utils.py index b00a53bda..d8916341f 100644 --- a/app/utils.py +++ b/app/utils.py @@ -68,17 +68,6 @@ def get_london_month_from_utc_column(column): ) -def cache_key_for_service_template_counter(service_id, limit_days=7): - return "{}-template-counter-limit-{}-days".format(service_id, limit_days) - - -def cache_key_for_service_template_usage_per_day(service_id, datetime): - """ - You should pass a BST datetime into this function - """ - return "service-{}-template-usage-{}".format(service_id, datetime.date().isoformat()) - - def get_public_notify_type_text(notify_type, plural=False): from app.models import (SMS_TYPE, UPLOAD_DOCUMENT, PRECOMPILED_LETTER) notify_type_text = notify_type diff --git a/app/v2/notifications/notification_schemas.py b/app/v2/notifications/notification_schemas.py index 39c78d727..733eb8aef 100644 --- a/app/v2/notifications/notification_schemas.py +++ b/app/v2/notifications/notification_schemas.py @@ -239,7 +239,8 @@ post_precompiled_letter_request = { "title": "POST v2/notifications/letter", "properties": { "reference": {"type": "string"}, - "content": {"type": "string"} + "content": {"type": "string"}, + "postage": {"type": "string", "format": "postage"} }, "required": ["reference", "content"], "additionalProperties": False diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index b40e4e4b6..511155a7e 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -94,7 +94,8 @@ def post_precompiled_letter_notification(): resp = { 'id': notification.id, - 'reference': notification.client_reference + 'reference': notification.client_reference, + 'postage': notification.postage } return jsonify(resp), 201 diff --git a/app/v2/template/template_schemas.py b/app/v2/template/template_schemas.py index b1b1f4820..ebd0b8342 100644 --- a/app/v2/template/template_schemas.py +++ b/app/v2/template/template_schemas.py @@ -37,6 +37,7 @@ get_template_by_id_response = { "body": {"type": "string"}, "subject": {"type": ["string", "null"]}, "name": {"type": "string"}, + "postage": {"type": "string"} }, "required": ["id", "type", "created_at", "updated_at", "version", "created_by", "body", "name"], } @@ -63,7 +64,8 @@ post_template_preview_response = { "type": {"enum": TEMPLATE_TYPES}, "version": {"type": "integer"}, "body": {"type": "string"}, - "subject": {"type": ["string", "null"]} + "subject": {"type": ["string", "null"]}, + "postage": {"type": "string"} }, "required": ["id", "type", "version", "body"] } @@ -77,5 +79,6 @@ def create_post_template_preview_response(template, template_object): "type": template.template_type, "version": template.version, "body": str(template_object), - "subject": subject + "subject": subject, + "postage": template.postage } diff --git a/manifest-api-base.yml b/manifest-api-base.yml index 10096f97e..3ef91e6bd 100644 --- a/manifest-api-base.yml +++ b/manifest-api-base.yml @@ -22,6 +22,7 @@ env: SECRET_KEY: null ROUTE_SECRET_KEY_1: null ROUTE_SECRET_KEY_2: null + CRONITOR_KEYS: null PERFORMANCE_PLATFORM_ENDPOINTS: null diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 5ce75e7fc..bf136b1df 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -20,6 +20,7 @@ env: SECRET_KEY: null ROUTE_SECRET_KEY_1: null ROUTE_SECRET_KEY_2: null + CRONITOR_KEYS: null PERFORMANCE_PLATFORM_ENDPOINTS: null @@ -67,7 +68,7 @@ applications: - name: notify-delivery-worker-sender command: scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO -Q send-sms-tasks,send-email-tasks - memory: 2G + memory: 3G env: NOTIFY_APP_NAME: delivery-worker-sender diff --git a/migrations/versions/0245_archived_flag_jobs.py b/migrations/versions/0245_archived_flag_jobs.py new file mode 100644 index 000000000..cfcbb8f1f --- /dev/null +++ b/migrations/versions/0245_archived_flag_jobs.py @@ -0,0 +1,28 @@ +""" + +Revision ID: 0245_archived_flag_jobs +Revises: 0244_another_letter_org +Create Date: 2018-11-22 16:32:01.105803 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = '0245_archived_flag_jobs' +down_revision = '0244_another_letter_org' + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('jobs', sa.Column('archived', sa.Boolean(), nullable=True)) + op.execute('update jobs set archived = false') + op.alter_column('jobs', 'archived', nullable=False, server_default=sa.false()) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('jobs', 'archived') + # ### end Alembic commands ### diff --git a/migrations/versions/0246_notifications_index.py b/migrations/versions/0246_notifications_index.py new file mode 100644 index 000000000..37d8fd772 --- /dev/null +++ b/migrations/versions/0246_notifications_index.py @@ -0,0 +1,26 @@ +""" + +Revision ID: 0246_notifications_index +Revises: 0245_archived_flag_jobs +Create Date: 2018-12-12 12:00:09.770775 + +""" +from alembic import op + +revision = '0246_notifications_index' +down_revision = '0245_archived_flag_jobs' + + +def upgrade(): + conn = op.get_bind() + conn.execute( + "CREATE INDEX IF NOT EXISTS ix_notifications_service_created_at ON notifications (service_id, created_at)" + ) + + +def downgrade(): + conn = op.get_bind() + conn.execute( + "DROP INDEX IF EXISTS ix_notifications_service_created_at" + ) + diff --git a/migrations/versions/0247_another_letter_org.py b/migrations/versions/0247_another_letter_org.py new file mode 100644 index 000000000..be2a988c0 --- /dev/null +++ b/migrations/versions/0247_another_letter_org.py @@ -0,0 +1,35 @@ +"""empty message + +Revision ID: 0247_another_letter_org +Revises: 0246_notifications_index + +""" + +# revision identifiers, used by Alembic. +revision = '0247_another_letter_org' +down_revision = '0246_notifications_index' + +from alembic import op + + +NEW_ORGANISATIONS = [ + ('520', 'Neath Port Talbot Council', 'npt'), +] + + +def upgrade(): + for numeric_id, name, filename in NEW_ORGANISATIONS: + op.execute(""" + INSERT + INTO dvla_organisation + VALUES ('{}', '{}', '{}') + """.format(numeric_id, name, filename)) + + +def downgrade(): + for numeric_id, _, _ in NEW_ORGANISATIONS: + op.execute(""" + DELETE + FROM dvla_organisation + WHERE id = '{}' + """.format(numeric_id)) diff --git a/migrations/versions/0248_enable_choose_postage.py b/migrations/versions/0248_enable_choose_postage.py new file mode 100644 index 000000000..b72d6749f --- /dev/null +++ b/migrations/versions/0248_enable_choose_postage.py @@ -0,0 +1,54 @@ +""" + +Revision ID: 0248_enable_choose_postage +Revises: 0247_another_letter_org +Create Date: 2018-12-14 12:09:31.375634 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = '0248_enable_choose_postage' +down_revision = '0247_another_letter_org' + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.execute("INSERT INTO service_permission_types VALUES ('choose_postage')") + op.add_column('templates', sa.Column('postage', sa.String(), nullable=True)) + op.add_column('templates_history', sa.Column('postage', sa.String(), nullable=True)) + op.execute(""" + ALTER TABLE templates ADD CONSTRAINT "chk_templates_postage_null" + CHECK ( + CASE WHEN template_type = 'letter' THEN + postage in ('first', 'second') OR + postage is null + ELSE + postage is null + END + ) + """) + op.execute(""" + ALTER TABLE templates_history ADD CONSTRAINT "chk_templates_history_postage_null" + CHECK ( + CASE WHEN template_type = 'letter' THEN + postage in ('first', 'second') OR + postage is null + ELSE + postage is null + END + ) + """) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('chk_templates_history_postage_null', 'templates_history', type_='check') + op.drop_constraint('chk_templates_postage_null', 'templates', type_='check') + op.drop_column('templates_history', 'postage') + op.drop_column('templates', 'postage') + op.execute("DELETE FROM service_permissions WHERE permission = 'choose_postage'") + op.execute("DELETE FROM service_permission_types WHERE name = 'choose_postage'") + # ### end Alembic commands ### diff --git a/migrations/versions/0249_another_letter_org.py b/migrations/versions/0249_another_letter_org.py new file mode 100644 index 000000000..e4423ede8 --- /dev/null +++ b/migrations/versions/0249_another_letter_org.py @@ -0,0 +1,35 @@ +"""empty message + +Revision ID: 0249_another_letter_org +Revises: 0248_enable_choose_postage + +""" + +# revision identifiers, used by Alembic. +revision = '0249_another_letter_org' +down_revision = '0248_enable_choose_postage' + +from alembic import op + + +NEW_ORGANISATIONS = [ + ('521', 'North Somerset Council', 'north-somerset'), +] + + +def upgrade(): + for numeric_id, name, filename in NEW_ORGANISATIONS: + op.execute(""" + INSERT + INTO dvla_organisation + VALUES ('{}', '{}', '{}') + """.format(numeric_id, name, filename)) + + +def downgrade(): + for numeric_id, _, _ in NEW_ORGANISATIONS: + op.execute(""" + DELETE + FROM dvla_organisation + WHERE id = '{}' + """.format(numeric_id)) diff --git a/migrations/versions/0250_drop_stats_template_table.py b/migrations/versions/0250_drop_stats_template_table.py new file mode 100644 index 000000000..f44af5384 --- /dev/null +++ b/migrations/versions/0250_drop_stats_template_table.py @@ -0,0 +1,36 @@ +""" + +Revision ID: 0250_drop_stats_template_table +Revises: 0249_another_letter_org +Create Date: 2019-01-15 16:47:08.049369 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0250_drop_stats_template_table' +down_revision = '0249_another_letter_org' + + +def upgrade(): + op.drop_index('ix_stats_template_usage_by_month_month', table_name='stats_template_usage_by_month') + op.drop_index('ix_stats_template_usage_by_month_template_id', table_name='stats_template_usage_by_month') + op.drop_index('ix_stats_template_usage_by_month_year', table_name='stats_template_usage_by_month') + op.drop_table('stats_template_usage_by_month') + + +def downgrade(): + op.create_table('stats_template_usage_by_month', + sa.Column('template_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column('month', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('year', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('count', sa.INTEGER(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['template_id'], ['templates.id'], + name='stats_template_usage_by_month_template_id_fkey'), + sa.PrimaryKeyConstraint('template_id', 'month', 'year', name='stats_template_usage_by_month_pkey') + ) + op.create_index('ix_stats_template_usage_by_month_year', 'stats_template_usage_by_month', ['year'], unique=False) + op.create_index('ix_stats_template_usage_by_month_template_id', 'stats_template_usage_by_month', ['template_id'], + unique=False) + op.create_index('ix_stats_template_usage_by_month_month', 'stats_template_usage_by_month', ['month'], unique=False) diff --git a/migrations/versions/0251_another_letter_org.py b/migrations/versions/0251_another_letter_org.py new file mode 100644 index 000000000..2344da9d5 --- /dev/null +++ b/migrations/versions/0251_another_letter_org.py @@ -0,0 +1,39 @@ +"""empty message + +Revision ID: 0251_another_letter_org +Revises: 0250_drop_stats_template_table + +""" + +# revision identifiers, used by Alembic. +revision = '0251_another_letter_org' +down_revision = '0250_drop_stats_template_table' + +from alembic import op + + +NEW_ORGANISATIONS = [ + ('522', 'Anglesey Council', 'anglesey'), + ('523', 'Angus Council', 'angus'), + ('524', 'Cheshire East Council', 'cheshire-east'), + ('525', 'Newham Council', 'newham'), + ('526', 'Warwickshire Council', 'warwickshire'), +] + + +def upgrade(): + for numeric_id, name, filename in NEW_ORGANISATIONS: + op.execute(""" + INSERT + INTO dvla_organisation + VALUES ('{}', '{}', '{}') + """.format(numeric_id, name, filename)) + + +def downgrade(): + for numeric_id, _, _ in NEW_ORGANISATIONS: + op.execute(""" + DELETE + FROM dvla_organisation + WHERE id = '{}' + """.format(numeric_id)) diff --git a/requirements-app.txt b/requirements-app.txt index 849b2bfca..648c4180b 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -7,7 +7,7 @@ docopt==0.6.2 Flask-Bcrypt==0.7.1 flask-marshmallow==0.9.0 Flask-Migrate==2.3.1 -Flask-SQLAlchemy==2.3.2 +git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 Flask==1.0.2 click-datetime==0.2 eventlet==0.23.0 @@ -29,6 +29,6 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@30.7.1#egg=notifications-utils==30.7.1 +git+https://github.com/alphagov/notifications-utils.git@30.7.4#egg=notifications-utils==30.7.4 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/requirements.txt b/requirements.txt index 1df4c529f..e508e33f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ docopt==0.6.2 Flask-Bcrypt==0.7.1 flask-marshmallow==0.9.0 Flask-Migrate==2.3.0 -Flask-SQLAlchemy==2.3.2 +git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 Flask==1.0.2 click-datetime==0.2 eventlet==0.23.0 @@ -31,22 +31,22 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@30.7.1#egg=notifications-utils==30.7.1 +git+https://github.com/alphagov/notifications-utils.git@30.7.4#egg=notifications-utils==30.7.4 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 ## The following requirements were added by pip freeze: -alembic==1.0.3 +alembic==1.0.5 amqp==1.4.9 anyjson==0.3.3 attrs==18.2.0 -awscli==1.16.62 -bcrypt==3.1.4 +awscli==1.16.85 +bcrypt==3.1.5 billiard==3.3.0.23 -bleach==2.1.3 +bleach==3.0.2 boto3==1.6.16 -botocore==1.12.52 -certifi==2018.10.15 +botocore==1.12.75 +certifi==2018.11.29 chardet==3.0.4 Click==7.0 colorama==0.3.9 @@ -54,33 +54,32 @@ docutils==0.14 Flask-Redis==0.3.0 future==0.17.1 greenlet==0.4.15 -html5lib==1.0.1 -idna==2.7 +idna==2.8 Jinja2==2.10 jmespath==0.9.3 kombu==3.0.37 Mako==1.0.7 MarkupSafe==1.1.0 -mistune==0.8.3 +mistune==0.8.4 monotonic==1.5 orderedset==2.0.1 -phonenumbers==8.9.4 -pyasn1==0.4.4 +phonenumbers==8.10.2 +pyasn1==0.4.5 pycparser==2.19 PyPDF2==1.26.0 -pyrsistent==0.14.7 +pyrsistent==0.14.9 python-dateutil==2.7.5 python-editor==1.0.3 -python-json-logger==0.1.8 -pytz==2018.7 +python-json-logger==0.1.10 +pytz==2018.9 PyYAML==3.12 -redis==2.10.6 -requests==2.20.1 +redis==3.0.1 +requests==2.21.0 rsa==3.4.2 s3transfer==0.1.13 -six==1.11.0 +six==1.12.0 smartypants==2.0.1 -statsd==3.2.2 +statsd==3.3.0 urllib3==1.24.1 webencodings==0.5.1 Werkzeug==0.14.1 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 8e750c8e0..13951402f 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -4,8 +4,8 @@ pytest==3.10.1 moto==1.3.7 pytest-env==0.6.2 pytest-mock==1.10.0 -pytest-cov==2.6.0 -pytest-xdist==1.24.1 +pytest-cov==2.6.1 +pytest-xdist==1.26.0 coveralls==1.5.1 freezegun==0.3.11 requests-mock==1.5.2 diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh index 3965ee577..6824923ea 100755 --- a/scripts/run_multi_worker_app_paas.sh +++ b/scripts/run_multi_worker_app_paas.sh @@ -39,50 +39,48 @@ log_stream_name = {hostname} EOF } -# For every PID, check if it's still running -# if it is, send the sigterm +# For every PID, check if it's still running. if it is, send the sigterm. then wait 9 seconds before sending sigkill function on_exit { + echo "multi worker app exiting" wait_time=0 - while true; do - # refresh pids to account for the case that - # some workers may have terminated but others not + + send_signal_to_celery_processes TERM + + # check if the apps are still running every second + while [[ "$wait_time" -le "$TERMINATE_TIMEOUT" ]]; do get_celery_pids # look here for explanation regarding this syntax: # https://unix.stackexchange.com/a/298942/230401 PROCESS_COUNT="${#APP_PIDS[@]}" if [[ "${PROCESS_COUNT}" -eq "0" ]]; then - echo "No more .pid files found, exiting" - break + echo "No celery process is running any more, exiting" + return 0 fi - echo "Terminating celery processes with pids "${APP_PIDS} - for APP_PID in ${APP_PIDS}; do - # if TERMINATE_TIMEOUT is reached, send SIGKILL - if [[ "$wait_time" -ge "$TERMINATE_TIMEOUT" ]]; then - echo "Timeout reached, killing process with pid ${APP_PID}" - kill -9 ${APP_PID} || true - continue - else - echo "Timeout not reached yet, checking " ${APP_PID} - # else, if process is still running send SIGTERM - if [[ $(kill -0 ${APP_PID} 2&>/dev/null) ]]; then - echo "Terminating celery process with pid ${APP_PID}" - kill ${APP_PID} || true - fi - fi - done let wait_time=wait_time+1 sleep 1 done + + send_signal_to_celery_processes KILL } function get_celery_pids { - if [[ $(ls /home/vcap/app/celery*.pid) ]]; then - APP_PIDS=`cat /home/vcap/app/celery*.pid` - else - APP_PIDS=() - fi + # get the PIDs of the process whose parent is the root process + # print only pid and their command, get the ones with "celery" in their name + # and keep only these PIDs + + set +o pipefail # so grep returning no matches does not premature fail pipe + APP_PIDS=$(pgrep -P 1 | xargs ps -o pid=,command= -p | grep celery | cut -f1 -d/) + set -o pipefail # pipefail should be set everywhere else +} + +function send_signal_to_celery_processes { + # refresh pids to account for the case that some workers may have terminated but others not + get_celery_pids + # send signal to all remaining apps + echo ${APP_PIDS} | tr -d '\n' | tr -s ' ' | xargs echo "Sending signal ${1} to processes with pids: " + echo ${APP_PIDS} | xargs kill -s ${1} } function start_application { @@ -103,9 +101,32 @@ function start_logs_tail { echo "tail pid: ${LOGS_TAIL_PID}" } +function ensure_celery_is_running { + if [ "${APP_PIDS}" = "" ]; then + echo "There are no celery processes running, this container is bad" + + echo "Exporting CF information for diagnosis" + + env | grep CF + + echo "Sleeping 15 seconds for logs to get shipped" + + sleep 15 + + echo "Killing awslogs_agent and tail" + kill -9 ${AWSLOGS_AGENT_PID} + kill -9 ${LOGS_TAIL_PID} + + exit 1 + fi +} + function run { while true; do get_celery_pids + + ensure_celery_is_running + for APP_PID in ${APP_PIDS}; do kill -0 ${APP_PID} 2&>/dev/null || return 1 done diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index 4af7edca2..ce7c9cd79 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -23,7 +23,6 @@ from app.celery.letters_pdf_tasks import ( process_virus_scan_failed, process_virus_scan_error, replay_letters_in_error, - _get_page_count, _sanitise_precompiled_pdf ) from app.letters.utils import get_letter_pdf_filename, ScanErrorType @@ -417,6 +416,7 @@ def test_process_letter_task_check_virus_scan_passed_when_sanitise_fails( process_virus_scan_passed(filename) assert sample_letter_notification.status == NOTIFICATION_VALIDATION_FAILED + assert sample_letter_notification.billable_units == 0 mock_sanitise.assert_called_once_with( ANY, sample_letter_notification, @@ -432,13 +432,44 @@ def test_process_letter_task_check_virus_scan_passed_when_sanitise_fails( ) -def test_get_page_count_set_notification_to_permanent_failure_when_not_pdf( - sample_letter_notification +@freeze_time('2018-01-01 18:00') +@mock_s3 +@pytest.mark.parametrize('key_type,is_test_letter', [ + (KEY_TYPE_NORMAL, False), (KEY_TYPE_TEST, True) +]) +def test_process_letter_task_check_virus_scan_passed_when_file_cannot_be_opened( + sample_letter_notification, mocker, key_type, is_test_letter ): - with pytest.raises(expected_exception=PdfReadError): - _get_page_count(sample_letter_notification, b'pdf_content') - updated_notification = Notification.query.filter_by(id=sample_letter_notification.id).first() - assert updated_notification.status == NOTIFICATION_VALIDATION_FAILED + filename = 'NOTIFY.{}'.format(sample_letter_notification.reference) + source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME'] + target_bucket_name = current_app.config['INVALID_PDF_BUCKET_NAME'] + + conn = boto3.resource('s3', region_name='eu-west-1') + conn.create_bucket(Bucket=source_bucket_name) + conn.create_bucket(Bucket=target_bucket_name) + + s3 = boto3.client('s3', region_name='eu-west-1') + s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'pdf_content') + + sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK + sample_letter_notification.key_type = key_type + mock_move_s3 = mocker.patch('app.letters.utils._move_s3_object') + + mock_get_page_count = mocker.patch('app.celery.letters_pdf_tasks._get_page_count', side_effect=PdfReadError) + mock_sanitise = mocker.patch('app.celery.letters_pdf_tasks._sanitise_precompiled_pdf') + + process_virus_scan_passed(filename) + + mock_sanitise.assert_not_called() + mock_get_page_count.assert_called_once_with( + sample_letter_notification, b'pdf_content' + ) + mock_move_s3.assert_called_once_with( + source_bucket_name, filename, + target_bucket_name, filename + ) + assert sample_letter_notification.status == NOTIFICATION_VALIDATION_FAILED + assert sample_letter_notification.billable_units == 0 def test_process_letter_task_check_virus_scan_failed(sample_letter_notification, mocker): @@ -508,6 +539,27 @@ def test_sanitise_precompiled_pdf_returns_none_on_validation_error(rmock, sample assert res is None +def test_sanitise_precompiled_pdf_passes_the_service_id_and_notification_id_to_template_preview( + mocker, + sample_letter_notification, +): + tp_mock = mocker.patch('app.celery.letters_pdf_tasks.requests_post') + sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK + mock_celery = Mock(**{'retry.side_effect': Retry}) + _sanitise_precompiled_pdf(mock_celery, sample_letter_notification, b'old_pdf') + + service_id = str(sample_letter_notification.service_id) + notification_id = str(sample_letter_notification.id) + + tp_mock.assert_called_once_with( + 'http://localhost:9999/precompiled/sanitise', + data=b'old_pdf', + headers={'Authorization': 'Token my-secret-key', + 'Service-ID': service_id, + 'Notification-ID': notification_id} + ) + + def test_sanitise_precompiled_pdf_retries_on_http_error(rmock, sample_letter_notification): sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK rmock.post('http://localhost:9999/precompiled/sanitise', content=b'new_pdf', status_code=500) diff --git a/tests/app/celery/test_nightly_tasks.py b/tests/app/celery/test_nightly_tasks.py new file mode 100644 index 000000000..93e047448 --- /dev/null +++ b/tests/app/celery/test_nightly_tasks.py @@ -0,0 +1,570 @@ +from datetime import datetime, timedelta +from functools import partial +from unittest.mock import call, patch, PropertyMock + +import pytest +import pytz +from flask import current_app +from freezegun import freeze_time +from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient + +from app.celery import nightly_tasks +from app.celery.nightly_tasks import ( + delete_dvla_response_files_older_than_seven_days, + delete_email_notifications_older_than_seven_days, + delete_inbound_sms_older_than_seven_days, + delete_letter_notifications_older_than_seven_days, + delete_sms_notifications_older_than_seven_days, + raise_alert_if_letter_notifications_still_sending, + remove_letter_csv_files, + remove_sms_email_csv_files, + remove_transformed_dvla_files, + s3, + send_daily_performance_platform_stats, + send_total_sent_notifications_to_performance_platform, + timeout_notifications, + letter_raise_alert_if_no_ack_file_for_zip, +) +from app.celery.service_callback_tasks import create_delivery_status_callback_data +from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient +from app.config import QueueNames +from app.exceptions import NotificationTechnicalFailureException +from app.models import ( + LETTER_TYPE, + SMS_TYPE, + EMAIL_TYPE +) +from app.utils import get_london_midnight_in_utc +from tests.app.aws.test_s3 import single_s3_object_stub +from tests.app.db import ( + create_notification, + create_service, + create_template, + create_job, + create_service_callback_api, + create_service_data_retention +) + +from tests.app.conftest import datetime_in_past + + +def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None): + if subfolder == '2018-01-11/zips_sent': + return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT'] + if subfolder == 'root/dispatch': + return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] + + +def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None): + if subfolder == '2018-01-11/zips_sent': + return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT', + 'NOTIFY.20180111175010.ZIP.TXT'] + if subfolder == 'root/dispatch': + return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] + + +@freeze_time('2016-10-18T10:00:00') +def test_will_remove_csv_files_for_jobs_older_than_seven_days( + notify_db, notify_db_session, mocker, sample_template +): + """ + Jobs older than seven days are deleted, but only two day's worth (two-day window) + """ + mocker.patch('app.celery.nightly_tasks.s3.remove_job_from_s3') + + seven_days_ago = datetime.utcnow() - timedelta(days=7) + just_under_seven_days = seven_days_ago + timedelta(seconds=1) + eight_days_ago = seven_days_ago - timedelta(days=1) + nine_days_ago = eight_days_ago - timedelta(days=1) + just_under_nine_days = nine_days_ago + timedelta(seconds=1) + nine_days_one_second_ago = nine_days_ago - timedelta(seconds=1) + + create_job(sample_template, created_at=nine_days_one_second_ago, archived=True) + job1_to_delete = create_job(sample_template, created_at=eight_days_ago) + job2_to_delete = create_job(sample_template, created_at=just_under_nine_days) + dont_delete_me_1 = create_job(sample_template, created_at=seven_days_ago) + create_job(sample_template, created_at=just_under_seven_days) + + remove_sms_email_csv_files() + + assert s3.remove_job_from_s3.call_args_list == [ + call(job1_to_delete.service_id, job1_to_delete.id), + call(job2_to_delete.service_id, job2_to_delete.id), + ] + assert job1_to_delete.archived is True + assert dont_delete_me_1.archived is False + + +@freeze_time('2016-10-18T10:00:00') +def test_will_remove_csv_files_for_jobs_older_than_retention_period( + notify_db, notify_db_session, mocker +): + """ + Jobs older than retention period are deleted, but only two day's worth (two-day window) + """ + mocker.patch('app.celery.nightly_tasks.s3.remove_job_from_s3') + service_1 = create_service(service_name='service 1') + service_2 = create_service(service_name='service 2') + create_service_data_retention(service_id=service_1.id, notification_type=SMS_TYPE, days_of_retention=3) + create_service_data_retention(service_id=service_2.id, notification_type=EMAIL_TYPE, days_of_retention=30) + sms_template_service_1 = create_template(service=service_1) + email_template_service_1 = create_template(service=service_1, template_type='email') + + sms_template_service_2 = create_template(service=service_2) + email_template_service_2 = create_template(service=service_2, template_type='email') + + four_days_ago = datetime.utcnow() - timedelta(days=4) + eight_days_ago = datetime.utcnow() - timedelta(days=8) + thirty_one_days_ago = datetime.utcnow() - timedelta(days=31) + + job1_to_delete = create_job(sms_template_service_1, created_at=four_days_ago) + job2_to_delete = create_job(email_template_service_1, created_at=eight_days_ago) + create_job(email_template_service_1, created_at=four_days_ago) + + create_job(email_template_service_2, created_at=eight_days_ago) + job3_to_delete = create_job(email_template_service_2, created_at=thirty_one_days_ago) + job4_to_delete = create_job(sms_template_service_2, created_at=eight_days_ago) + + remove_sms_email_csv_files() + + s3.remove_job_from_s3.assert_has_calls([ + call(job1_to_delete.service_id, job1_to_delete.id), + call(job2_to_delete.service_id, job2_to_delete.id), + call(job3_to_delete.service_id, job3_to_delete.id), + call(job4_to_delete.service_id, job4_to_delete.id) + ], any_order=True) + + +@freeze_time('2017-01-01 10:00:00') +def test_remove_csv_files_filters_by_type(mocker, sample_service): + mocker.patch('app.celery.nightly_tasks.s3.remove_job_from_s3') + """ + Jobs older than seven days are deleted, but only two day's worth (two-day window) + """ + letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) + sms_template = create_template(service=sample_service, template_type=SMS_TYPE) + + eight_days_ago = datetime.utcnow() - timedelta(days=8) + + job_to_delete = create_job(template=letter_template, created_at=eight_days_ago) + create_job(template=sms_template, created_at=eight_days_ago) + + remove_letter_csv_files() + + assert s3.remove_job_from_s3.call_args_list == [ + call(job_to_delete.service_id, job_to_delete.id), + ] + + +def test_should_call_delete_sms_notifications_more_than_week_in_task(notify_api, mocker): + mocked = mocker.patch('app.celery.nightly_tasks.delete_notifications_created_more_than_a_week_ago_by_type') + delete_sms_notifications_older_than_seven_days() + mocked.assert_called_once_with('sms') + + +def test_should_call_delete_email_notifications_more_than_week_in_task(notify_api, mocker): + mocked_notifications = mocker.patch( + 'app.celery.nightly_tasks.delete_notifications_created_more_than_a_week_ago_by_type') + delete_email_notifications_older_than_seven_days() + mocked_notifications.assert_called_once_with('email') + + +def test_should_call_delete_letter_notifications_more_than_week_in_task(notify_api, mocker): + mocked = mocker.patch('app.celery.nightly_tasks.delete_notifications_created_more_than_a_week_ago_by_type') + delete_letter_notifications_older_than_seven_days() + mocked.assert_called_once_with('letter') + + +def test_update_status_of_notifications_after_timeout(notify_api, sample_template): + with notify_api.test_request_context(): + not1 = create_notification( + template=sample_template, + status='sending', + created_at=datetime.utcnow() - timedelta( + seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) + not2 = create_notification( + template=sample_template, + status='created', + created_at=datetime.utcnow() - timedelta( + seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) + not3 = create_notification( + template=sample_template, + status='pending', + created_at=datetime.utcnow() - timedelta( + seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) + with pytest.raises(NotificationTechnicalFailureException) as e: + timeout_notifications() + assert str(not2.id) in e.value.message + assert not1.status == 'temporary-failure' + assert not2.status == 'technical-failure' + assert not3.status == 'temporary-failure' + + +def test_not_update_status_of_notification_before_timeout(notify_api, sample_template): + with notify_api.test_request_context(): + not1 = create_notification( + template=sample_template, + status='sending', + created_at=datetime.utcnow() - timedelta( + seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') - 10)) + timeout_notifications() + assert not1.status == 'sending' + + +def test_should_not_update_status_of_letter_notifications(client, sample_letter_template): + created_at = datetime.utcnow() - timedelta(days=5) + not1 = create_notification(template=sample_letter_template, status='sending', created_at=created_at) + not2 = create_notification(template=sample_letter_template, status='created', created_at=created_at) + + timeout_notifications() + + assert not1.status == 'sending' + assert not2.status == 'created' + + +def test_timeout_notifications_sends_status_update_to_service(client, sample_template, mocker): + callback_api = create_service_callback_api(service=sample_template.service) + mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async') + notification = create_notification( + template=sample_template, + status='sending', + created_at=datetime.utcnow() - timedelta( + seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) + timeout_notifications() + + encrypted_data = create_delivery_status_callback_data(notification, callback_api) + mocked.assert_called_once_with([str(notification.id), encrypted_data], queue=QueueNames.CALLBACKS) + + +def test_send_daily_performance_stats_calls_does_not_send_if_inactive(client, mocker): + send_mock = mocker.patch( + 'app.celery.nightly_tasks.total_sent_notifications.send_total_notifications_sent_for_day_stats') # noqa + + with patch.object( + PerformancePlatformClient, + 'active', + new_callable=PropertyMock + ) as mock_active: + mock_active.return_value = False + send_daily_performance_platform_stats() + + assert send_mock.call_count == 0 + + +@freeze_time("2016-01-11 12:30:00") +def test_send_total_sent_notifications_to_performance_platform_calls_with_correct_totals( + notify_db, + notify_db_session, + sample_template, + sample_email_template, + mocker +): + sms = sample_template + email = sample_email_template + + perf_mock = mocker.patch( + 'app.celery.nightly_tasks.total_sent_notifications.send_total_notifications_sent_for_day_stats') # noqa + + create_notification(email, status='delivered') + create_notification(sms, status='delivered') + + # Create some notifications for the day before + yesterday = datetime(2016, 1, 10, 15, 30, 0, 0) + with freeze_time(yesterday): + create_notification(sms, status='delivered') + create_notification(sms, status='delivered') + create_notification(email, status='delivered') + create_notification(email, status='delivered') + create_notification(email, status='delivered') + + with patch.object( + PerformancePlatformClient, + 'active', + new_callable=PropertyMock + ) as mock_active: + mock_active.return_value = True + send_total_sent_notifications_to_performance_platform(yesterday) + + perf_mock.assert_has_calls([ + call(get_london_midnight_in_utc(yesterday), 'sms', 2), + call(get_london_midnight_in_utc(yesterday), 'email', 3) + ]) + + +def test_should_call_delete_inbound_sms_older_than_seven_days(notify_api, mocker): + mocker.patch('app.celery.nightly_tasks.delete_inbound_sms_created_more_than_a_week_ago') + delete_inbound_sms_older_than_seven_days() + assert nightly_tasks.delete_inbound_sms_created_more_than_a_week_ago.call_count == 1 + + +@freeze_time('2017-01-01 10:00:00') +def test_remove_dvla_transformed_files_removes_expected_files(mocker, sample_service): + mocker.patch('app.celery.nightly_tasks.s3.remove_transformed_dvla_file') + + letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) + + job = partial(create_job, template=letter_template) + + seven_days_ago = datetime.utcnow() - timedelta(days=7) + just_under_seven_days = seven_days_ago + timedelta(seconds=1) + just_over_seven_days = seven_days_ago - timedelta(seconds=1) + eight_days_ago = seven_days_ago - timedelta(days=1) + nine_days_ago = eight_days_ago - timedelta(days=1) + ten_days_ago = nine_days_ago - timedelta(days=1) + just_under_nine_days = nine_days_ago + timedelta(seconds=1) + just_over_nine_days = nine_days_ago - timedelta(seconds=1) + just_over_ten_days = ten_days_ago - timedelta(seconds=1) + + job(created_at=just_under_seven_days) + job(created_at=just_over_seven_days) + job_to_delete_1 = job(created_at=eight_days_ago) + job_to_delete_2 = job(created_at=nine_days_ago) + job_to_delete_3 = job(created_at=just_under_nine_days) + job_to_delete_4 = job(created_at=just_over_nine_days) + job(created_at=just_over_ten_days) + remove_transformed_dvla_files() + + s3.remove_transformed_dvla_file.assert_has_calls([ + call(job_to_delete_1.id), + call(job_to_delete_2.id), + call(job_to_delete_3.id), + call(job_to_delete_4.id), + ], any_order=True) + + +def test_remove_dvla_transformed_files_does_not_remove_files(mocker, sample_service): + mocker.patch('app.celery.nightly_tasks.s3.remove_transformed_dvla_file') + + letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) + + job = partial(create_job, template=letter_template) + + yesterday = datetime.utcnow() - timedelta(days=1) + six_days_ago = datetime.utcnow() - timedelta(days=6) + seven_days_ago = six_days_ago - timedelta(days=1) + just_over_nine_days = seven_days_ago - timedelta(days=2, seconds=1) + + job(created_at=yesterday) + job(created_at=six_days_ago) + job(created_at=seven_days_ago) + job(created_at=just_over_nine_days) + + remove_transformed_dvla_files() + + s3.remove_transformed_dvla_file.assert_has_calls([]) + + +@freeze_time("2016-01-01 11:00:00") +def test_delete_dvla_response_files_older_than_seven_days_removes_old_files(notify_api, mocker): + AFTER_SEVEN_DAYS = datetime_in_past(days=8) + single_page_s3_objects = [{ + "Contents": [ + single_s3_object_stub('bar/foo1.txt', AFTER_SEVEN_DAYS), + single_s3_object_stub('bar/foo2.txt', AFTER_SEVEN_DAYS), + ] + }] + mocker.patch( + 'app.celery.nightly_tasks.s3.get_s3_bucket_objects', return_value=single_page_s3_objects[0]["Contents"] + ) + remove_s3_mock = mocker.patch('app.celery.nightly_tasks.s3.remove_s3_object') + + delete_dvla_response_files_older_than_seven_days() + + remove_s3_mock.assert_has_calls([ + call(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], single_page_s3_objects[0]["Contents"][0]["Key"]), + call(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], single_page_s3_objects[0]["Contents"][1]["Key"]) + ]) + + +@freeze_time("2016-01-01 11:00:00") +def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(notify_api, mocker): + START_DATE = datetime_in_past(days=9) + JUST_BEFORE_START_DATE = datetime_in_past(days=9, seconds=1) + END_DATE = datetime_in_past(days=7) + JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1) + + single_page_s3_objects = [{ + "Contents": [ + single_s3_object_stub('bar/foo1.txt', JUST_BEFORE_START_DATE), + single_s3_object_stub('bar/foo2.txt', START_DATE), + single_s3_object_stub('bar/foo3.txt', END_DATE), + single_s3_object_stub('bar/foo4.txt', JUST_AFTER_END_DATE), + ] + }] + mocker.patch( + 'app.celery.nightly_tasks.s3.get_s3_bucket_objects', return_value=single_page_s3_objects[0]["Contents"] + ) + remove_s3_mock = mocker.patch('app.celery.nightly_tasks.s3.remove_s3_object') + delete_dvla_response_files_older_than_seven_days() + + remove_s3_mock.assert_not_called() + + +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending(sample_letter_template, mocker): + two_days_ago = datetime(2018, 1, 15, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=two_days_ago) + + mock_create_ticket = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_create_ticket.assert_called_once_with( + subject="[test] Letters still sending", + message="There are 1 letters in the 'sending' state from Monday 15 January", + ticket_type=ZendeskClient.TYPE_INCIDENT + ) + + +def test_alert_if_letter_notifications_still_sending_a_day_ago_no_alert(sample_letter_template, mocker): + today = datetime.utcnow() + one_day_ago = today - timedelta(days=1) + create_notification(template=sample_letter_template, status='sending', sent_at=one_day_ago) + + mock_create_ticket = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + assert not mock_create_ticket.called + + +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_letter_template, mocker): + two_days_ago = datetime(2018, 1, 15, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=two_days_ago) + create_notification(template=sample_letter_template, status='delivered', sent_at=two_days_ago) + create_notification(template=sample_letter_template, status='failed', sent_at=two_days_ago) + + mock_create_ticket = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_create_ticket.assert_called_once_with( + subject="[test] Letters still sending", + message="There are 1 letters in the 'sending' state from Monday 15 January", + ticket_type='incident' + ) + + +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending_alerts_for_older_than_offset(sample_letter_template, mocker): + three_days_ago = datetime(2018, 1, 14, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=three_days_ago) + + mock_create_ticket = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_create_ticket.assert_called_once_with( + subject="[test] Letters still sending", + message="There are 1 letters in the 'sending' state from Monday 15 January", + ticket_type='incident' + ) + + +@freeze_time("2018-01-14 17:00:00") +def test_alert_if_letter_notifications_still_sending_does_nothing_on_the_weekend(sample_letter_template, mocker): + yesterday = datetime(2018, 1, 13, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_create_ticket = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + assert not mock_create_ticket.called + + +@freeze_time("2018-01-15 17:00:00") +def test_monday_alert_if_letter_notifications_still_sending_reports_thursday_letters(sample_letter_template, mocker): + thursday = datetime(2018, 1, 11, 13, 30) + yesterday = datetime(2018, 1, 14, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=thursday) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_create_ticket = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_create_ticket.assert_called_once_with( + subject="[test] Letters still sending", + message="There are 1 letters in the 'sending' state from Thursday 11 January", + ticket_type='incident' + ) + + +@freeze_time("2018-01-16 17:00:00") +def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_letters(sample_letter_template, mocker): + friday = datetime(2018, 1, 12, 13, 30) + yesterday = datetime(2018, 1, 14, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=friday) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_create_ticket = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_create_ticket.assert_called_once_with( + subject="[test] Letters still sending", + message="There are 1 letters in the 'sending' state from Friday 12 January", + ticket_type='incident' + ) + + +@freeze_time('2018-01-11T23:00:00') +def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db): + mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match) + mock_get_file = mocker.patch("app.aws.s3.get_s3_file", + return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' + 'NOTIFY.20180111175008.ZIP|20180111175734') + + letter_raise_alert_if_no_ack_file_for_zip() + + yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # Datatime format on AWS + subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent' + assert mock_file_list.call_count == 2 + assert mock_file_list.call_args_list == [ + call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'), + call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', + suffix='.ACK.txt', last_modified=yesterday), + ] + assert mock_get_file.call_count == 1 + + +@freeze_time('2018-01-11T23:00:00') +def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db): + mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff) + mock_get_file = mocker.patch("app.aws.s3.get_s3_file", + return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' + 'NOTIFY.20180111175008.ZIP|20180111175734') + mock_zendesk = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") + + letter_raise_alert_if_no_ack_file_for_zip() + + assert mock_file_list.call_count == 2 + assert mock_get_file.call_count == 1 + + message = "Letter ack file does not contain all zip files sent. " \ + "Missing ack for zip files: {}, " \ + "pdf bucket: {}, subfolder: {}, " \ + "ack bucket: {}".format(str(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']), + current_app.config['LETTERS_PDF_BUCKET_NAME'], + datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', + current_app.config['DVLA_RESPONSE_BUCKET_NAME']) + + mock_zendesk.assert_called_once_with( + subject="Letter acknowledge error", + message=message, + ticket_type='incident' + ) + + +@freeze_time('2018-01-11T23:00:00') +def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db): + mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None) + mock_get_file = mocker.patch("app.aws.s3.get_s3_file", + return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' + 'NOTIFY.20180111175008.ZIP|20180111175734') + + letter_raise_alert_if_no_ack_file_for_zip() + + assert mock_file_list.call_count == 2 + assert mock_get_file.call_count == 0 diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index 8918a33ce..ade175db2 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -20,10 +20,6 @@ from app import db from tests.app.db import create_service, create_template, create_notification -def test_reporting_should_have_decorated_tasks_functions(): - assert create_nightly_billing.__wrapped__.__name__ == 'create_nightly_billing' - - def mocker_get_rate( non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None, post_class="second" ): diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 5ce737764..bf4eb9507 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -1,42 +1,20 @@ -import functools from datetime import datetime, timedelta -from functools import partial -from unittest.mock import call, patch, PropertyMock +from unittest.mock import call import pytest -import pytz -from flask import current_app from freezegun import freeze_time -from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient from app import db from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( check_job_status, - delete_dvla_response_files_older_than_seven_days, - delete_email_notifications_older_than_seven_days, - delete_inbound_sms_older_than_seven_days, delete_invitations, - delete_notifications_created_more_than_a_week_ago_by_type, - delete_letter_notifications_older_than_seven_days, - delete_sms_notifications_older_than_seven_days, delete_verify_codes, - raise_alert_if_letter_notifications_still_sending, - remove_csv_files, - remove_transformed_dvla_files, run_scheduled_jobs, - s3, - send_daily_performance_platform_stats, send_scheduled_notifications, - send_total_sent_notifications_to_performance_platform, switch_current_sms_provider_on_slow_delivery, - timeout_notifications, - daily_stats_template_usage_by_month, - letter_raise_alert_if_no_ack_file_for_zip, replay_created_notifications ) -from app.celery.service_callback_tasks import create_delivery_status_callback_data -from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.config import QueueNames, TaskNames from app.dao.jobs_dao import dao_get_job_by_id from app.dao.notifications_dao import dao_get_scheduled_notifications @@ -44,105 +22,40 @@ from app.dao.provider_details_dao import ( dao_update_provider_details, get_current_provider ) -from app.exceptions import NotificationTechnicalFailureException from app.models import ( - NotificationHistory, - Service, - StatsTemplateUsageByMonth, JOB_STATUS_IN_PROGRESS, JOB_STATUS_ERROR, - LETTER_TYPE, - SMS_TYPE + JOB_STATUS_FINISHED, ) -from app.utils import get_london_midnight_in_utc from app.v2.errors import JobIncompleteError -from tests.app.aws.test_s3 import single_s3_object_stub -from tests.app.conftest import ( - sample_job as create_sample_job, - sample_notification_history as create_notification_history, - sample_template as create_sample_template, - create_custom_template, - datetime_in_past -) + from tests.app.db import ( - create_notification, create_service, create_template, create_job, create_service_callback_api + create_notification, + create_template, + create_job, ) -from tests.conftest import set_config_values +from tests.app.conftest import sample_job as create_sample_job -def _create_slow_delivery_notification(provider='mmg'): +def _create_slow_delivery_notification(template, provider='mmg'): now = datetime.utcnow() five_minutes_from_now = now + timedelta(minutes=5) - service = Service.query.get(current_app.config['FUNCTIONAL_TEST_PROVIDER_SERVICE_ID']) - if not service: - service = create_service( - service_id=current_app.config.get('FUNCTIONAL_TEST_PROVIDER_SERVICE_ID') - ) - - template = create_custom_template( - service=service, - user=service.users[0], - template_config_name='FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID', - template_type='sms' - ) create_notification( template=template, status='delivered', sent_by=provider, - updated_at=five_minutes_from_now + updated_at=five_minutes_from_now, + sent_at=now, ) -@pytest.mark.skip(reason="This doesn't actually test the celery task wraps the function") -def test_should_have_decorated_tasks_functions(): - """ - TODO: This test needs to be reviewed as this doesn't actually - test that the celery task is wrapping the function. We're also - running similar tests elsewhere which also need review. - """ - assert delete_verify_codes.__wrapped__.__name__ == 'delete_verify_codes' - assert delete_notifications_created_more_than_a_week_ago_by_type.__wrapped__.__name__ == \ - 'delete_notifications_created_more_than_a_week_ago_by_type' - assert timeout_notifications.__wrapped__.__name__ == 'timeout_notifications' - assert delete_invitations.__wrapped__.__name__ == 'delete_invitations' - assert run_scheduled_jobs.__wrapped__.__name__ == 'run_scheduled_jobs' - assert remove_csv_files.__wrapped__.__name__ == 'remove_csv_files' - assert send_daily_performance_platform_stats.__wrapped__.__name__ == 'send_daily_performance_platform_stats' - assert switch_current_sms_provider_on_slow_delivery.__wrapped__.__name__ == \ - 'switch_current_sms_provider_on_slow_delivery' - assert delete_inbound_sms_older_than_seven_days.__wrapped__.__name__ == \ - 'delete_inbound_sms_older_than_seven_days' - assert remove_transformed_dvla_files.__wrapped__.__name__ == \ - 'remove_transformed_dvla_files' - assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \ - 'delete_dvla_response_files_older_than_seven_days' - - @pytest.fixture(scope='function') def prepare_current_provider(restore_provider_details): initial_provider = get_current_provider('sms') - initial_provider.updated_at = datetime.utcnow() - timedelta(minutes=30) dao_update_provider_details(initial_provider) - - -def test_should_call_delete_sms_notifications_more_than_week_in_task(notify_api, mocker): - mocked = mocker.patch('app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago_by_type') - delete_sms_notifications_older_than_seven_days() - mocked.assert_called_once_with('sms') - - -def test_should_call_delete_email_notifications_more_than_week_in_task(notify_api, mocker): - mocked_notifications = mocker.patch( - 'app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago_by_type') - delete_email_notifications_older_than_seven_days() - mocked_notifications.assert_called_once_with('email') - - -def test_should_call_delete_letter_notifications_more_than_week_in_task(notify_api, mocker): - mocked = mocker.patch('app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago_by_type') - delete_letter_notifications_older_than_seven_days() - mocked.assert_called_once_with('letter') + initial_provider.updated_at = datetime.utcnow() - timedelta(minutes=30) + db.session.commit() def test_should_call_delete_codes_on_delete_verify_codes_task(notify_api, mocker): @@ -157,67 +70,6 @@ def test_should_call_delete_invotations_on_delete_invitations_task(notify_api, m assert scheduled_tasks.delete_invitations_created_more_than_two_days_ago.call_count == 1 -def test_update_status_of_notifications_after_timeout(notify_api, sample_template): - with notify_api.test_request_context(): - not1 = create_notification( - template=sample_template, - status='sending', - created_at=datetime.utcnow() - timedelta( - seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) - not2 = create_notification( - template=sample_template, - status='created', - created_at=datetime.utcnow() - timedelta( - seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) - not3 = create_notification( - template=sample_template, - status='pending', - created_at=datetime.utcnow() - timedelta( - seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) - with pytest.raises(NotificationTechnicalFailureException) as e: - timeout_notifications() - assert str(not2.id) in e.value.message - assert not1.status == 'temporary-failure' - assert not2.status == 'technical-failure' - assert not3.status == 'temporary-failure' - - -def test_not_update_status_of_notification_before_timeout(notify_api, sample_template): - with notify_api.test_request_context(): - not1 = create_notification( - template=sample_template, - status='sending', - created_at=datetime.utcnow() - timedelta( - seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') - 10)) - timeout_notifications() - assert not1.status == 'sending' - - -def test_should_not_update_status_of_letter_notifications(client, sample_letter_template): - created_at = datetime.utcnow() - timedelta(days=5) - not1 = create_notification(template=sample_letter_template, status='sending', created_at=created_at) - not2 = create_notification(template=sample_letter_template, status='created', created_at=created_at) - - timeout_notifications() - - assert not1.status == 'sending' - assert not2.status == 'created' - - -def test_timeout_notifications_sends_status_update_to_service(client, sample_template, mocker): - callback_api = create_service_callback_api(service=sample_template.service) - mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async') - notification = create_notification( - template=sample_template, - status='sending', - created_at=datetime.utcnow() - timedelta( - seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) - timeout_notifications() - - encrypted_data = create_delivery_status_callback_data(notification, callback_api) - mocked.assert_called_once_with([str(notification.id), encrypted_data], queue=QueueNames.CALLBACKS) - - def test_should_update_scheduled_jobs_and_put_on_queue(notify_db, notify_db_session, mocker): mocked = mocker.patch('app.celery.tasks.process_job.apply_async') @@ -269,222 +121,31 @@ def test_should_update_all_scheduled_jobs_and_put_on_queue(notify_db, notify_db_ ]) -@freeze_time('2016-10-18T10:00:00') -def test_will_remove_csv_files_for_jobs_older_than_seven_days( - notify_db, notify_db_session, mocker, sample_template -): - mocker.patch('app.celery.scheduled_tasks.s3.remove_job_from_s3') - """ - Jobs older than seven days are deleted, but only two day's worth (two-day window) - """ - seven_days_ago = datetime.utcnow() - timedelta(days=7) - just_under_seven_days = seven_days_ago + timedelta(seconds=1) - eight_days_ago = seven_days_ago - timedelta(days=1) - nine_days_ago = eight_days_ago - timedelta(days=1) - just_under_nine_days = nine_days_ago + timedelta(seconds=1) - nine_days_one_second_ago = nine_days_ago - timedelta(seconds=1) - - create_sample_job(notify_db, notify_db_session, created_at=nine_days_one_second_ago) - job1_to_delete = create_sample_job(notify_db, notify_db_session, created_at=eight_days_ago) - job2_to_delete = create_sample_job(notify_db, notify_db_session, created_at=just_under_nine_days) - create_sample_job(notify_db, notify_db_session, created_at=seven_days_ago) - create_sample_job(notify_db, notify_db_session, created_at=just_under_seven_days) - - remove_csv_files(job_types=[sample_template.template_type]) - - assert s3.remove_job_from_s3.call_args_list == [ - call(job1_to_delete.service_id, job1_to_delete.id), - call(job2_to_delete.service_id, job2_to_delete.id) - ] - - -def test_send_daily_performance_stats_calls_does_not_send_if_inactive(client, mocker): - send_mock = mocker.patch( - 'app.celery.scheduled_tasks.total_sent_notifications.send_total_notifications_sent_for_day_stats') # noqa - - with patch.object( - PerformancePlatformClient, - 'active', - new_callable=PropertyMock - ) as mock_active: - mock_active.return_value = False - send_daily_performance_platform_stats() - - assert send_mock.call_count == 0 - - -@freeze_time("2016-01-11 12:30:00") -def test_send_total_sent_notifications_to_performance_platform_calls_with_correct_totals( - notify_db, - notify_db_session, - sample_template, - mocker -): - perf_mock = mocker.patch( - 'app.celery.scheduled_tasks.total_sent_notifications.send_total_notifications_sent_for_day_stats') # noqa - - notification_history = partial( - create_notification_history, - notify_db, - notify_db_session, - sample_template, - status='delivered' - ) - - notification_history(notification_type='email') - notification_history(notification_type='sms') - - # Create some notifications for the day before - yesterday = datetime(2016, 1, 10, 15, 30, 0, 0) - with freeze_time(yesterday): - notification_history(notification_type='sms') - notification_history(notification_type='sms') - notification_history(notification_type='email') - notification_history(notification_type='email') - notification_history(notification_type='email') - - with patch.object( - PerformancePlatformClient, - 'active', - new_callable=PropertyMock - ) as mock_active: - mock_active.return_value = True - send_total_sent_notifications_to_performance_platform(yesterday) - - perf_mock.assert_has_calls([ - call(get_london_midnight_in_utc(yesterday), 'sms', 2), - call(get_london_midnight_in_utc(yesterday), 'email', 3) - ]) - - -def test_switch_current_sms_provider_on_slow_delivery_does_not_run_if_config_unset( - notify_api, - mocker -): - get_notifications_mock = mocker.patch( - 'app.celery.scheduled_tasks.is_delivery_slow_for_provider' - ) - toggle_sms_mock = mocker.patch('app.celery.scheduled_tasks.dao_toggle_sms_provider') - - with set_config_values(notify_api, { - 'FUNCTIONAL_TEST_PROVIDER_SERVICE_ID': None, - 'FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID': None - }): - switch_current_sms_provider_on_slow_delivery() - - assert get_notifications_mock.called is False - assert toggle_sms_mock.called is False - - -def test_switch_providers_on_slow_delivery_runs_if_config_set( - notify_api, - mocker, - prepare_current_provider -): - get_notifications_mock = mocker.patch( - 'app.celery.scheduled_tasks.is_delivery_slow_for_provider', - return_value=[] - ) - - with set_config_values(notify_api, { - 'FUNCTIONAL_TEST_PROVIDER_SERVICE_ID': '7954469d-8c6d-43dc-b8f7-86be2d69f5f3', - 'FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID': '331a63e6-f1aa-4588-ad3f-96c268788ae7' - }): - switch_current_sms_provider_on_slow_delivery() - - assert get_notifications_mock.called is True - - -def test_switch_providers_triggers_on_slow_notification_delivery( - notify_api, - mocker, - prepare_current_provider, - sample_user -): - mocker.patch('app.provider_details.switch_providers.get_user_by_id', return_value=sample_user) - starting_provider = get_current_provider('sms') - - with set_config_values(notify_api, { - 'FUNCTIONAL_TEST_PROVIDER_SERVICE_ID': '7954469d-8c6d-43dc-b8f7-86be2d69f5f3', - 'FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID': '331a63e6-f1aa-4588-ad3f-96c268788ae7' - }): - _create_slow_delivery_notification(starting_provider.identifier) - _create_slow_delivery_notification(starting_provider.identifier) - switch_current_sms_provider_on_slow_delivery() - - new_provider = get_current_provider('sms') - assert new_provider.identifier != starting_provider.identifier - assert new_provider.priority < starting_provider.priority - - -def test_switch_providers_on_slow_delivery_does_not_switch_if_already_switched( - notify_api, - mocker, - prepare_current_provider, - sample_user -): - mocker.patch('app.provider_details.switch_providers.get_user_by_id', return_value=sample_user) - starting_provider = get_current_provider('sms') - - with set_config_values(notify_api, { - 'FUNCTIONAL_TEST_PROVIDER_SERVICE_ID': '7954469d-8c6d-43dc-b8f7-86be2d69f5f3', - 'FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID': '331a63e6-f1aa-4588-ad3f-96c268788ae7' - }): - _create_slow_delivery_notification() - _create_slow_delivery_notification() - - switch_current_sms_provider_on_slow_delivery() - switch_current_sms_provider_on_slow_delivery() - - new_provider = get_current_provider('sms') - assert new_provider.identifier != starting_provider.identifier - assert new_provider.priority < starting_provider.priority - - -def test_switch_providers_on_slow_delivery_does_not_switch_based_on_older_notifications( +def test_switch_providers_on_slow_delivery_switches_once_then_does_not_switch_if_already_switched( notify_api, mocker, prepare_current_provider, sample_user, - + sample_template ): - """ - Assume we have three slow delivery notifications for the current provider x. This triggers - a switch to provider y. If we experience some slow delivery notifications on this provider, - we switch back to provider x. - - Provider x had three slow deliveries initially, but we do not want to trigger another switch - based on these as they are old. We only want to look for slow notifications after the point at - which we switched back to provider x. - """ mocker.patch('app.provider_details.switch_providers.get_user_by_id', return_value=sample_user) starting_provider = get_current_provider('sms') - with set_config_values(notify_api, { - 'FUNCTIONAL_TEST_PROVIDER_SERVICE_ID': '7954469d-8c6d-43dc-b8f7-86be2d69f5f3', - 'FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID': '331a63e6-f1aa-4588-ad3f-96c268788ae7' - }): - # Provider x -> y - _create_slow_delivery_notification(starting_provider.identifier) - _create_slow_delivery_notification(starting_provider.identifier) - _create_slow_delivery_notification(starting_provider.identifier) - switch_current_sms_provider_on_slow_delivery() + _create_slow_delivery_notification(sample_template) + _create_slow_delivery_notification(sample_template) - current_provider = get_current_provider('sms') - assert current_provider.identifier != starting_provider.identifier + switch_current_sms_provider_on_slow_delivery() - # Provider y -> x - _create_slow_delivery_notification(current_provider.identifier) - _create_slow_delivery_notification(current_provider.identifier) - switch_current_sms_provider_on_slow_delivery() + new_provider = get_current_provider('sms') + _create_slow_delivery_notification(sample_template, new_provider.identifier) + _create_slow_delivery_notification(sample_template, new_provider.identifier) + switch_current_sms_provider_on_slow_delivery() - new_provider = get_current_provider('sms') - assert new_provider.identifier != current_provider.identifier + final_provider = get_current_provider('sms') - # Expect to stay on provider x - switch_current_sms_provider_on_slow_delivery() - current_provider = get_current_provider('sms') - assert starting_provider.identifier == current_provider.identifier + assert new_provider.identifier != starting_provider.identifier + assert new_provider.priority < starting_provider.priority + assert final_provider.identifier == new_provider.identifier @freeze_time("2017-05-01 14:00:00") @@ -505,244 +166,6 @@ def test_should_send_all_scheduled_notifications_to_deliver_queue(sample_templat assert not scheduled_notifications -def test_should_call_delete_inbound_sms_older_than_seven_days(notify_api, mocker): - mocker.patch('app.celery.scheduled_tasks.delete_inbound_sms_created_more_than_a_week_ago') - delete_inbound_sms_older_than_seven_days() - assert scheduled_tasks.delete_inbound_sms_created_more_than_a_week_ago.call_count == 1 - - -@freeze_time('2017-01-01 10:00:00') -def test_remove_csv_files_filters_by_type(mocker, sample_service): - mocker.patch('app.celery.scheduled_tasks.s3.remove_job_from_s3') - """ - Jobs older than seven days are deleted, but only two day's worth (two-day window) - """ - letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) - sms_template = create_template(service=sample_service, template_type=SMS_TYPE) - - eight_days_ago = datetime.utcnow() - timedelta(days=8) - - job_to_delete = create_job(template=letter_template, created_at=eight_days_ago) - create_job(template=sms_template, created_at=eight_days_ago) - - remove_csv_files(job_types=[LETTER_TYPE]) - - assert s3.remove_job_from_s3.call_args_list == [ - call(job_to_delete.service_id, job_to_delete.id), - ] - - -@freeze_time('2017-01-01 10:00:00') -def test_remove_dvla_transformed_files_removes_expected_files(mocker, sample_service): - mocker.patch('app.celery.scheduled_tasks.s3.remove_transformed_dvla_file') - - letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) - - job = partial(create_job, template=letter_template) - - seven_days_ago = datetime.utcnow() - timedelta(days=7) - just_under_seven_days = seven_days_ago + timedelta(seconds=1) - just_over_seven_days = seven_days_ago - timedelta(seconds=1) - eight_days_ago = seven_days_ago - timedelta(days=1) - nine_days_ago = eight_days_ago - timedelta(days=1) - just_under_nine_days = nine_days_ago + timedelta(seconds=1) - just_over_nine_days = nine_days_ago - timedelta(seconds=1) - - job(created_at=seven_days_ago) - job(created_at=just_under_seven_days) - job_to_delete_1 = job(created_at=just_over_seven_days) - job_to_delete_2 = job(created_at=eight_days_ago) - job_to_delete_3 = job(created_at=nine_days_ago) - job_to_delete_4 = job(created_at=just_under_nine_days) - job(created_at=just_over_nine_days) - - remove_transformed_dvla_files() - - s3.remove_transformed_dvla_file.assert_has_calls([ - call(job_to_delete_1.id), - call(job_to_delete_2.id), - call(job_to_delete_3.id), - call(job_to_delete_4.id), - ], any_order=True) - - -def test_remove_dvla_transformed_files_does_not_remove_files(mocker, sample_service): - mocker.patch('app.celery.scheduled_tasks.s3.remove_transformed_dvla_file') - - letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) - - job = partial(create_job, template=letter_template) - - yesterday = datetime.utcnow() - timedelta(days=1) - six_days_ago = datetime.utcnow() - timedelta(days=6) - seven_days_ago = six_days_ago - timedelta(days=1) - just_over_nine_days = seven_days_ago - timedelta(days=2, seconds=1) - - job(created_at=yesterday) - job(created_at=six_days_ago) - job(created_at=seven_days_ago) - job(created_at=just_over_nine_days) - - remove_transformed_dvla_files() - - s3.remove_transformed_dvla_file.assert_has_calls([]) - - -@freeze_time("2016-01-01 11:00:00") -def test_delete_dvla_response_files_older_than_seven_days_removes_old_files(notify_api, mocker): - AFTER_SEVEN_DAYS = datetime_in_past(days=8) - single_page_s3_objects = [{ - "Contents": [ - single_s3_object_stub('bar/foo1.txt', AFTER_SEVEN_DAYS), - single_s3_object_stub('bar/foo2.txt', AFTER_SEVEN_DAYS), - ] - }] - mocker.patch( - 'app.celery.scheduled_tasks.s3.get_s3_bucket_objects', return_value=single_page_s3_objects[0]["Contents"] - ) - remove_s3_mock = mocker.patch('app.celery.scheduled_tasks.s3.remove_s3_object') - - delete_dvla_response_files_older_than_seven_days() - - remove_s3_mock.assert_has_calls([ - call(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], single_page_s3_objects[0]["Contents"][0]["Key"]), - call(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], single_page_s3_objects[0]["Contents"][1]["Key"]) - ]) - - -@freeze_time("2016-01-01 11:00:00") -def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(notify_api, mocker): - START_DATE = datetime_in_past(days=9) - JUST_BEFORE_START_DATE = datetime_in_past(days=9, seconds=1) - END_DATE = datetime_in_past(days=7) - JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1) - - single_page_s3_objects = [{ - "Contents": [ - single_s3_object_stub('bar/foo1.txt', JUST_BEFORE_START_DATE), - single_s3_object_stub('bar/foo2.txt', START_DATE), - single_s3_object_stub('bar/foo3.txt', END_DATE), - single_s3_object_stub('bar/foo4.txt', JUST_AFTER_END_DATE), - ] - }] - mocker.patch( - 'app.celery.scheduled_tasks.s3.get_s3_bucket_objects', return_value=single_page_s3_objects[0]["Contents"] - ) - remove_s3_mock = mocker.patch('app.celery.scheduled_tasks.s3.remove_s3_object') - delete_dvla_response_files_older_than_seven_days() - - remove_s3_mock.assert_not_called() - - -@freeze_time("2018-01-17 17:00:00") -def test_alert_if_letter_notifications_still_sending(sample_letter_template, mocker): - two_days_ago = datetime(2018, 1, 15, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=two_days_ago) - - mock_create_ticket = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - raise_alert_if_letter_notifications_still_sending() - - mock_create_ticket.assert_called_once_with( - subject="[test] Letters still sending", - message="There are 1 letters in the 'sending' state from Monday 15 January", - ticket_type=ZendeskClient.TYPE_INCIDENT - ) - - -def test_alert_if_letter_notifications_still_sending_a_day_ago_no_alert(sample_letter_template, mocker): - today = datetime.utcnow() - one_day_ago = today - timedelta(days=1) - create_notification(template=sample_letter_template, status='sending', sent_at=one_day_ago) - - mock_create_ticket = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - raise_alert_if_letter_notifications_still_sending() - assert not mock_create_ticket.called - - -@freeze_time("2018-01-17 17:00:00") -def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_letter_template, mocker): - two_days_ago = datetime(2018, 1, 15, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=two_days_ago) - create_notification(template=sample_letter_template, status='delivered', sent_at=two_days_ago) - create_notification(template=sample_letter_template, status='failed', sent_at=two_days_ago) - - mock_create_ticket = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - raise_alert_if_letter_notifications_still_sending() - - mock_create_ticket.assert_called_once_with( - subject="[test] Letters still sending", - message="There are 1 letters in the 'sending' state from Monday 15 January", - ticket_type='incident' - ) - - -@freeze_time("2018-01-17 17:00:00") -def test_alert_if_letter_notifications_still_sending_alerts_for_older_than_offset(sample_letter_template, mocker): - three_days_ago = datetime(2018, 1, 14, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=three_days_ago) - - mock_create_ticket = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - raise_alert_if_letter_notifications_still_sending() - - mock_create_ticket.assert_called_once_with( - subject="[test] Letters still sending", - message="There are 1 letters in the 'sending' state from Monday 15 January", - ticket_type='incident' - ) - - -@freeze_time("2018-01-14 17:00:00") -def test_alert_if_letter_notifications_still_sending_does_nothing_on_the_weekend(sample_letter_template, mocker): - yesterday = datetime(2018, 1, 13, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) - - mock_create_ticket = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - raise_alert_if_letter_notifications_still_sending() - - assert not mock_create_ticket.called - - -@freeze_time("2018-01-15 17:00:00") -def test_monday_alert_if_letter_notifications_still_sending_reports_thursday_letters(sample_letter_template, mocker): - thursday = datetime(2018, 1, 11, 13, 30) - yesterday = datetime(2018, 1, 14, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=thursday) - create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) - - mock_create_ticket = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - raise_alert_if_letter_notifications_still_sending() - - mock_create_ticket.assert_called_once_with( - subject="[test] Letters still sending", - message="There are 1 letters in the 'sending' state from Thursday 11 January", - ticket_type='incident' - ) - - -@freeze_time("2018-01-16 17:00:00") -def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_letters(sample_letter_template, mocker): - friday = datetime(2018, 1, 12, 13, 30) - yesterday = datetime(2018, 1, 14, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=friday) - create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) - - mock_create_ticket = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - raise_alert_if_letter_notifications_still_sending() - - mock_create_ticket.assert_called_once_with( - subject="[test] Letters still sending", - message="There are 1 letters in the 'sending' state from Friday 12 January", - ticket_type='incident' - ) - - def test_check_job_status_task_raises_job_incomplete_error(mocker, sample_template): mock_celery = mocker.patch('app.celery.tasks.notify_celery.send_task') job = create_job(template=sample_template, notification_count=3, @@ -865,227 +288,6 @@ def test_check_job_status_task_sets_jobs_to_error(mocker, sample_template): assert job_2.job_status == JOB_STATUS_IN_PROGRESS -def test_daily_stats_template_usage_by_month(notify_db, notify_db_session): - notification_history = functools.partial( - create_notification_history, - notify_db, - notify_db_session, - status='delivered' - ) - - template_one = create_sample_template(notify_db, notify_db_session) - template_two = create_sample_template(notify_db, notify_db_session) - - notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one) - notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) - notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) - notification_history(created_at=datetime.now(), sample_template=template_two) - - daily_stats_template_usage_by_month() - - result = db.session.query( - StatsTemplateUsageByMonth - ).order_by( - StatsTemplateUsageByMonth.year, - StatsTemplateUsageByMonth.month - ).all() - - assert len(result) == 2 - - assert result[0].template_id == template_two.id - assert result[0].month == 4 - assert result[0].year == 2016 - assert result[0].count == 2 - - assert result[1].template_id == template_one.id - assert result[1].month == 10 - assert result[1].year == 2017 - assert result[1].count == 1 - - -def test_daily_stats_template_usage_by_month_no_data(): - daily_stats_template_usage_by_month() - - results = db.session.query(StatsTemplateUsageByMonth).all() - - assert len(results) == 0 - - -def test_daily_stats_template_usage_by_month_multiple_runs(notify_db, notify_db_session): - notification_history = functools.partial( - create_notification_history, - notify_db, - notify_db_session, - status='delivered' - ) - - template_one = create_sample_template(notify_db, notify_db_session) - template_two = create_sample_template(notify_db, notify_db_session) - - notification_history(created_at=datetime(2017, 11, 1), sample_template=template_one) - notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) - notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) - notification_history(created_at=datetime.now(), sample_template=template_two) - - daily_stats_template_usage_by_month() - - template_three = create_sample_template(notify_db, notify_db_session) - - notification_history(created_at=datetime(2017, 10, 1), sample_template=template_three) - notification_history(created_at=datetime(2017, 9, 1), sample_template=template_three) - notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) - notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) - notification_history(created_at=datetime.now(), sample_template=template_two) - - daily_stats_template_usage_by_month() - - result = db.session.query( - StatsTemplateUsageByMonth - ).order_by( - StatsTemplateUsageByMonth.year, - StatsTemplateUsageByMonth.month - ).all() - - assert len(result) == 4 - - assert result[0].template_id == template_two.id - assert result[0].month == 4 - assert result[0].year == 2016 - assert result[0].count == 4 - - assert result[1].template_id == template_three.id - assert result[1].month == 9 - assert result[1].year == 2017 - assert result[1].count == 1 - - assert result[2].template_id == template_three.id - assert result[2].month == 10 - assert result[2].year == 2017 - assert result[2].count == 1 - - assert result[3].template_id == template_one.id - assert result[3].month == 11 - assert result[3].year == 2017 - assert result[3].count == 1 - - -def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_counted(notify_db, notify_db_session): - notification_history = functools.partial( - create_notification_history, - notify_db, - notify_db_session, - status='delivered' - ) - - template_one = create_sample_template(notify_db, notify_db_session, template_name='1') - history = notification_history(created_at=datetime(2017, 2, 1), sample_template=template_one) - - NotificationHistory.query.filter( - NotificationHistory.id == history.id - ).update( - { - 'template_id': None - } - ) - - daily_stats_template_usage_by_month() - - result = db.session.query( - StatsTemplateUsageByMonth - ).all() - - assert len(result) == 0 - - notification_history(created_at=datetime(2017, 2, 1), sample_template=template_one) - - daily_stats_template_usage_by_month() - - result = db.session.query( - StatsTemplateUsageByMonth - ).order_by( - StatsTemplateUsageByMonth.year, - StatsTemplateUsageByMonth.month - ).all() - - assert len(result) == 1 - - -def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None): - if subfolder == '2018-01-11/zips_sent': - return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT'] - if subfolder == 'root/dispatch': - return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] - - -def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None): - if subfolder == '2018-01-11/zips_sent': - return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT', - 'NOTIFY.20180111175010.ZIP.TXT'] - if subfolder == 'root/dispatch': - return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] - - -@freeze_time('2018-01-11T23:00:00') -def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db): - mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match) - mock_get_file = mocker.patch("app.aws.s3.get_s3_file", - return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' - 'NOTIFY.20180111175008.ZIP|20180111175734') - - letter_raise_alert_if_no_ack_file_for_zip() - - yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # Datatime format on AWS - subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent' - assert mock_file_list.call_count == 2 - assert mock_file_list.call_args_list == [ - call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'), - call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', - suffix='.ACK.txt', last_modified=yesterday), - ] - assert mock_get_file.call_count == 1 - - -@freeze_time('2018-01-11T23:00:00') -def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db): - mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff) - mock_get_file = mocker.patch("app.aws.s3.get_s3_file", - return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' - 'NOTIFY.20180111175008.ZIP|20180111175734') - mock_zendesk = mocker.patch("app.celery.scheduled_tasks.zendesk_client.create_ticket") - - letter_raise_alert_if_no_ack_file_for_zip() - - assert mock_file_list.call_count == 2 - assert mock_get_file.call_count == 1 - - message = "Letter ack file does not contain all zip files sent. " \ - "Missing ack for zip files: {}, " \ - "pdf bucket: {}, subfolder: {}, " \ - "ack bucket: {}".format(str(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']), - current_app.config['LETTERS_PDF_BUCKET_NAME'], - datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', - current_app.config['DVLA_RESPONSE_BUCKET_NAME']) - - mock_zendesk.assert_called_once_with( - subject="Letter acknowledge error", - message=message, - ticket_type='incident' - ) - - -@freeze_time('2018-01-11T23:00:00') -def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db): - mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None) - mock_get_file = mocker.patch("app.aws.s3.get_s3_file", - return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' - 'NOTIFY.20180111175008.ZIP|20180111175734') - - letter_raise_alert_if_no_ack_file_for_zip() - - assert mock_file_list.call_count == 2 - assert mock_get_file.call_count == 0 - - def test_replay_created_notifications(notify_db_session, sample_service, mocker): email_delivery_queue = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') sms_delivery_queue = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') @@ -1114,3 +316,21 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker) queue='send-email-tasks') sms_delivery_queue.assert_called_once_with([str(old_sms.id)], queue="send-sms-tasks") + + +def test_check_job_status_task_does_not_raise_error(sample_template): + create_job( + template=sample_template, + notification_count=3, + created_at=datetime.utcnow() - timedelta(hours=2), + scheduled_for=datetime.utcnow() - timedelta(minutes=31), + processing_started=datetime.utcnow() - timedelta(minutes=31), + job_status=JOB_STATUS_FINISHED) + create_job( + template=sample_template, + notification_count=3, + created_at=datetime.utcnow() - timedelta(minutes=31), + processing_started=datetime.utcnow() - timedelta(minutes=31), + job_status=JOB_STATUS_FINISHED) + + check_job_status() diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 84ad84cab..13c6f9b4f 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -15,7 +15,6 @@ from notifications_utils.columns import Row from app import (encryption, DATETIME_FORMAT) from app.celery import provider_tasks from app.celery import tasks -from app.celery.scheduled_tasks import check_job_status from app.celery.tasks import ( process_job, process_row, @@ -1396,24 +1395,6 @@ def test_send_inbound_sms_to_service_does_not_retries_if_request_returns_404(not mocked.call_count == 0 -def test_check_job_status_task_does_not_raise_error(sample_template): - create_job( - template=sample_template, - notification_count=3, - created_at=datetime.utcnow() - timedelta(hours=2), - scheduled_for=datetime.utcnow() - timedelta(minutes=31), - processing_started=datetime.utcnow() - timedelta(minutes=31), - job_status=JOB_STATUS_FINISHED) - create_job( - template=sample_template, - notification_count=3, - created_at=datetime.utcnow() - timedelta(minutes=31), - processing_started=datetime.utcnow() - timedelta(minutes=31), - job_status=JOB_STATUS_FINISHED) - - check_job_status() - - def test_process_incomplete_job_sms(mocker, sample_template): mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('multiple_sms')) diff --git a/tests/app/commands/test_populate_redis.py b/tests/app/commands/test_populate_redis.py deleted file mode 100644 index 25001642a..000000000 --- a/tests/app/commands/test_populate_redis.py +++ /dev/null @@ -1,73 +0,0 @@ -from datetime import datetime - -from freezegun import freeze_time -import pytest - -from app.commands import populate_redis_template_usage - -from tests.conftest import set_config -from tests.app.db import create_notification, create_template, create_service - - -def test_populate_redis_template_usage_does_nothing_if_redis_disabled(mocker, notify_api, sample_service): - mock_redis = mocker.patch('app.commands.redis_store') - with set_config(notify_api, 'REDIS_ENABLED', False): - with pytest.raises(SystemExit) as exit_signal: - populate_redis_template_usage.callback.__wrapped__(sample_service.id, datetime.utcnow()) - - assert mock_redis.mock_calls == [] - # sys.exit with nonzero exit code - assert exit_signal.value.code != 0 - - -def test_populate_redis_template_usage_does_nothing_if_no_data(mocker, notify_api, sample_service): - mock_redis = mocker.patch('app.commands.redis_store') - with set_config(notify_api, 'REDIS_ENABLED', True): - populate_redis_template_usage.callback.__wrapped__(sample_service.id, datetime.utcnow()) - - assert mock_redis.mock_calls == [] - - -@freeze_time('2017-06-12') -def test_populate_redis_template_usage_only_populates_for_today(mocker, notify_api, sample_template): - mock_redis = mocker.patch('app.commands.redis_store') - # created at in utc - create_notification(sample_template, created_at=datetime(2017, 6, 9, 23, 0, 0)) - create_notification(sample_template, created_at=datetime(2017, 6, 9, 23, 0, 0)) - create_notification(sample_template, created_at=datetime(2017, 6, 10, 0, 0, 0)) - create_notification(sample_template, created_at=datetime(2017, 6, 10, 23, 0, 0)) # actually on 11th BST - - with set_config(notify_api, 'REDIS_ENABLED', True): - populate_redis_template_usage.callback.__wrapped__(sample_template.service_id, datetime(2017, 6, 10)) - - mock_redis.set_hash_and_expire.assert_called_once_with( - 'service-{}-template-usage-2017-06-10'.format(sample_template.service_id), - {str(sample_template.id): 3}, - notify_api.config['EXPIRE_CACHE_EIGHT_DAYS'], - raise_exception=True - ) - - -@freeze_time('2017-06-12') -def test_populate_redis_template_usage_only_populates_for_given_service(mocker, notify_api, notify_db_session): - mock_redis = mocker.patch('app.commands.redis_store') - # created at in utc - s1 = create_service(service_name='a') - s2 = create_service(service_name='b') - t1 = create_template(s1) - t2 = create_template(s2) - - create_notification(t1, created_at=datetime(2017, 6, 10)) - create_notification(t1, created_at=datetime(2017, 6, 10)) - - create_notification(t2, created_at=datetime(2017, 6, 10)) - - with set_config(notify_api, 'REDIS_ENABLED', True): - populate_redis_template_usage.callback.__wrapped__(s1.id, datetime(2017, 6, 10)) - - mock_redis.set_hash_and_expire.assert_called_once_with( - 'service-{}-template-usage-2017-06-10'.format(s1.id), - {str(t1.id): 2}, - notify_api.config['EXPIRE_CACHE_EIGHT_DAYS'], - raise_exception=True - ) diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 04d77a60e..5b6aa2234 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -76,20 +76,26 @@ def service_factory(notify_db, notify_db_session): user = create_user() if not email_from: email_from = service_name - service = sample_service(notify_db, notify_db_session, service_name, user, email_from=email_from) + + service = create_service( + email_from=email_from, + service_name=service_name, + service_permissions=None, + user=user, + check_if_service_exists=True, + ) if template_type == 'email': - sample_template( - notify_db, - notify_db_session, + create_template( + service, + template_name="Template Name", template_type=template_type, - subject_line=service.email_from, - service=service + subject=service.email_from, ) else: - sample_template( - notify_db, - notify_db_session, - service=service + create_template( + service, + template_name="Template Name", + template_type='sms', ) return service @@ -158,6 +164,7 @@ def sample_service( email_from=None, permissions=None, research_mode=None, + postage="second", ): if user is None: user = create_user() @@ -170,6 +177,7 @@ def sample_service( 'restricted': restricted, 'email_from': email_from, 'created_by': user, + "postage": postage, } service = Service.query.filter_by(name=service_name).first() if not service: @@ -193,7 +201,8 @@ def sample_service( def _sample_service_full_permissions(notify_db_session): service = create_service( service_name="sample service full permissions", - service_permissions=set(SERVICE_PERMISSION_TYPES) + service_permissions=set(SERVICE_PERMISSION_TYPES), + check_if_service_exists=True ) create_inbound_number('12345', service_id=service.id) return service @@ -226,7 +235,7 @@ def sample_template( if service is None: service = Service.query.filter_by(name='Sample service').first() if not service: - service = create_service(service_permissions=permissions) + service = create_service(service_permissions=permissions, check_if_service_exists=True) if created_by is None: created_by = create_user() @@ -288,7 +297,7 @@ def sample_email_template( if user is None: user = create_user() if service is None: - service = sample_service(notify_db, notify_db_session, permissions=permissions) + service = create_service(user=user, service_permissions=permissions, check_if_service_exists=True) data = { 'name': template_name, 'template_type': template_type, @@ -343,7 +352,7 @@ def sample_api_key(notify_db, key_type=KEY_TYPE_NORMAL, name=None): if service is None: - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) data = {'service': service, 'name': name or uuid.uuid4(), 'created_by': service.created_by, 'key_type': key_type} api_key = ApiKey(**data) save_model_api_key(api_key) @@ -371,13 +380,13 @@ def sample_job( job_status='pending', scheduled_for=None, processing_started=None, - original_file_name='some.csv' + original_file_name='some.csv', + archived=False ): if service is None: - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) if template is None: - template = sample_template(notify_db, notify_db_session, - service=service) + template = create_template(service=service) data = { 'id': uuid.uuid4(), 'service_id': service.id, @@ -390,7 +399,8 @@ def sample_job( 'created_by': service.created_by, 'job_status': job_status, 'scheduled_for': scheduled_for, - 'processing_started': processing_started + 'processing_started': processing_started, + 'archived': archived } job = Job(**data) dao_create_job(job) @@ -433,7 +443,7 @@ def sample_email_job(notify_db, service=None, template=None): if service is None: - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) if template is None: template = sample_email_template( notify_db, @@ -541,9 +551,9 @@ def sample_notification( if created_at is None: created_at = datetime.utcnow() if service is None: - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) if template is None: - template = sample_template(notify_db, notify_db_session, service=service) + template = create_template(service=service) if job is None and api_key is None: # we didn't specify in test - lets create it @@ -630,7 +640,7 @@ def sample_notification_with_api_key(notify_db, notify_db_session): @pytest.fixture(scope='function') def sample_email_notification(notify_db, notify_db_session): created_at = datetime.utcnow() - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) template = sample_email_template(notify_db, notify_db_session, service=service) job = sample_job(notify_db, notify_db_session, service=service, template=template) @@ -732,7 +742,7 @@ def sample_invited_user(notify_db, to_email_address=None): if service is None: - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) if to_email_address is None: to_email_address = 'invited_user@digital.gov.uk' @@ -772,7 +782,7 @@ def sample_permission(notify_db, 'permission': permission } if service is None: - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) if service: data['service'] = service p_model = Permission.query.filter_by( @@ -793,7 +803,7 @@ def sample_user_service_permission( if user is None: user = create_user() if service is None: - service = sample_service(notify_db, notify_db_session, user=user) + service = create_service(user=user, check_if_service_exists=True) data = { 'user': user, 'service': service, @@ -1027,7 +1037,7 @@ def notify_service(notify_db, notify_db_session): @pytest.fixture(scope='function') def sample_service_whitelist(notify_db, notify_db_session, service=None, email_address=None, mobile_number=None): if service is None: - service = sample_service(notify_db, notify_db_session) + service = create_service(check_if_service_exists=True) if email_address: whitelisted_user = ServiceWhitelist.from_string(service.id, EMAIL_TYPE, email_address) @@ -1052,7 +1062,7 @@ def sample_provider_rate(notify_db, notify_db_session, valid_from=None, rate=Non @pytest.fixture def sample_inbound_numbers(notify_db, notify_db_session, sample_service): - service = create_service(service_name='sample service 2') + service = create_service(service_name='sample service 2', check_if_service_exists=True) inbound_numbers = list() inbound_numbers.append(create_inbound_number(number='1', provider='mmg')) inbound_numbers.append(create_inbound_number(number='2', provider='mmg', active=False, service_id=service.id)) @@ -1099,7 +1109,9 @@ def restore_provider_details(notify_db, notify_db_session): @pytest.fixture def admin_request(client): + class AdminRequest: + app = client.application @staticmethod def get(endpoint, _expected_status=200, **endpoint_kwargs): diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index ad3a9aa41..90dc73035 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -1,10 +1,12 @@ import uuid -from datetime import datetime, timedelta, date +from datetime import datetime, timedelta from functools import partial import pytest from freezegun import freeze_time from sqlalchemy.exc import SQLAlchemyError, IntegrityError +from sqlalchemy.orm.exc import NoResultFound + from app.dao.notifications_dao import ( dao_create_notification, @@ -14,7 +16,6 @@ from app.dao.notifications_dao import ( dao_get_last_template_usage, dao_get_notifications_by_to_field, dao_get_scheduled_notifications, - dao_get_template_usage, dao_timeout_notifications, dao_update_notification, dao_update_notifications_by_reference, @@ -33,7 +34,6 @@ from app.dao.notifications_dao import ( dao_get_notifications_by_references, dao_get_notification_history_by_reference, notifications_not_yet_sent, - fetch_aggregate_stats_by_date_range_for_all_services, ) from app.dao.services_dao import dao_update_service from app.models import ( @@ -43,6 +43,8 @@ from app.models import ( ScheduledNotification, NOTIFICATION_STATUS_TYPES, NOTIFICATION_STATUS_TYPES_FAILED, + NOTIFICATION_TEMPORARY_FAILURE, + NOTIFICATION_SENDING, NOTIFICATION_SENT, NOTIFICATION_DELIVERED, KEY_TYPE_NORMAL, @@ -67,7 +69,6 @@ from tests.app.db import ( def test_should_have_decorated_notifications_dao_functions(): assert dao_get_last_template_usage.__wrapped__.__name__ == 'dao_get_last_template_usage' # noqa - assert dao_get_template_usage.__wrapped__.__name__ == 'dao_get_template_usage' # noqa assert dao_create_notification.__wrapped__.__name__ == 'dao_create_notification' # noqa assert update_notification_status_by_id.__wrapped__.__name__ == 'update_notification_status_by_id' # noqa assert dao_update_notification.__wrapped__.__name__ == 'dao_update_notification' # noqa @@ -141,6 +142,14 @@ def test_should_update_status_by_id_if_created(notify_db, notify_db_session): assert updated.status == 'failed' +def test_should_update_status_by_id_if_pending_virus_check(notify_db, notify_db_session): + notification = sample_notification(notify_db, notify_db_session, status='pending-virus-check') + assert Notification.query.get(notification.id).status == 'pending-virus-check' + updated = update_notification_status_by_id(notification.id, 'cancelled') + assert Notification.query.get(notification.id).status == 'cancelled' + assert updated.status == 'cancelled' + + def test_should_update_status_by_id_and_set_sent_by(notify_db, notify_db_session): notification = sample_notification(notify_db, notify_db_session, status='sending') @@ -512,7 +521,7 @@ def test_save_notification_with_no_job(sample_template, mmg_provider): assert notification_from_db.status == 'created' -def test_get_notification_by_id(notify_db, notify_db_session, sample_template): +def test_get_notification_with_personalisation_by_id(notify_db, notify_db_session, sample_template): notification = sample_notification(notify_db=notify_db, notify_db_session=notify_db_session, template=sample_template, scheduled_for='2017-05-05 14:15', @@ -526,6 +535,25 @@ def test_get_notification_by_id(notify_db, notify_db_session, sample_template): assert notification_from_db.scheduled_notification.scheduled_for == datetime(2017, 5, 5, 14, 15) +def test_get_notification_by_id_when_notification_exists(sample_notification): + notification_from_db = get_notification_by_id(sample_notification.id) + + assert sample_notification == notification_from_db + + +def test_get_notification_by_id_when_notification_does_not_exist(notify_db_session, fake_uuid): + notification_from_db = get_notification_by_id(fake_uuid) + + assert notification_from_db is None + + +def test_get_notification_by_id_when_notification_exists_for_different_service(sample_notification): + another_service = create_service(service_name='Another service') + + with pytest.raises(NoResultFound): + get_notification_by_id(sample_notification.id, another_service.id, _raise=True) + + def test_get_notifications_by_reference(sample_template): client_reference = 'some-client-ref' assert len(Notification.query.all()) == 0 @@ -879,6 +907,16 @@ def test_should_return_notifications_including_one_offs_by_default(sample_user, assert len(include_one_offs_by_default) == 2 +def test_should_not_count_pages_when_given_a_flag(sample_user, sample_template): + create_notification(sample_template) + notification = create_notification(sample_template) + + pagination = get_notifications_for_service(sample_template.service_id, count_pages=False, page_size=1) + assert len(pagination.items) == 1 + assert pagination.total is None + assert pagination.items[0].id == notification.id + + def test_get_notifications_created_by_api_or_csv_are_returned_correctly_excluding_test_key_notifications( notify_db, notify_db_session, @@ -1177,131 +1215,85 @@ def test_get_total_sent_notifications_for_email_excludes_sms_counts( assert total_count == 2 -@freeze_time("2016-01-10 12:00:00.000000") -def test_slow_provider_delivery_returns_for_sent_notifications( - sample_template +@pytest.mark.parametrize( + "normal_sending,slow_sending,normal_delivered,slow_delivered,threshold,expected_result", + [ + (0, 0, 0, 0, 0.1, False), + (1, 0, 0, 0, 0.1, False), + (1, 1, 0, 0, 0.1, True), + (0, 0, 1, 1, 0.1, True), + (1, 1, 1, 1, 0.5, True), + (1, 1, 1, 1, 0.6, False), + (45, 5, 45, 5, 0.1, True), + ] +) +@freeze_time("2018-12-04 12:00:00.000000") +def test_is_delivery_slow_for_provider( + notify_db_session, + sample_template, + normal_sending, + slow_sending, + normal_delivered, + slow_delivered, + threshold, + expected_result ): - now = datetime.utcnow() - one_minute_from_now = now + timedelta(minutes=1) - five_minutes_from_now = now + timedelta(minutes=5) - - notification_five_minutes_to_deliver = partial( + normal_notification = partial( create_notification, template=sample_template, - status='delivered', sent_by='mmg', - updated_at=five_minutes_from_now + sent_at=datetime.now(), + updated_at=datetime.now() ) - notification_five_minutes_to_deliver(sent_at=now) - notification_five_minutes_to_deliver(sent_at=one_minute_from_now) - notification_five_minutes_to_deliver(sent_at=one_minute_from_now) - - slow_delivery = is_delivery_slow_for_provider( - sent_at=one_minute_from_now, - provider='mmg', - threshold=2, - delivery_time=timedelta(minutes=3), - service_id=sample_template.service.id, - template_id=sample_template.id - ) - - assert slow_delivery - - -@freeze_time("2016-01-10 12:00:00.000000") -def test_slow_provider_delivery_observes_threshold( - sample_template -): - now = datetime.utcnow() - five_minutes_from_now = now + timedelta(minutes=5) - - notification_five_minutes_to_deliver = partial( + slow_notification = partial( create_notification, template=sample_template, - status='delivered', - sent_at=now, sent_by='mmg', - updated_at=five_minutes_from_now + sent_at=datetime.now() - timedelta(minutes=5), + updated_at=datetime.now() ) - notification_five_minutes_to_deliver() - notification_five_minutes_to_deliver() + for _ in range(normal_sending): + normal_notification(status='sending') + for _ in range(slow_sending): + slow_notification(status='sending') + for _ in range(normal_delivered): + normal_notification(status='delivered') + for _ in range(slow_delivered): + slow_notification(status='delivered') - slow_delivery = is_delivery_slow_for_provider( - sent_at=now, - provider='mmg', - threshold=3, - delivery_time=timedelta(minutes=5), - service_id=sample_template.service.id, - template_id=sample_template.id - ) - - assert not slow_delivery + assert is_delivery_slow_for_provider(datetime.utcnow(), "mmg", threshold, timedelta(minutes=4)) is expected_result -@freeze_time("2016-01-10 12:00:00.000000") -def test_slow_provider_delivery_returns_for_delivered_notifications_only( - sample_template +@pytest.mark.parametrize("options,expected_result", [ + ({"status": NOTIFICATION_TEMPORARY_FAILURE, "sent_by": "mmg"}, False), + ({"status": NOTIFICATION_DELIVERED, "sent_by": "firetext"}, False), + ({"status": NOTIFICATION_DELIVERED, "sent_by": "mmg"}, True), + ({"status": NOTIFICATION_DELIVERED, "sent_by": "mmg", "sent_at": None}, False), + ({"status": NOTIFICATION_DELIVERED, "sent_by": "mmg", "key_type": KEY_TYPE_TEST}, False), + ({"status": NOTIFICATION_SENDING, "sent_by": "firetext"}, False), + ({"status": NOTIFICATION_SENDING, "sent_by": "mmg"}, True), + ({"status": NOTIFICATION_SENDING, "sent_by": "mmg", "sent_at": None}, False), + ({"status": NOTIFICATION_SENDING, "sent_by": "mmg", "key_type": KEY_TYPE_TEST}, False), +]) +@freeze_time("2018-12-04 12:00:00.000000") +def test_delivery_is_delivery_slow_for_provider_filters_out_notifications_it_should_not_count( + notify_db_session, + sample_template, + options, + expected_result ): - now = datetime.utcnow() - five_minutes_from_now = now + timedelta(minutes=5) - - notification_five_minutes_to_deliver = partial( - create_notification, - template=sample_template, - sent_at=now, - sent_by='firetext', - created_at=now, - updated_at=five_minutes_from_now + create_notification_with = { + "template": sample_template, + "sent_at": datetime.now() - timedelta(minutes=5), + "updated_at": datetime.now(), + } + create_notification_with.update(options) + create_notification( + **create_notification_with ) - - notification_five_minutes_to_deliver(status='sending') - notification_five_minutes_to_deliver(status='delivered') - notification_five_minutes_to_deliver(status='delivered') - - slow_delivery = is_delivery_slow_for_provider( - sent_at=now, - provider='firetext', - threshold=2, - delivery_time=timedelta(minutes=5), - service_id=sample_template.service.id, - template_id=sample_template.id - ) - - assert slow_delivery - - -@freeze_time("2016-01-10 12:00:00.000000") -def test_slow_provider_delivery_does_not_return_for_standard_delivery_time( - sample_template -): - now = datetime.utcnow() - five_minutes_from_now = now + timedelta(minutes=5) - - notification = partial( - create_notification, - template=sample_template, - created_at=now, - sent_at=now, - sent_by='mmg', - status='delivered' - ) - - notification(updated_at=five_minutes_from_now - timedelta(seconds=1)) - notification(updated_at=five_minutes_from_now - timedelta(seconds=1)) - notification(updated_at=five_minutes_from_now) - - slow_delivery = is_delivery_slow_for_provider( - sent_at=now, - provider='mmg', - threshold=2, - delivery_time=timedelta(minutes=5), - service_id=sample_template.service.id, - template_id=sample_template.id - ) - - assert not slow_delivery + assert is_delivery_slow_for_provider(datetime.utcnow(), "mmg", 0.1, timedelta(minutes=4)) is expected_result def test_dao_get_notifications_by_to_field(sample_template): @@ -1802,51 +1794,3 @@ def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_ results = notifications_not_yet_sent(older_than, notification_type) assert len(results) == 0 - - -def test_fetch_aggregate_stats_by_date_range_for_all_services_returns_empty_list_when_no_stats(notify_db_session): - start_date = date(2018, 1, 1) - end_date = date(2018, 1, 5) - - result = fetch_aggregate_stats_by_date_range_for_all_services(start_date, end_date) - assert result == [] - - -@freeze_time('2018-01-08') -def test_fetch_aggregate_stats_by_date_range_for_all_services_groups_stats( - sample_template, - sample_email_template, - sample_letter_template, -): - today = datetime.now().date() - - for i in range(3): - create_notification(template=sample_email_template, status='permanent-failure', - created_at=today) - - create_notification(template=sample_email_template, status='sent', created_at=today) - create_notification(template=sample_template, status='sent', created_at=today) - create_notification(template=sample_template, status='sent', created_at=today, - key_type=KEY_TYPE_TEAM) - create_notification(template=sample_letter_template, status='virus-scan-failed', - created_at=today) - - result = fetch_aggregate_stats_by_date_range_for_all_services(today, today) - - assert len(result) == 5 - assert ('email', 'permanent-failure', 'normal', 3) in result - assert ('email', 'sent', 'normal', 1) in result - assert ('sms', 'sent', 'normal', 1) in result - assert ('sms', 'sent', 'team', 1) in result - assert ('letter', 'virus-scan-failed', 'normal', 1) in result - - -def test_fetch_aggregate_stats_by_date_range_for_all_services_uses_bst_date(sample_template): - query_day = datetime(2018, 6, 5).date() - create_notification(sample_template, status='sent', created_at=datetime(2018, 6, 4, 23, 59)) - create_notification(sample_template, status='created', created_at=datetime(2018, 6, 5, 23, 00)) - - result = fetch_aggregate_stats_by_date_range_for_all_services(query_day, query_day) - - assert len(result) == 1 - assert result[0].status == 'sent' diff --git a/tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py b/tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py index 1479921bd..4b3d8c9e8 100644 --- a/tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py +++ b/tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py @@ -182,10 +182,24 @@ def test_delete_notifications_does_try_to_delete_from_s3_when_letter_has_not_bee create_notification(template=letter_template, status='sending', reference='LETTER_REF') - delete_notifications_created_more_than_a_week_ago_by_type('email') + delete_notifications_created_more_than_a_week_ago_by_type('email', qry_limit=1) mock_get_s3.assert_not_called() +def test_delete_notifications_calls_subquery( + notify_db_session, mocker +): + service = create_service() + sms_template = create_template(service=service) + create_notification(template=sms_template, created_at=datetime.now() - timedelta(days=8)) + create_notification(template=sms_template, created_at=datetime.now() - timedelta(days=8)) + create_notification(template=sms_template, created_at=datetime.now() - timedelta(days=8)) + + assert Notification.query.count() == 3 + delete_notifications_created_more_than_a_week_ago_by_type('sms', qry_limit=1) + assert Notification.query.count() == 0 + + def _create_templates(sample_service): email_template = create_template(service=sample_service, template_type='email') sms_template = create_template(service=sample_service) diff --git a/tests/app/dao/notification_dao/test_notification_dao_template_usage.py b/tests/app/dao/notification_dao/test_notification_dao_template_usage.py index 76800694f..4006fd9c2 100644 --- a/tests/app/dao/notification_dao/test_notification_dao_template_usage.py +++ b/tests/app/dao/notification_dao/test_notification_dao_template_usage.py @@ -1,28 +1,12 @@ -import uuid -from datetime import datetime, timedelta, date - +from datetime import datetime, timedelta import pytest -from freezegun import freeze_time - -from app.dao.notifications_dao import ( - dao_get_last_template_usage, - dao_get_template_usage -) -from app.models import ( - KEY_TYPE_NORMAL, - KEY_TYPE_TEST, - KEY_TYPE_TEAM -) -from tests.app.db import ( - create_notification, - create_service, - create_template -) +from app.dao.notifications_dao import dao_get_last_template_usage +from tests.app.db import create_notification, create_template def test_last_template_usage_should_get_right_data(sample_notification): results = dao_get_last_template_usage(sample_notification.template_id, 'sms', sample_notification.service_id) - assert results.template.name == 'Template Name' + assert results.template.name == 'sms Template Name' assert results.template.template_type == 'sms' assert results.created_at == sample_notification.created_at assert results.template_id == sample_notification.template_id @@ -70,117 +54,3 @@ def test_last_template_usage_should_be_able_to_get_no_template_usage_history_if_ sample_template): results = dao_get_last_template_usage(sample_template.id, 'sms', sample_template.service_id) assert not results - - -@freeze_time('2018-01-01') -def test_should_by_able_to_get_template_count(sample_template, sample_email_template): - create_notification(sample_template) - create_notification(sample_template) - create_notification(sample_template) - create_notification(sample_email_template) - create_notification(sample_email_template) - - results = dao_get_template_usage(sample_template.service_id, date.today()) - assert results[0].name == sample_email_template.name - assert results[0].template_type == sample_email_template.template_type - assert results[0].count == 2 - - assert results[1].name == sample_template.name - assert results[1].template_type == sample_template.template_type - assert results[1].count == 3 - - -@freeze_time('2018-01-01') -def test_template_usage_should_ignore_test_keys( - sample_team_api_key, - sample_test_api_key, - sample_api_key, - sample_template -): - - create_notification(sample_template, api_key=sample_api_key, key_type=KEY_TYPE_NORMAL) - create_notification(sample_template, api_key=sample_team_api_key, key_type=KEY_TYPE_TEAM) - create_notification(sample_template, api_key=sample_test_api_key, key_type=KEY_TYPE_TEST) - create_notification(sample_template) - - results = dao_get_template_usage(sample_template.service_id, date.today()) - assert results[0].name == sample_template.name - assert results[0].template_type == sample_template.template_type - assert results[0].count == 3 - - -def test_template_usage_should_filter_by_service(notify_db_session): - service_1 = create_service(service_name='test1') - service_2 = create_service(service_name='test2') - service_3 = create_service(service_name='test3') - - template_1 = create_template(service_1) - template_2 = create_template(service_2) # noqa - template_3a = create_template(service_3, template_name='a') - template_3b = create_template(service_3, template_name='b') # noqa - - # two for service_1, one for service_3 - create_notification(template_1) - create_notification(template_1) - - create_notification(template_3a) - - res1 = dao_get_template_usage(service_1.id, date.today()) - res2 = dao_get_template_usage(service_2.id, date.today()) - res3 = dao_get_template_usage(service_3.id, date.today()) - - assert len(res1) == 1 - assert res1[0].count == 2 - - assert len(res2) == 1 - assert res2[0].count == 0 - - assert len(res3) == 2 - assert res3[0].count == 1 - assert res3[1].count == 0 - - -def test_template_usage_should_by_able_to_get_zero_count_from_notifications_history_if_no_rows(sample_service): - results = dao_get_template_usage(sample_service.id, date.today()) - assert len(results) == 0 - - -def test_template_usage_should_by_able_to_get_zero_count_from_notifications_history_if_no_service(): - results = dao_get_template_usage(str(uuid.uuid4()), date.today()) - assert len(results) == 0 - - -def test_template_usage_should_by_able_to_get_template_count_for_specific_day(sample_template): - # too early - create_notification(sample_template, created_at=datetime(2017, 6, 7, 22, 59, 0)) - # just right - create_notification(sample_template, created_at=datetime(2017, 6, 7, 23, 0, 0)) - create_notification(sample_template, created_at=datetime(2017, 6, 7, 23, 0, 0)) - create_notification(sample_template, created_at=datetime(2017, 6, 8, 22, 59, 0)) - create_notification(sample_template, created_at=datetime(2017, 6, 8, 22, 59, 0)) - create_notification(sample_template, created_at=datetime(2017, 6, 8, 22, 59, 0)) - # too late - create_notification(sample_template, created_at=datetime(2017, 6, 8, 23, 0, 0)) - - results = dao_get_template_usage(sample_template.service_id, day=date(2017, 6, 8)) - - assert len(results) == 1 - assert results[0].count == 5 - - -def test_template_usage_should_by_able_to_get_template_count_for_specific_timezone_boundary(sample_template): - # too early - create_notification(sample_template, created_at=datetime(2018, 3, 24, 23, 59, 0)) - # just right - create_notification(sample_template, created_at=datetime(2018, 3, 25, 0, 0, 0)) - create_notification(sample_template, created_at=datetime(2018, 3, 25, 0, 0, 0)) - create_notification(sample_template, created_at=datetime(2018, 3, 25, 22, 59, 0)) - create_notification(sample_template, created_at=datetime(2018, 3, 25, 22, 59, 0)) - create_notification(sample_template, created_at=datetime(2018, 3, 25, 22, 59, 0)) - # too late - create_notification(sample_template, created_at=datetime(2018, 3, 25, 23, 0, 0)) - - results = dao_get_template_usage(sample_template.service_id, day=date(2018, 3, 25)) - - assert len(results) == 1 - assert results[0].count == 5 diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index 5d0296b15..0b727a726 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -1,16 +1,22 @@ from datetime import timedelta, datetime, date from uuid import UUID +import pytest +import mock + from app.dao.fact_notification_status_dao import ( update_fact_notification_status, fetch_notification_status_for_day, fetch_notification_status_for_service_by_month, fetch_notification_status_for_service_for_day, - fetch_notification_status_for_service_for_today_and_7_previous_days + fetch_notification_status_for_service_for_today_and_7_previous_days, + fetch_notification_status_totals_for_all_services, + fetch_notification_statuses_for_job, + fetch_stats_for_all_services_by_date_range, fetch_monthly_template_usage_for_service ) -from app.models import FactNotificationStatus, KEY_TYPE_TEST, KEY_TYPE_TEAM, EMAIL_TYPE, SMS_TYPE +from app.models import FactNotificationStatus, KEY_TYPE_TEST, KEY_TYPE_TEAM, EMAIL_TYPE, SMS_TYPE, LETTER_TYPE from freezegun import freeze_time -from tests.app.db import create_notification, create_service, create_template, create_ft_notification_status +from tests.app.db import create_notification, create_service, create_template, create_ft_notification_status, create_job def test_update_fact_notification_status(notify_db_session): @@ -183,6 +189,7 @@ def test_fetch_notification_status_for_service_for_day(notify_db_session): def test_fetch_notification_status_for_service_for_today_and_7_previous_days(notify_db_session): service_1 = create_service(service_name='service_1') sms_template = create_template(service=service_1, template_type=SMS_TYPE) + sms_template_2 = create_template(service=service_1, template_type=SMS_TYPE) email_template = create_template(service=service_1, template_type=EMAIL_TYPE) create_ft_notification_status(date(2018, 10, 29), 'sms', service_1, count=10) @@ -192,6 +199,7 @@ def test_fetch_notification_status_for_service_for_today_and_7_previous_days(not create_ft_notification_status(date(2018, 10, 26), 'letter', service_1, count=5) create_notification(sms_template, created_at=datetime(2018, 10, 31, 11, 0, 0)) + create_notification(sms_template_2, created_at=datetime(2018, 10, 31, 11, 0, 0)) create_notification(sms_template, created_at=datetime(2018, 10, 31, 12, 0, 0), status='delivered') create_notification(email_template, created_at=datetime(2018, 10, 31, 13, 0, 0), status='delivered') @@ -215,8 +223,306 @@ def test_fetch_notification_status_for_service_for_today_and_7_previous_days(not assert results[2].notification_type == 'sms' assert results[2].status == 'created' - assert results[2].count == 2 + assert results[2].count == 3 assert results[3].notification_type == 'sms' assert results[3].status == 'delivered' assert results[3].count == 19 + + +@freeze_time('2018-10-31T18:00:00') +def test_fetch_notification_status_by_template_for_service_for_today_and_7_previous_days(notify_db_session): + service_1 = create_service(service_name='service_1') + sms_template = create_template(template_name='sms Template 1', service=service_1, template_type=SMS_TYPE) + sms_template_2 = create_template(template_name='sms Template 2', service=service_1, template_type=SMS_TYPE) + email_template = create_template(service=service_1, template_type=EMAIL_TYPE) + + # create unused email template + create_template(service=service_1, template_type=EMAIL_TYPE) + + create_ft_notification_status(date(2018, 10, 29), 'sms', service_1, count=10) + create_ft_notification_status(date(2018, 10, 29), 'sms', service_1, count=11) + create_ft_notification_status(date(2018, 10, 24), 'sms', service_1, count=8) + create_ft_notification_status(date(2018, 10, 29), 'sms', service_1, notification_status='created') + create_ft_notification_status(date(2018, 10, 29), 'email', service_1, count=3) + create_ft_notification_status(date(2018, 10, 26), 'letter', service_1, count=5) + + create_notification(sms_template, created_at=datetime(2018, 10, 31, 11, 0, 0)) + create_notification(sms_template, created_at=datetime(2018, 10, 31, 12, 0, 0), status='delivered') + create_notification(sms_template_2, created_at=datetime(2018, 10, 31, 12, 0, 0), status='delivered') + create_notification(email_template, created_at=datetime(2018, 10, 31, 13, 0, 0), status='delivered') + + # too early, shouldn't be included + create_notification(service_1.templates[0], created_at=datetime(2018, 10, 30, 12, 0, 0), status='delivered') + + results = fetch_notification_status_for_service_for_today_and_7_previous_days(service_1.id, by_template=True) + + assert [ + ('email Template Name', False, mock.ANY, 'email', 'delivered', 1), + ('email Template Name', False, mock.ANY, 'email', 'delivered', 3), + ('letter Template Name', False, mock.ANY, 'letter', 'delivered', 5), + ('sms Template 1', False, mock.ANY, 'sms', 'created', 1), + ('sms Template Name', False, mock.ANY, 'sms', 'created', 1), + ('sms Template 1', False, mock.ANY, 'sms', 'delivered', 1), + ('sms Template 2', False, mock.ANY, 'sms', 'delivered', 1), + ('sms Template Name', False, mock.ANY, 'sms', 'delivered', 8), + ('sms Template Name', False, mock.ANY, 'sms', 'delivered', 10), + ('sms Template Name', False, mock.ANY, 'sms', 'delivered', 11), + ] == sorted(results, key=lambda x: (x.notification_type, x.status, x.template_name, x.count)) + + +@pytest.mark.parametrize( + "start_date, end_date, expected_email, expected_letters, expected_sms, expected_created_sms", + [ + (29, 30, 3, 10, 10, 1), # not including today + (29, 31, 4, 10, 11, 2), # today included + (26, 31, 4, 15, 11, 2), + ] + +) +@freeze_time('2018-10-31 14:00') +def test_fetch_notification_status_totals_for_all_services( + notify_db_session, + start_date, + end_date, + expected_email, + expected_letters, + expected_sms, + expected_created_sms +): + set_up_data() + + results = sorted( + fetch_notification_status_totals_for_all_services( + start_date=date(2018, 10, start_date), end_date=date(2018, 10, end_date)), + key=lambda x: (x.notification_type, x.status) + ) + + assert len(results) == 4 + + assert results[0].notification_type == 'email' + assert results[0].status == 'delivered' + assert results[0].count == expected_email + + assert results[1].notification_type == 'letter' + assert results[1].status == 'delivered' + assert results[1].count == expected_letters + + assert results[2].notification_type == 'sms' + assert results[2].status == 'created' + assert results[2].count == expected_created_sms + + assert results[3].notification_type == 'sms' + assert results[3].status == 'delivered' + assert results[3].count == expected_sms + + +def set_up_data(): + service_2 = create_service(service_name='service_2') + create_template(service=service_2, template_type=LETTER_TYPE) + service_1 = create_service(service_name='service_1') + sms_template = create_template(service=service_1, template_type=SMS_TYPE) + email_template = create_template(service=service_1, template_type=EMAIL_TYPE) + create_ft_notification_status(date(2018, 10, 24), 'sms', service_1, count=8) + create_ft_notification_status(date(2018, 10, 26), 'letter', service_1, count=5) + create_ft_notification_status(date(2018, 10, 29), 'sms', service_1, count=10) + create_ft_notification_status(date(2018, 10, 29), 'sms', service_1, notification_status='created') + create_ft_notification_status(date(2018, 10, 29), 'email', service_1, count=3) + create_ft_notification_status(date(2018, 10, 29), 'letter', service_2, count=10) + + create_notification(service_1.templates[0], created_at=datetime(2018, 10, 30, 12, 0, 0), status='delivered') + create_notification(sms_template, created_at=datetime(2018, 10, 31, 11, 0, 0)) + create_notification(sms_template, created_at=datetime(2018, 10, 31, 12, 0, 0), status='delivered') + create_notification(email_template, created_at=datetime(2018, 10, 31, 13, 0, 0), status='delivered') + return service_1, service_2 + + +def test_fetch_notification_statuses_for_job(sample_template): + j1 = create_job(sample_template) + j2 = create_job(sample_template) + + create_ft_notification_status(date(2018, 10, 1), job=j1, notification_status='created', count=1) + create_ft_notification_status(date(2018, 10, 1), job=j1, notification_status='delivered', count=2) + create_ft_notification_status(date(2018, 10, 2), job=j1, notification_status='created', count=4) + create_ft_notification_status(date(2018, 10, 1), job=j2, notification_status='created', count=8) + + assert {x.status: x.count for x in fetch_notification_statuses_for_job(j1.id)} == { + 'created': 5, + 'delivered': 2 + } + + +@freeze_time('2018-10-31 14:00') +def test_fetch_stats_for_all_services_by_date_range(notify_db_session): + service_1, service_2 = set_up_data() + results = fetch_stats_for_all_services_by_date_range(start_date=date(2018, 10, 29), + end_date=date(2018, 10, 31)) + assert len(results) == 5 + + assert results[0].service_id == service_1.id + assert results[0].notification_type == 'email' + assert results[0].status == 'delivered' + assert results[0].count == 4 + + assert results[1].service_id == service_1.id + assert results[1].notification_type == 'sms' + assert results[1].status == 'created' + assert results[1].count == 2 + + assert results[2].service_id == service_1.id + assert results[2].notification_type == 'sms' + assert results[2].status == 'delivered' + assert results[2].count == 11 + + assert results[3].service_id == service_2.id + assert results[3].notification_type == 'letter' + assert results[3].status == 'delivered' + assert results[3].count == 10 + + assert results[4].service_id == service_2.id + assert not results[4].notification_type + assert not results[4].status + assert not results[4].count + + +@freeze_time('2018-03-30 14:00') +def test_fetch_monthly_template_usage_for_service(sample_service): + template_one = create_template(service=sample_service, template_type='sms', template_name='a') + template_two = create_template(service=sample_service, template_type='email', template_name='b') + template_three = create_template(service=sample_service, template_type='letter', template_name='c') + + create_ft_notification_status(bst_date=date(2017, 12, 10), + service=sample_service, + template=template_two, + count=3) + create_ft_notification_status(bst_date=date(2017, 12, 10), + service=sample_service, + template=template_one, + count=6) + + create_ft_notification_status(bst_date=date(2018, 1, 1), + service=sample_service, + template=template_one, + count=4) + + create_ft_notification_status(bst_date=date(2018, 3, 1), + service=sample_service, + template=template_three, + count=5) + create_notification(template=template_three, created_at=datetime.utcnow() - timedelta(days=1)) + create_notification(template=template_three, created_at=datetime.utcnow()) + results = fetch_monthly_template_usage_for_service( + datetime(2017, 4, 1), datetime(2018, 3, 31), sample_service.id + ) + + assert len(results) == 4 + + assert results[0].template_id == template_one.id + assert results[0].name == template_one.name + assert results[0].is_precompiled_letter is False + assert results[0].template_type == template_one.template_type + assert results[0].month == 12 + assert results[0].year == 2017 + assert results[0].count == 6 + assert results[1].template_id == template_two.id + assert results[1].name == template_two.name + assert results[1].is_precompiled_letter is False + assert results[1].template_type == template_two.template_type + assert results[1].month == 12 + assert results[1].year == 2017 + assert results[1].count == 3 + + assert results[2].template_id == template_one.id + assert results[2].name == template_one.name + assert results[2].is_precompiled_letter is False + assert results[2].template_type == template_one.template_type + assert results[2].month == 1 + assert results[2].year == 2018 + assert results[2].count == 4 + + assert results[3].template_id == template_three.id + assert results[3].name == template_three.name + assert results[3].is_precompiled_letter is False + assert results[3].template_type == template_three.template_type + assert results[3].month == 3 + assert results[3].year == 2018 + assert results[3].count == 6 + + +@freeze_time('2018-03-30 14:00') +def test_fetch_monthly_template_usage_for_service_does_join_to_notifications_if_today_is_not_in_date_range( + sample_service +): + template_one = create_template(service=sample_service, template_type='sms', template_name='a') + template_two = create_template(service=sample_service, template_type='email', template_name='b') + create_ft_notification_status(bst_date=date(2018, 2, 1), + service=template_two.service, + template=template_two, + count=15) + create_ft_notification_status(bst_date=date(2018, 2, 2), + service=template_one.service, + template=template_one, + count=20) + create_ft_notification_status(bst_date=date(2018, 3, 1), + service=template_one.service, + template=template_one, + count=3) + create_notification(template=template_one, created_at=datetime.utcnow()) + results = fetch_monthly_template_usage_for_service( + datetime(2018, 1, 1), datetime(2018, 2, 20), template_one.service_id + ) + + assert len(results) == 2 + + assert results[0].template_id == template_one.id + assert results[0].name == template_one.name + assert results[0].is_precompiled_letter == template_one.is_precompiled_letter + assert results[0].template_type == template_one.template_type + assert results[0].month == 2 + assert results[0].year == 2018 + assert results[0].count == 20 + assert results[1].template_id == template_two.id + assert results[1].name == template_two.name + assert results[1].is_precompiled_letter == template_two.is_precompiled_letter + assert results[1].template_type == template_two.template_type + assert results[1].month == 2 + assert results[1].year == 2018 + assert results[1].count == 15 + + +@freeze_time('2018-03-30 14:00') +def test_fetch_monthly_template_usage_for_service_does_not_include_cancelled_status( + sample_template +): + create_ft_notification_status(bst_date=date(2018, 3, 1), + service=sample_template.service, + template=sample_template, + notification_status='cancelled', + count=15) + create_notification(template=sample_template, created_at=datetime.utcnow(), status='cancelled') + results = fetch_monthly_template_usage_for_service( + datetime(2018, 1, 1), datetime(2018, 3, 31), sample_template.service_id + ) + + assert len(results) == 0 + + +@freeze_time('2018-03-30 14:00') +def test_fetch_monthly_template_usage_for_service_does_not_include_test_notifications( + sample_template +): + create_ft_notification_status(bst_date=date(2018, 3, 1), + service=sample_template.service, + template=sample_template, + notification_status='delivered', + key_type='test', + count=15) + create_notification(template=sample_template, + created_at=datetime.utcnow(), + status='delivered', + key_type='test',) + results = fetch_monthly_template_usage_for_service( + datetime(2018, 1, 1), datetime(2018, 3, 31), sample_template.service_id + ) + + assert len(results) == 0 diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index a6bea2d48..524d17f8e 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -13,112 +13,61 @@ from app.dao.jobs_dao import ( dao_set_scheduled_jobs_to_pending, dao_get_future_scheduled_job_by_id_and_service_id, dao_get_notification_outcomes_for_job, - dao_get_jobs_older_than_limited_by + dao_get_jobs_older_than_data_retention, ) from app.models import ( Job, EMAIL_TYPE, SMS_TYPE, LETTER_TYPE ) -from tests.app.conftest import sample_job as create_job -from tests.app.conftest import sample_notification as create_notification -from tests.app.conftest import sample_service as create_service -from tests.app.conftest import sample_template as create_template -from tests.app.db import ( - create_user -) +from tests.app.db import create_job, create_service, create_template, create_notification def test_should_have_decorated_notifications_dao_functions(): assert dao_get_notification_outcomes_for_job.__wrapped__.__name__ == 'dao_get_notification_outcomes_for_job' # noqa -def test_should_get_all_statuses_for_notifications_associated_with_job( - notify_db, - notify_db_session, - sample_service, - sample_job -): - notification = partial(create_notification, notify_db, notify_db_session, service=sample_service, job=sample_job) - notification(status='created') - notification(status='sending') - notification(status='delivered') - notification(status='pending') - notification(status='failed') - notification(status='technical-failure') - notification(status='temporary-failure') - notification(status='permanent-failure') - notification(status='sent') +def test_should_count_of_statuses_for_notifications_associated_with_job(sample_template, sample_job): + create_notification(sample_template, job=sample_job, status='created') + create_notification(sample_template, job=sample_job, status='created') + create_notification(sample_template, job=sample_job, status='created') + create_notification(sample_template, job=sample_job, status='sending') + create_notification(sample_template, job=sample_job, status='delivered') - results = dao_get_notification_outcomes_for_job(sample_service.id, sample_job.id) - assert set([(row.count, row.status) for row in results]) == set([ - (1, 'created'), - (1, 'sending'), - (1, 'delivered'), - (1, 'pending'), - (1, 'failed'), - (1, 'technical-failure'), - (1, 'temporary-failure'), - (1, 'permanent-failure'), - (1, 'sent') - ]) - - -def test_should_count_of_statuses_for_notifications_associated_with_job( - notify_db, - notify_db_session, - sample_service, - sample_job -): - notification = partial(create_notification, notify_db, notify_db_session, service=sample_service, job=sample_job) - notification(status='created') - notification(status='created') - - notification(status='sending') - notification(status='sending') - notification(status='sending') - notification(status='sending') - notification(status='delivered') - notification(status='delivered') - - results = dao_get_notification_outcomes_for_job(sample_service.id, sample_job.id) - assert set([(row.count, row.status) for row in results]) == set([ - (2, 'created'), - (4, 'sending'), - (2, 'delivered') - ]) + results = dao_get_notification_outcomes_for_job(sample_template.service_id, sample_job.id) + assert {row.status: row.count for row in results} == { + 'created': 3, + 'sending': 1, + 'delivered': 1, + } def test_should_return_zero_length_array_if_no_notifications_for_job(sample_service, sample_job): assert len(dao_get_notification_outcomes_for_job(sample_job.id, sample_service.id)) == 0 -def test_should_return_notifications_only_for_this_job(notify_db, notify_db_session, sample_service): - job_1 = create_job(notify_db, notify_db_session, service=sample_service) - job_2 = create_job(notify_db, notify_db_session, service=sample_service) +def test_should_return_notifications_only_for_this_job(sample_template): + job_1 = create_job(sample_template) + job_2 = create_job(sample_template) - create_notification(notify_db, notify_db_session, service=sample_service, job=job_1, status='created') - create_notification(notify_db, notify_db_session, service=sample_service, job=job_2, status='created') + create_notification(sample_template, job=job_1, status='created') + create_notification(sample_template, job=job_2, status='sent') - results = dao_get_notification_outcomes_for_job(sample_service.id, job_1.id) - assert [(row.count, row.status) for row in results] == [ - (1, 'created') - ] + results = dao_get_notification_outcomes_for_job(sample_template.service_id, job_1.id) + assert {row.status: row.count for row in results} == {'created': 1} -def test_should_return_notifications_only_for_this_service(notify_db, notify_db_session): - service_1 = create_service(notify_db, notify_db_session, service_name="one", email_from="one") - service_2 = create_service(notify_db, notify_db_session, service_name="two", email_from="two") +def test_should_return_notifications_only_for_this_service(sample_notification_with_job): + other_service = create_service(service_name='one') + other_template = create_template(service=other_service) + other_job = create_job(other_template) - job_1 = create_job(notify_db, notify_db_session, service=service_1) - job_2 = create_job(notify_db, notify_db_session, service=service_2) + create_notification(other_template, job=other_job) - create_notification(notify_db, notify_db_session, service=service_1, job=job_1, status='created') - create_notification(notify_db, notify_db_session, service=service_2, job=job_2, status='created') - - assert len(dao_get_notification_outcomes_for_job(service_1.id, job_2.id)) == 0 + assert len(dao_get_notification_outcomes_for_job(sample_notification_with_job.service_id, other_job.id)) == 0 + assert len(dao_get_notification_outcomes_for_job(other_service.id, sample_notification_with_job.id)) == 0 -def test_create_job(sample_template): +def test_create_sample_job(sample_template): assert Job.query.count() == 0 job_id = uuid.uuid4() @@ -147,14 +96,12 @@ def test_get_job_by_id(sample_job): assert sample_job == job_from_db -def test_get_jobs_for_service(notify_db, notify_db_session, sample_template): - one_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template) +def test_get_jobs_for_service(sample_template): + one_job = create_job(sample_template) - other_user = create_user(email="test@digital.cabinet-office.gov.uk") - other_service = create_service(notify_db, notify_db_session, user=other_user, service_name="other service", - email_from='other.service') - other_template = create_template(notify_db, notify_db_session, service=other_service) - other_job = create_job(notify_db, notify_db_session, service=other_service, template=other_template) + other_service = create_service(service_name="other service") + other_template = create_template(service=other_service) + other_job = create_job(other_template) one_job_from_db = dao_get_jobs_by_service_id(one_job.service_id).items other_job_from_db = dao_get_jobs_by_service_id(other_job.service_id).items @@ -168,10 +115,9 @@ def test_get_jobs_for_service(notify_db, notify_db_session, sample_template): assert one_job_from_db != other_job_from_db -def test_get_jobs_for_service_with_limit_days_param(notify_db, notify_db_session, sample_template): - one_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template) - old_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template, - created_at=datetime.now() - timedelta(days=8)) +def test_get_jobs_for_service_with_limit_days_param(sample_template): + one_job = create_job(sample_template) + old_job = create_job(sample_template, created_at=datetime.now() - timedelta(days=8)) jobs = dao_get_jobs_by_service_id(one_job.service_id).items @@ -185,34 +131,27 @@ def test_get_jobs_for_service_with_limit_days_param(notify_db, notify_db_session assert old_job not in jobs_limit_days -def test_get_jobs_for_service_with_limit_days_edge_case(notify_db, notify_db_session, sample_template): - one_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template) - job_two = create_job(notify_db, notify_db_session, sample_template.service, sample_template, - created_at=(datetime.now() - timedelta(days=7)).date()) - one_second_after_midnight = datetime.combine((datetime.now() - timedelta(days=7)).date(), - datetime.strptime("000001", "%H%M%S").time()) - just_after_midnight_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template, - created_at=one_second_after_midnight) - job_eight_days_old = create_job(notify_db, notify_db_session, sample_template.service, sample_template, - created_at=datetime.now() - timedelta(days=8)) +@freeze_time('2017-06-10') +def test_get_jobs_for_service_with_limit_days_edge_case(sample_template): + one_job = create_job(sample_template) + just_after_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 2, 23, 0, 1)) + just_before_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 2, 22, 59, 0)) jobs_limit_days = dao_get_jobs_by_service_id(one_job.service_id, limit_days=7).items - assert len(jobs_limit_days) == 3 + assert len(jobs_limit_days) == 2 assert one_job in jobs_limit_days - assert job_two in jobs_limit_days assert just_after_midnight_job in jobs_limit_days - assert job_eight_days_old not in jobs_limit_days + assert just_before_midnight_job not in jobs_limit_days def test_get_jobs_for_service_in_processed_at_then_created_at_order(notify_db, notify_db_session, sample_template): - _create_job = partial(create_job, notify_db, notify_db_session, sample_template.service, sample_template) from_hour = partial(datetime, 2001, 1, 1) created_jobs = [ - _create_job(created_at=from_hour(2), processing_started=None), - _create_job(created_at=from_hour(1), processing_started=None), - _create_job(created_at=from_hour(1), processing_started=from_hour(4)), - _create_job(created_at=from_hour(2), processing_started=from_hour(3)), + create_job(sample_template, created_at=from_hour(2), processing_started=None), + create_job(sample_template, created_at=from_hour(1), processing_started=None), + create_job(sample_template, created_at=from_hour(1), processing_started=from_hour(4)), + create_job(sample_template, created_at=from_hour(2), processing_started=from_hour(3)), ] jobs = dao_get_jobs_by_service_id(sample_template.service.id).items @@ -235,21 +174,20 @@ def test_update_job(sample_job): assert job_from_db.job_status == 'in progress' -def test_set_scheduled_jobs_to_pending_gets_all_jobs_in_scheduled_state_before_now(notify_db, notify_db_session): +def test_set_scheduled_jobs_to_pending_gets_all_jobs_in_scheduled_state_before_now(sample_template): one_minute_ago = datetime.utcnow() - timedelta(minutes=1) one_hour_ago = datetime.utcnow() - timedelta(minutes=60) - job_new = create_job(notify_db, notify_db_session, scheduled_for=one_minute_ago, job_status='scheduled') - job_old = create_job(notify_db, notify_db_session, scheduled_for=one_hour_ago, job_status='scheduled') + job_new = create_job(sample_template, scheduled_for=one_minute_ago, job_status='scheduled') + job_old = create_job(sample_template, scheduled_for=one_hour_ago, job_status='scheduled') jobs = dao_set_scheduled_jobs_to_pending() assert len(jobs) == 2 assert jobs[0].id == job_old.id assert jobs[1].id == job_new.id -def test_set_scheduled_jobs_to_pending_gets_ignores_jobs_not_scheduled(notify_db, notify_db_session): +def test_set_scheduled_jobs_to_pending_gets_ignores_jobs_not_scheduled(sample_template, sample_job): one_minute_ago = datetime.utcnow() - timedelta(minutes=1) - create_job(notify_db, notify_db_session) - job_scheduled = create_job(notify_db, notify_db_session, scheduled_for=one_minute_ago, job_status='scheduled') + job_scheduled = create_job(sample_template, scheduled_for=one_minute_ago, job_status='scheduled') jobs = dao_set_scheduled_jobs_to_pending() assert len(jobs) == 1 assert jobs[0].id == job_scheduled.id @@ -260,11 +198,11 @@ def test_set_scheduled_jobs_to_pending_gets_ignores_jobs_scheduled_in_the_future assert len(jobs) == 0 -def test_set_scheduled_jobs_to_pending_updates_rows(notify_db, notify_db_session): +def test_set_scheduled_jobs_to_pending_updates_rows(sample_template): one_minute_ago = datetime.utcnow() - timedelta(minutes=1) one_hour_ago = datetime.utcnow() - timedelta(minutes=60) - create_job(notify_db, notify_db_session, scheduled_for=one_minute_ago, job_status='scheduled') - create_job(notify_db, notify_db_session, scheduled_for=one_hour_ago, job_status='scheduled') + create_job(sample_template, scheduled_for=one_minute_ago, job_status='scheduled') + create_job(sample_template, scheduled_for=one_hour_ago, job_status='scheduled') jobs = dao_set_scheduled_jobs_to_pending() assert len(jobs) == 2 assert jobs[0].job_status == 'pending' @@ -277,7 +215,7 @@ def test_get_future_scheduled_job_gets_a_job_yet_to_send(sample_scheduled_job): @freeze_time('2016-10-31 10:00:00') -def test_should_get_jobs_seven_days_old(notify_db, notify_db_session, sample_template): +def test_should_get_jobs_seven_days_old(sample_template): """ Jobs older than seven days are deleted, but only two day's worth (two-day window) """ @@ -289,14 +227,13 @@ def test_should_get_jobs_seven_days_old(notify_db, notify_db_session, sample_tem nine_days_ago = eight_days_ago - timedelta(days=2) nine_days_one_second_ago = nine_days_ago - timedelta(seconds=1) - job = partial(create_job, notify_db, notify_db_session) - job(created_at=seven_days_ago) - job(created_at=within_seven_days) - job_to_delete = job(created_at=eight_days_ago) - job(created_at=nine_days_ago) - job(created_at=nine_days_one_second_ago) + create_job(sample_template, created_at=seven_days_ago) + create_job(sample_template, created_at=within_seven_days) + job_to_delete = create_job(sample_template, created_at=eight_days_ago) + create_job(sample_template, created_at=nine_days_ago, archived=True) + create_job(sample_template, created_at=nine_days_one_second_ago, archived=True) - jobs = dao_get_jobs_older_than_limited_by(job_types=[sample_template.template_type]) + jobs = dao_get_jobs_older_than_data_retention(notification_types=[sample_template.template_type]) assert len(jobs) == 1 assert jobs[0].id == job_to_delete.id @@ -306,7 +243,7 @@ def test_get_jobs_for_service_is_paginated(notify_db, notify_db_session, sample_ with freeze_time('2015-01-01T00:00:00') as the_time: for _ in range(10): the_time.tick(timedelta(hours=1)) - create_job(notify_db, notify_db_session, sample_service, sample_template) + create_job(sample_template) res = dao_get_jobs_by_service_id(sample_service.id, page=1, page_size=2) @@ -328,19 +265,11 @@ def test_get_jobs_for_service_is_paginated(notify_db, notify_db_session, sample_ 'Report', ]) def test_get_jobs_for_service_doesnt_return_test_messages( - notify_db, - notify_db_session, sample_template, sample_job, file_name, ): - create_job( - notify_db, - notify_db_session, - sample_template.service, - sample_template, - original_file_name=file_name, - ) + create_job(sample_template, original_file_name=file_name,) jobs = dao_get_jobs_by_service_id(sample_job.service_id).items @@ -348,19 +277,18 @@ def test_get_jobs_for_service_doesnt_return_test_messages( @freeze_time('2016-10-31 10:00:00') -def test_should_get_jobs_seven_days_old_filters_type(notify_db, notify_db_session): +def test_should_get_jobs_seven_days_old_filters_type(sample_service): eight_days_ago = datetime.utcnow() - timedelta(days=8) - letter_template = create_template(notify_db, notify_db_session, template_type=LETTER_TYPE) - sms_template = create_template(notify_db, notify_db_session, template_type=SMS_TYPE) - email_template = create_template(notify_db, notify_db_session, template_type=EMAIL_TYPE) + letter_template = create_template(sample_service, template_type=LETTER_TYPE) + sms_template = create_template(sample_service, template_type=SMS_TYPE) + email_template = create_template(sample_service, template_type=EMAIL_TYPE) - job = partial(create_job, notify_db, notify_db_session, created_at=eight_days_ago) - job_to_remain = job(template=letter_template) - job(template=sms_template) - job(template=email_template) + job_to_remain = create_job(letter_template, created_at=eight_days_ago) + create_job(sms_template, created_at=eight_days_ago) + create_job(email_template, created_at=eight_days_ago) - jobs = dao_get_jobs_older_than_limited_by( - job_types=[EMAIL_TYPE, SMS_TYPE] + jobs = dao_get_jobs_older_than_data_retention( + notification_types=[EMAIL_TYPE, SMS_TYPE] ) assert len(jobs) == 2 diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py deleted file mode 100644 index 4e1bb27ab..000000000 --- a/tests/app/dao/test_notification_usage_dao.py +++ /dev/null @@ -1,235 +0,0 @@ -import uuid -from datetime import datetime, timedelta - -from freezegun import freeze_time - -from app.dao.date_util import get_financial_year -from app.dao.notification_usage_dao import ( - get_rates_for_daterange, - get_billing_data_for_month, - get_monthly_billing_data, - billing_letter_data_per_month_query -) -from app.models import ( - Rate, - SMS_TYPE, -) -from tests.app.db import create_notification, create_rate, create_template, create_service - - -def test_get_rates_for_daterange(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) - set_up_rate(notify_db, datetime(2017, 3, 31, 23), 0.0158) - start_date, end_date = get_financial_year(2017) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 1 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-03-31 23:00:00" - assert rates[0].rate == 0.0158 - - -def test_get_rates_for_daterange_multiple_result_per_year(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) - set_up_rate(notify_db, datetime(2017, 4, 1), 0.0158) - start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 2 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00" - assert rates[0].rate == 0.015 - assert datetime.strftime(rates[1].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-05-18 00:00:00" - assert rates[1].rate == 0.016 - - -def test_get_rates_for_daterange_returns_correct_rates(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2017) - rates_2017 = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates_2017) == 2 - assert datetime.strftime(rates_2017[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-09-01 00:00:00" - assert rates_2017[0].rate == 0.016 - assert datetime.strftime(rates_2017[1].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" - assert rates_2017[1].rate == 0.0175 - - -def test_get_rates_for_daterange_in_the_future(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2018) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" - assert rates[0].rate == 0.0175 - - -def test_get_rates_for_daterange_returns_empty_list_if_year_is_before_earliest_rate(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2015) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert rates == [] - - -def test_get_rates_for_daterange_early_rate(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2015, 6, 1), 0.014) - set_up_rate(notify_db, datetime(2016, 6, 1), 0.015) - set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 3 - - -def test_get_rates_for_daterange_edge_case(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 3, 31, 23, 00), 0.015) - set_up_rate(notify_db, datetime(2017, 3, 31, 23, 00), 0.0175) - start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 1 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-03-31 23:00:00" - assert rates[0].rate == 0.015 - - -def test_get_rates_for_daterange_where_daterange_is_one_month_that_falls_between_rate_valid_from( - notify_db, notify_db_session -): - set_up_rate(notify_db, datetime(2017, 1, 1), 0.175) - set_up_rate(notify_db, datetime(2017, 3, 31), 0.123) - start_date = datetime(2017, 2, 1, 00, 00, 00) - end_date = datetime(2017, 2, 28, 23, 59, 59, 99999) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 1 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-01-01 00:00:00" - assert rates[0].rate == 0.175 - - -def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - # previous year - create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), - status='sending', billable_units=1) - # current year - create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), - status='sending', billable_units=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3, rate_multiplier=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3, rate_multiplier=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 30), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=4) - - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - # next year - create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), - sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) - results = get_monthly_billing_data(sample_template.service_id, 2016) - assert len(results) == 4 - # (billable_units, rate_multiplier, international, type, rate) - assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014) - assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014) - assert results[2] == ('July', 7, 1, False, SMS_TYPE, 0.014) - assert results[3] == ('July', 6, 2, False, SMS_TYPE, 0.014) - - -def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_session, sample_template, - sample_email_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - set_up_rate(notify_db, datetime(2016, 6, 5), 0.0175) - set_up_rate(notify_db, datetime(2017, 7, 5), 0.018) - # previous year - create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), - status='sending', billable_units=1) - # current year - create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), - status='sending', billable_units=2) - create_notification(template=sample_template, created_at=datetime(2016, 6, 1), sent_at=datetime(2016, 6, 1), - status='sending', billable_units=3) - create_notification(template=sample_template, created_at=datetime(2016, 6, 15), sent_at=datetime(2016, 6, 15), - status='sending', billable_units=4) - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), - sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), - sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - # next year - create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), - sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) - results = get_monthly_billing_data(sample_template.service_id, 2016) - assert len(results) == 4 - assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014) - assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014) - assert results[2] == ('June', 3, 1, False, SMS_TYPE, 0.014) - assert results[3] == ('June', 4, 1, False, SMS_TYPE, 0.0175) - - -def test_get_monthly_billing_data_with_no_notifications_for_daterange(notify_db, notify_db_session, sample_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - results = get_monthly_billing_data(sample_template.service_id, 2016) - assert results == [] - - -def set_up_rate(notify_db, start_date, value): - rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=SMS_TYPE) - notify_db.session.add(rate) - - -@freeze_time("2016-05-01") -def test_get_billing_data_for_month_where_start_date_before_rate_returns_empty( - sample_template -): - create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE) - - results = get_monthly_billing_data( - service_id=sample_template.service_id, - year=2015 - ) - - assert not results - - -@freeze_time("2016-05-01") -def test_get_monthly_billing_data_where_start_date_before_rate_returns_empty( - sample_template -): - now = datetime.utcnow() - create_rate(now, 0.014, SMS_TYPE) - - results = get_billing_data_for_month( - service_id=sample_template.service_id, - start_date=now - timedelta(days=2), - end_date=now - timedelta(days=1), - notification_type=SMS_TYPE - ) - - assert not results - - -def test_billing_letter_data_per_month_query( - notify_db_session -): - service = create_service() - template = create_template(service=service, template_type='letter') - create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21), - status='delivered') - create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21), - status='delivered') - create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21), - status='delivered') - - results = billing_letter_data_per_month_query(service_id=service.id, - start_date=datetime(2017, 2, 1), - end_date=datetime(2017, 2, 28)) - - assert len(results) == 1 - assert results[0].rate == 0.3 - assert results[0].billing_units == 3 diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 6ea7123e3..4a1ec0b4b 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime, timedelta +from datetime import datetime import pytest from freezegun import freeze_time @@ -7,7 +7,6 @@ from sqlalchemy.exc import IntegrityError, SQLAlchemyError from sqlalchemy.orm.exc import FlushError, NoResultFound from app import db -from app.celery.scheduled_tasks import daily_stats_template_usage_by_month from app.dao.inbound_numbers_dao import ( dao_set_inbound_number_to_service, dao_get_available_inbound_numbers, @@ -27,13 +26,10 @@ from app.dao.services_dao import ( dao_fetch_todays_stats_for_service, fetch_todays_total_message_count, dao_fetch_todays_stats_for_all_services, - fetch_stats_by_date_range_for_all_services, dao_suspend_service, dao_resume_service, dao_fetch_active_users_for_service, dao_fetch_service_by_inbound_number, - dao_fetch_monthly_historical_stats_by_template, - dao_fetch_monthly_historical_usage_by_template_for_service ) from app.dao.users_dao import save_model_user, create_user_code from app.models import ( @@ -775,25 +771,6 @@ def test_dao_fetch_todays_stats_for_all_services_can_exclude_from_test_key(notif assert stats[0].count == 2 -def test_fetch_stats_by_date_range_for_all_services(notify_db_session): - template = create_template(service=create_service()) - create_notification(template=template, created_at=datetime.now() - timedelta(days=4)) - create_notification(template=template, created_at=datetime.now() - timedelta(days=3)) - result_one = create_notification(template=template, created_at=datetime.now() - timedelta(days=2)) - create_notification(template=template, created_at=datetime.now() - timedelta(days=1)) - create_notification(template=template, created_at=datetime.now()) - - start_date = (datetime.utcnow() - timedelta(days=2)).date() - end_date = (datetime.utcnow() - timedelta(days=1)).date() - - results = fetch_stats_by_date_range_for_all_services(start_date, end_date) - - assert len(results) == 1 - assert results[0] == (result_one.service.id, result_one.service.name, result_one.service.restricted, - result_one.service.research_mode, result_one.service.active, - result_one.service.created_at, 'sms', 'created', 2) - - @freeze_time('2001-01-01T23:59:00') def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys(notify_db_session): service = create_service() @@ -807,64 +784,6 @@ def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys(noti assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) -@pytest.mark.parametrize("start_delta, end_delta, expected", - [("5", "1", "4"), # a date range less than 7 days ago returns test and normal notifications - ("9", "8", "1"), # a date range older than 9 days does not return test notifications. - ("8", "4", "2")]) # a date range that starts more than 7 days ago -@freeze_time('2017-10-23T00:00:00') -def test_fetch_stats_by_date_range_for_all_services_returns_test_notifications(notify_db_session, - start_delta, - end_delta, - expected): - template = create_template(service=create_service()) - result_one = create_notification(template=template, created_at=datetime.now(), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=2), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=3), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=4), key_type='normal') - create_notification(template=template, created_at=datetime.now() - timedelta(days=4), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=8), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=8), key_type='normal') - - start_date = (datetime.utcnow() - timedelta(days=int(start_delta))).date() - end_date = (datetime.utcnow() - timedelta(days=int(end_delta))).date() - - results = fetch_stats_by_date_range_for_all_services(start_date, end_date, include_from_test_key=True) - - assert len(results) == 1 - assert results[0] == (result_one.service.id, result_one.service.name, result_one.service.restricted, - result_one.service.research_mode, result_one.service.active, result_one.service.created_at, - 'sms', 'created', int(expected)) - - -@pytest.mark.parametrize("start_delta, end_delta, expected", - [("5", "1", "4"), # a date range less than 7 days ago returns test and normal notifications - ("9", "8", "1"), # a date range older than 9 days does not return test notifications. - ("8", "4", "2")]) # a date range that starts more than 7 days ago -@freeze_time('2017-10-23T23:00:00') -def test_fetch_stats_by_date_range_during_bst_hour_for_all_services_returns_test_notifications( - notify_db_session, start_delta, end_delta, expected -): - template = create_template(service=create_service()) - result_one = create_notification(template=template, created_at=datetime.now(), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=2), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=3), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=4), key_type='normal') - create_notification(template=template, created_at=datetime.now() - timedelta(days=4), key_type='test') - create_notification(template=template, created_at=datetime.now() - timedelta(days=8), key_type='normal') - create_notification(template=template, created_at=datetime.now() - timedelta(days=9), key_type='normal') - create_notification(template=template, created_at=datetime.now() - timedelta(days=9), key_type='test') - - start_date = (datetime.utcnow() - timedelta(days=int(start_delta))).date() - end_date = (datetime.utcnow() - timedelta(days=int(end_delta))).date() - - results = fetch_stats_by_date_range_for_all_services(start_date, end_date, include_from_test_key=True) - - assert len(results) == 1 - assert results[0] == (result_one.service.id, result_one.service.name, result_one.service.restricted, - result_one.service.research_mode, result_one.service.active, result_one.service.created_at, - 'sms', 'created', int(expected)) - - @freeze_time('2001-01-01T23:59:00') def test_dao_resume_service_marks_service_as_active_and_api_keys_are_still_revoked(notify_db_session): service = create_service() @@ -954,466 +873,9 @@ def _assert_service_permissions(service_permissions, expected): assert set(expected) == set(p.permission for p in service_permissions) -def test_dao_fetch_monthly_historical_stats_by_template(notify_db_session): - service = create_service() - template_one = create_template(service=service, template_name='1') - template_two = create_template(service=service, template_name='2') - - create_notification(created_at=datetime(2017, 10, 1), template=template_one, status='delivered') - create_notification(created_at=datetime(2016, 4, 1), template=template_two, status='delivered') - create_notification(created_at=datetime(2016, 4, 1), template=template_two, status='delivered') - create_notification(created_at=datetime.now(), template=template_two, status='delivered') - - result = sorted(dao_fetch_monthly_historical_stats_by_template(), key=lambda x: (x.month, x.year)) - - assert len(result) == 2 - - assert result[0].template_id == template_two.id - assert result[0].month == 4 - assert result[0].year == 2016 - assert result[0].count == 2 - - assert result[1].template_id == template_one.id - assert result[1].month == 10 - assert result[1].year == 2017 - assert result[1].count == 1 - - -def test_dao_fetch_monthly_historical_usage_by_template_for_service_no_stats_today( - notify_db_session, -): - service = create_service() - template_one = create_template(service=service, template_name='1') - template_two = create_template(service=service, template_name='2') - - n = create_notification(created_at=datetime(2017, 10, 1), template=template_one, status='delivered') - create_notification(created_at=datetime(2017, 4, 1), template=template_two, status='delivered') - create_notification(created_at=datetime(2017, 4, 1), template=template_two, status='delivered') - create_notification(created_at=datetime.now(), template=template_two, status='delivered') - - daily_stats_template_usage_by_month() - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 2 - - assert result[0].template_id == template_two.id - assert result[0].name == template_two.name - assert result[0].template_type == template_two.template_type - assert result[0].month == 4 - assert result[0].year == 2017 - assert result[0].count == 2 - - assert result[1].template_id == template_one.id - assert result[1].name == template_one.name - assert result[1].template_type == template_two.template_type - assert result[1].month == 10 - assert result[1].year == 2017 - assert result[1].count == 1 - - -@freeze_time("2017-11-10 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_add_to_historical( - notify_db_session, -): - service = create_service() - template_one = create_template(service=service, template_name='1') - template_two = create_template(service=service, template_name='2') - template_three = create_template(service=service, template_name='3') - - date = datetime.now() - day = date.day - month = date.month - year = date.year - - n = create_notification(created_at=datetime(2017, 9, 1), template=template_one, status='delivered') - create_notification(created_at=datetime(year, month, day) - timedelta(days=1), template=template_two, - status='delivered') - create_notification(created_at=datetime(year, month, day) - timedelta(days=1), template=template_two, - status='delivered') - - daily_stats_template_usage_by_month() - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 2 - - assert result[0].template_id == template_one.id - assert result[0].name == template_one.name - assert result[0].template_type == template_one.template_type - assert result[0].month == 9 - assert result[0].year == 2017 - assert result[0].count == 1 - - assert result[1].template_id == template_two.id - assert result[1].name == template_two.name - assert result[1].template_type == template_two.template_type - assert result[1].month == 11 - assert result[1].year == 2017 - assert result[1].count == 2 - - create_notification( - template=template_three, - created_at=datetime.now(), - status='delivered' - ) - create_notification( - template=template_two, - created_at=datetime.now(), - status='delivered' - ) - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 3 - - assert result[0].template_id == template_one.id - assert result[0].name == template_one.name - assert result[0].template_type == template_one.template_type - assert result[0].month == 9 - assert result[0].year == 2017 - assert result[0].count == 1 - - assert result[1].template_id == template_two.id - assert result[1].name == template_two.name - assert result[1].template_type == template_two.template_type - assert result[1].month == month - assert result[1].year == year - assert result[1].count == 3 - - assert result[2].template_id == template_three.id - assert result[2].name == template_three.name - assert result[2].template_type == template_three.template_type - assert result[2].month == 11 - assert result[2].year == 2017 - assert result[2].count == 1 - - -@freeze_time("2017-11-10 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_does_add_old_notification( - notify_db_session, -): - template_one, template_three, template_two = create_email_sms_letter_template() - - date = datetime.now() - day = date.day - month = date.month - year = date.year - - n = create_notification(created_at=datetime(2017, 9, 1), template=template_one, status='delivered') - create_notification(created_at=datetime(year, month, day) - timedelta(days=1), template=template_two, - status='delivered') - create_notification(created_at=datetime(year, month, day) - timedelta(days=1), template=template_two, - status='delivered') - - daily_stats_template_usage_by_month() - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 2 - - assert result[0].template_id == template_one.id - assert result[0].name == template_one.name - assert result[0].template_type == template_one.template_type - assert result[0].month == 9 - assert result[0].year == 2017 - assert result[0].count == 1 - - assert result[1].template_id == template_two.id - assert result[1].name == template_two.name - assert result[1].template_type == template_two.template_type - assert result[1].month == 11 - assert result[1].year == 2017 - assert result[1].count == 2 - - create_notification( - template=template_three, - created_at=datetime.utcnow() - timedelta(days=2), - status='delivered' - ) - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 2 - - -@freeze_time("2017-11-10 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_get_this_year_only( - notify_db_session, -): - template_one, template_three, template_two = create_email_sms_letter_template() - - date = datetime.now() - day = date.day - month = date.month - year = date.year - - n = create_notification(created_at=datetime(2016, 9, 1), template=template_one, status='delivered') - create_notification(created_at=datetime(year, month, day) - timedelta(days=1), template=template_two, - status='delivered') - create_notification(created_at=datetime(year, month, day) - timedelta(days=1), template=template_two, - status='delivered') - - daily_stats_template_usage_by_month() - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 1 - - assert result[0].template_id == template_two.id - assert result[0].name == template_two.name - assert result[0].template_type == template_two.template_type - assert result[0].month == 11 - assert result[0].year == 2017 - assert result[0].count == 2 - - create_notification( - template=template_three, - created_at=datetime.utcnow() - timedelta(days=2) - ) - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 1 - - create_notification( - template=template_three, - created_at=datetime.utcnow() - ) - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 2 - - def create_email_sms_letter_template(): service = create_service() template_one = create_template(service=service, template_name='1', template_type='email') template_two = create_template(service=service, template_name='2', template_type='sms') template_three = create_template(service=service, template_name='3', template_type='letter') return template_one, template_three, template_two - - -@freeze_time("2017-11-10 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_combined_historical_current( - notify_db_session, -): - template_one = create_template(service=create_service(), template_name='1') - - date = datetime.now() - day = date.day - month = date.month - year = date.year - - n = create_notification(status='delivered', created_at=datetime(year, month, day) - timedelta(days=30), - template=template_one) - - daily_stats_template_usage_by_month() - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 1 - - assert result[0].template_id == template_one.id - assert result[0].name == template_one.name - assert result[0].template_type == template_one.template_type - assert result[0].month == 10 - assert result[0].year == 2017 - assert result[0].count == 1 - - create_notification( - template=template_one, - created_at=datetime.utcnow() - ) - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 2 - - assert result[0].template_id == template_one.id - assert result[0].name == template_one.name - assert result[0].template_type == template_one.template_type - assert result[0].month == 10 - assert result[0].year == 2017 - assert result[0].count == 1 - - assert result[1].template_id == template_one.id - assert result[1].name == template_one.name - assert result[1].template_type == template_one.template_type - assert result[1].month == 11 - assert result[1].year == 2017 - assert result[1].count == 1 - - -@freeze_time("2017-11-10 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_does_not_return_double_precision_values( - notify_db_session, -): - template_one = create_template(service=create_service()) - - n = create_notification( - template=template_one, - created_at=datetime.utcnow() - ) - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.month, x.year) - ) - - assert len(result) == 1 - - assert result[0].template_id == template_one.id - assert result[0].name == template_one.name - assert result[0].template_type == template_one.template_type - assert result[0].month == 11 - assert len(str(result[0].month)) == 2 - assert result[0].year == 2017 - assert len(str(result[0].year)) == 4 - assert result[0].count == 1 - - -@freeze_time("2018-03-10 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_returns_financial_year( - notify_db, - notify_db_session, -): - service = create_service() - template_one = create_template(service=service, template_name='1', template_type='email') - - date = datetime.now() - day = date.day - year = date.year - - create_notification(template=template_one, status='delivered', created_at=datetime(year - 1, 1, day)) - create_notification(template=template_one, status='delivered', created_at=datetime(year - 1, 3, day)) - create_notification(template=template_one, status='delivered', created_at=datetime(year - 1, 4, day)) - create_notification(template=template_one, status='delivered', created_at=datetime(year - 1, 5, day)) - create_notification(template=template_one, status='delivered', created_at=datetime(year, 1, day)) - create_notification(template=template_one, status='delivered', created_at=datetime(year, 2, day)) - - daily_stats_template_usage_by_month() - - n = create_notification( - template=template_one, - created_at=datetime.utcnow() - ) - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), - key=lambda x: (x.year, x.month) - ) - - assert len(result) == 5 - - assert result[0].month == 4 - assert result[0].year == 2017 - assert result[1].month == 5 - assert result[1].year == 2017 - assert result[2].month == 1 - assert result[2].year == 2018 - assert result[3].month == 2 - assert result[3].year == 2018 - assert result[4].month == 3 - assert result[4].year == 2018 - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2014), - key=lambda x: (x.year, x.month) - ) - - assert len(result) == 0 - - -@freeze_time("2018-03-10 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_only_returns_for_service( - notify_db_session -): - template_one = create_template(service=create_service(), template_name='1', template_type='email') - - date = datetime.now() - day = date.day - year = date.year - - create_notification(template=template_one, created_at=datetime(year, 1, day)) - create_notification(template=template_one, created_at=datetime(year, 2, day)) - create_notification(template=template_one, created_at=datetime(year, 3, day)) - - service_two = create_service(service_name='other_service', user=create_user()) - template_two = create_template(service=service_two, template_name='1', template_type='email') - - create_notification(template=template_two) - create_notification(template=template_two) - - daily_stats_template_usage_by_month() - - x = dao_fetch_monthly_historical_usage_by_template_for_service(template_one.service_id, 2017) - - result = sorted( - x, - key=lambda x: (x.year, x.month) - ) - - assert len(result) == 3 - - result = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(service_two.id, 2017), - key=lambda x: (x.year, x.month) - ) - - assert len(result) == 1 - - -@freeze_time("2018-01-01 11:09:00.000000") -def test_dao_fetch_monthly_historical_usage_by_template_for_service_ignores_test_api_keys(notify_db_session): - service = create_service() - template_1 = create_template(service, template_name='1') - template_2 = create_template(service, template_name='2') - template_3 = create_template(service, template_name='3') - - create_notification(template_1, key_type=KEY_TYPE_TEST) - create_notification(template_2, key_type=KEY_TYPE_TEAM) - create_notification(template_3, key_type=KEY_TYPE_NORMAL) - - results = sorted( - dao_fetch_monthly_historical_usage_by_template_for_service(service.id, 2017), - key=lambda x: x.name - ) - - assert len(results) == 2 - # template_1 only used with test keys - assert results[0].template_id == template_2.id - assert results[0].count == 1 - - assert results[1].template_id == template_3.id - assert results[1].count == 1 diff --git a/tests/app/dao/test_stats_template_usage_by_month_dao.py b/tests/app/dao/test_stats_template_usage_by_month_dao.py deleted file mode 100644 index 676e00952..000000000 --- a/tests/app/dao/test_stats_template_usage_by_month_dao.py +++ /dev/null @@ -1,155 +0,0 @@ -from app import db -from app.dao.stats_template_usage_by_month_dao import ( - insert_or_update_stats_for_template, - dao_get_template_usage_stats_by_service -) -from app.models import StatsTemplateUsageByMonth, LETTER_TYPE, PRECOMPILED_TEMPLATE_NAME - -from tests.app.db import create_service, create_template - - -def test_create_stats_for_template(notify_db_session, sample_template): - assert StatsTemplateUsageByMonth.query.count() == 0 - - insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10) - stats_by_month = StatsTemplateUsageByMonth.query.filter( - StatsTemplateUsageByMonth.template_id == sample_template.id - ).all() - - assert len(stats_by_month) == 1 - assert stats_by_month[0].template_id == sample_template.id - assert stats_by_month[0].month == 1 - assert stats_by_month[0].year == 2017 - assert stats_by_month[0].count == 10 - - -def test_update_stats_for_template(notify_db_session, sample_template): - assert StatsTemplateUsageByMonth.query.count() == 0 - - insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10) - insert_or_update_stats_for_template(sample_template.id, 1, 2017, 20) - insert_or_update_stats_for_template(sample_template.id, 2, 2017, 30) - - stats_by_month = StatsTemplateUsageByMonth.query.filter( - StatsTemplateUsageByMonth.template_id == sample_template.id - ).order_by(StatsTemplateUsageByMonth.template_id).all() - - assert len(stats_by_month) == 2 - - assert stats_by_month[0].template_id == sample_template.id - assert stats_by_month[0].month == 1 - assert stats_by_month[0].year == 2017 - assert stats_by_month[0].count == 20 - - assert stats_by_month[1].template_id == sample_template.id - assert stats_by_month[1].month == 2 - assert stats_by_month[1].year == 2017 - assert stats_by_month[1].count == 30 - - -def test_dao_get_template_usage_stats_by_service(sample_service): - - email_template = create_template(service=sample_service, template_type="email") - - new_service = create_service(service_name="service_one") - - template_new_service = create_template(service=new_service) - - db.session.add(StatsTemplateUsageByMonth( - template_id=email_template.id, - month=4, - year=2017, - count=10 - )) - - db.session.add(StatsTemplateUsageByMonth( - template_id=template_new_service.id, - month=4, - year=2017, - count=10 - )) - - result = dao_get_template_usage_stats_by_service(sample_service.id, 2017) - - assert len(result) == 1 - - -def test_dao_get_template_usage_stats_by_service_for_precompiled_letters(sample_service): - - letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) - - precompiled_letter_template = create_template( - service=sample_service, template_name=PRECOMPILED_TEMPLATE_NAME, hidden=True, template_type=LETTER_TYPE) - - db.session.add(StatsTemplateUsageByMonth( - template_id=letter_template.id, - month=5, - year=2017, - count=10 - )) - - db.session.add(StatsTemplateUsageByMonth( - template_id=precompiled_letter_template.id, - month=4, - year=2017, - count=20 - )) - - result = dao_get_template_usage_stats_by_service(sample_service.id, 2017) - - assert len(result) == 2 - assert [ - (letter_template.id, 'letter Template Name', 'letter', False, 5, 2017, 10), - (precompiled_letter_template.id, PRECOMPILED_TEMPLATE_NAME, 'letter', True, 4, 2017, 20) - ] == result - - -def test_dao_get_template_usage_stats_by_service_specific_year(sample_service): - - email_template = create_template(service=sample_service, template_type="email") - - db.session.add(StatsTemplateUsageByMonth( - template_id=email_template.id, - month=3, - year=2017, - count=10 - )) - - db.session.add(StatsTemplateUsageByMonth( - template_id=email_template.id, - month=4, - year=2017, - count=10 - )) - - db.session.add(StatsTemplateUsageByMonth( - template_id=email_template.id, - month=3, - year=2018, - count=10 - )) - - db.session.add(StatsTemplateUsageByMonth( - template_id=email_template.id, - month=4, - year=2018, - count=10 - )) - - result = dao_get_template_usage_stats_by_service(sample_service.id, 2017) - - assert len(result) == 2 - - assert result[0].template_id == email_template.id - assert result[0].name == email_template.name - assert result[0].template_type == email_template.template_type - assert result[0].month == 4 - assert result[0].year == 2017 - assert result[0].count == 10 - - assert result[1].template_id == email_template.id - assert result[1].name == email_template.name - assert result[1].template_type == email_template.template_type - assert result[1].month == 3 - assert result[1].year == 2018 - assert result[1].count == 10 diff --git a/tests/app/dao/test_templates_dao.py b/tests/app/dao/test_templates_dao.py index b3d27a6d6..f585e2b5d 100644 --- a/tests/app/dao/test_templates_dao.py +++ b/tests/app/dao/test_templates_dao.py @@ -1,6 +1,7 @@ from datetime import datetime from freezegun import freeze_time +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm.exc import NoResultFound import pytest @@ -10,7 +11,6 @@ from app.dao.templates_dao import ( dao_get_all_templates_for_service, dao_update_template, dao_get_template_versions, - dao_get_multiple_template_details, dao_redact_template, dao_update_template_reply_to ) from app.models import ( @@ -152,7 +152,7 @@ def test_dao_update_template_reply_to_none_to_some(sample_service, sample_user): assert template_history.updated_at == updated.updated_at -def test_dao_update_tempalte_reply_to_some_to_some(sample_service, sample_user): +def test_dao_update_template_reply_to_some_to_some(sample_service, sample_user): letter_contact = create_letter_contact(sample_service, 'Edinburgh, ED1 1AA') letter_contact_2 = create_letter_contact(sample_service, 'London, N1 1DE') @@ -510,16 +510,35 @@ def test_get_template_versions_is_empty_for_hidden_templates(notify_db, notify_d assert len(versions) == 0 -def test_get_multiple_template_details_returns_templates_for_list_of_ids(sample_service): - t1 = create_template(sample_service) - t2 = create_template(sample_service) - create_template(sample_service) # t3 +@pytest.mark.parametrize("template_type,postage", [('letter', 'third'), ('sms', 'second')]) +def test_template_postage_constraint_on_create(sample_service, sample_user, template_type, postage): + data = { + 'name': 'Sample Template', + 'template_type': template_type, + 'content': "Template content", + 'service': sample_service, + 'created_by': sample_user, + 'postage': postage + } + template = Template(**data) + with pytest.raises(expected_exception=SQLAlchemyError): + dao_create_template(template) - res = dao_get_multiple_template_details([t1.id, t2.id]) - assert {x.id for x in res} == {t1.id, t2.id} - # make sure correct properties are on each row - assert res[0].id - assert res[0].template_type - assert res[0].name - assert not res[0].is_precompiled_letter +def test_template_postage_constraint_on_update(sample_service, sample_user): + data = { + 'name': 'Sample Template', + 'template_type': "letter", + 'content': "Template content", + 'service': sample_service, + 'created_by': sample_user, + 'postage': 'second' + } + template = Template(**data) + dao_create_template(template) + created = dao_get_all_templates_for_service(sample_service.id)[0] + assert created.name == 'Sample Template' + + created.postage = 'third' + with pytest.raises(expected_exception=SQLAlchemyError): + dao_update_template(created) diff --git a/tests/app/db.py b/tests/app/db.py index 3990a24cf..5c4889057 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -17,7 +17,7 @@ from app.dao.service_data_retention_dao import insert_service_data_retention from app.dao.service_inbound_api_dao import save_service_inbound_api from app.dao.service_permissions_dao import dao_add_service_permission from app.dao.service_sms_sender_dao import update_existing_sms_sender_with_inbound_number, dao_update_service_sms_sender -from app.dao.services_dao import dao_create_service +from app.dao.services_dao import dao_create_service, dao_add_user_to_service from app.dao.templates_dao import dao_create_template, dao_update_template from app.dao.users_dao import save_model_user from app.models import ( @@ -81,21 +81,29 @@ def create_service( prefix_sms=True, message_limit=1000, organisation_type='central', + postage='second', + check_if_service_exists=False ): - service = Service( - name=service_name, - message_limit=message_limit, - restricted=restricted, - email_from=email_from if email_from else service_name.lower().replace(' ', '.'), - created_by=user or create_user(email='{}@digital.cabinet-office.gov.uk'.format(uuid.uuid4())), - prefix_sms=prefix_sms, - organisation_type=organisation_type, - ) + if check_if_service_exists: + service = Service.query.filter_by(name=service_name).first() + if (not check_if_service_exists) or (check_if_service_exists and not service): + service = Service( + name=service_name, + message_limit=message_limit, + restricted=restricted, + email_from=email_from if email_from else service_name.lower().replace(' ', '.'), + created_by=user if user else create_user(email='{}@digital.cabinet-office.gov.uk'.format(uuid.uuid4())), + prefix_sms=prefix_sms, + organisation_type=organisation_type, + postage=postage + ) + dao_create_service(service, service.created_by, service_id, service_permissions=service_permissions) - dao_create_service(service, service.created_by, service_id, service_permissions=service_permissions) - - service.active = active - service.research_mode = research_mode + service.active = active + service.research_mode = research_mode + else: + if user and user not in service.users: + dao_add_user_to_service(service, user) return service @@ -140,6 +148,7 @@ def create_template( hidden=False, archived=False, folder=None, + postage=None, ): data = { 'name': template_name or '{} Template Name'.format(template_type), @@ -150,6 +159,7 @@ def create_template( 'reply_to': reply_to, 'hidden': hidden, 'folder': folder, + 'postage': postage, } if template_type != SMS_TYPE: data['subject'] = subject @@ -164,7 +174,7 @@ def create_template( def create_notification( - template, + template=None, job=None, job_row_number=None, to_field=None, @@ -190,6 +200,10 @@ def create_notification( created_by_id=None, postage=None ): + assert job or template + if job: + template = job.template + if created_at is None: created_at = datetime.utcnow() @@ -261,7 +275,8 @@ def create_job( job_status='pending', scheduled_for=None, processing_started=None, - original_file_name='some.csv' + original_file_name='some.csv', + archived=False ): data = { 'id': uuid.uuid4(), @@ -275,7 +290,8 @@ def create_job( 'created_by': template.created_by, 'job_status': job_status, 'scheduled_for': scheduled_for, - 'processing_started': processing_started + 'processing_started': processing_started, + 'archived': archived } job = Job(**data) dao_create_job(job) @@ -555,6 +571,8 @@ def create_ft_notification_status( notification_status='delivered', count=1 ): + if job: + template = job.template if template: service = template.service notification_type = template.template_type diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index dcf0cf5f8..03d4a289e 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -1,21 +1,18 @@ import json import uuid -from datetime import datetime, timedelta -from functools import partial +from datetime import datetime, timedelta, date from freezegun import freeze_time import pytest import pytz + import app.celery.tasks +from app.dao.templates_dao import dao_update_template +from app.models import JOB_STATUS_TYPES, JOB_STATUS_PENDING from tests import create_authorization_header from tests.conftest import set_config -from tests.app.conftest import ( - sample_job as create_job, - sample_notification as create_notification -) -from app.dao.templates_dao import dao_update_template -from app.models import NOTIFICATION_STATUS_TYPES, JOB_STATUS_TYPES, JOB_STATUS_PENDING +from tests.app.db import create_ft_notification_status, create_job, create_notification def test_get_job_with_invalid_service_id_returns404(client, sample_service): @@ -444,54 +441,26 @@ def test_create_job_returns_400_if_archived_template(client, sample_template, mo assert 'Template has been deleted' in resp_json['message']['template'] -def _setup_jobs(notify_db, notify_db_session, template, number_of_jobs=5): +def _setup_jobs(template, number_of_jobs=5): for i in range(number_of_jobs): - create_job( - notify_db, - notify_db_session, - service=template.service, - template=template) + create_job(template=template) -def test_get_all_notifications_for_job_in_order_of_job_number( - client, notify_db, notify_db_session, sample_service -): - main_job = create_job(notify_db, notify_db_session, service=sample_service) - another_job = create_job(notify_db, notify_db_session, service=sample_service) +def test_get_all_notifications_for_job_in_order_of_job_number(admin_request, sample_template): + main_job = create_job(sample_template) + another_job = create_job(sample_template) - notification_1 = create_notification( - notify_db, - notify_db_session, - job=main_job, - to_field="1", - created_at=datetime.utcnow(), - job_row_number=1 + notification_1 = create_notification(job=main_job, to_field="1", job_row_number=1) + notification_2 = create_notification(job=main_job, to_field="2", job_row_number=2) + notification_3 = create_notification(job=main_job, to_field="3", job_row_number=3) + create_notification(job=another_job) + + resp = admin_request.get( + 'job.get_all_notifications_for_service_job', + service_id=main_job.service_id, + job_id=main_job.id ) - notification_2 = create_notification( - notify_db, - notify_db_session, - job=main_job, - to_field="2", - created_at=datetime.utcnow(), - job_row_number=2 - ) - notification_3 = create_notification( - notify_db, - notify_db_session, - job=main_job, - to_field="3", - created_at=datetime.utcnow(), - job_row_number=3 - ) - create_notification(notify_db, notify_db_session, job=another_job) - auth_header = create_authorization_header() - - response = client.get( - path='/service/{}/job/{}/notifications'.format(sample_service.id, main_job.id), - headers=[auth_header]) - - resp = json.loads(response.get_data(as_text=True)) assert len(resp['notifications']) == 3 assert resp['notifications'][0]['to'] == notification_1.to assert resp['notifications'][0]['job_row_number'] == notification_1.job_row_number @@ -499,133 +468,76 @@ def test_get_all_notifications_for_job_in_order_of_job_number( assert resp['notifications'][1]['job_row_number'] == notification_2.job_row_number assert resp['notifications'][2]['to'] == notification_3.to assert resp['notifications'][2]['job_row_number'] == notification_3.job_row_number - assert response.status_code == 200 @pytest.mark.parametrize( "expected_notification_count, status_args", [ - (1, '?status={}'.format(NOTIFICATION_STATUS_TYPES[0])), - (0, '?status={}'.format(NOTIFICATION_STATUS_TYPES[1])), - (1, '?status={}&status={}&status={}'.format(*NOTIFICATION_STATUS_TYPES[0:3])), - (0, '?status={}&status={}&status={}'.format(*NOTIFICATION_STATUS_TYPES[3:6])), + (1, ['created']), + (0, ['sending']), + (1, ['created', 'sending']), + (0, ['sending', 'delivered']), ] ) def test_get_all_notifications_for_job_filtered_by_status( - client, - notify_db, - notify_db_session, - sample_service, + admin_request, + sample_job, expected_notification_count, status_args ): - job = create_job(notify_db, notify_db_session, service=sample_service) + create_notification(job=sample_job, to_field="1", status='created') - create_notification( - notify_db, - notify_db_session, - job=job, - to_field="1", - created_at=datetime.utcnow(), - status=NOTIFICATION_STATUS_TYPES[0], - job_row_number=1 + resp = admin_request.get( + 'job.get_all_notifications_for_service_job', + service_id=sample_job.service_id, + job_id=sample_job.id, + status=status_args ) - - response = client.get( - path='/service/{}/job/{}/notifications{}'.format(sample_service.id, job.id, status_args), - headers=[create_authorization_header()] - ) - resp = json.loads(response.get_data(as_text=True)) assert len(resp['notifications']) == expected_notification_count - assert response.status_code == 200 def test_get_all_notifications_for_job_returns_correct_format( - client, + admin_request, sample_notification_with_job ): service_id = sample_notification_with_job.service_id job_id = sample_notification_with_job.job_id - response = client.get( - path='/service/{}/job/{}/notifications'.format(service_id, job_id), - headers=[create_authorization_header()] - ) - assert response.status_code == 200 - resp = json.loads(response.get_data(as_text=True)) + + resp = admin_request.get('job.get_all_notifications_for_service_job', service_id=service_id, job_id=job_id) + assert len(resp['notifications']) == 1 assert resp['notifications'][0]['id'] == str(sample_notification_with_job.id) assert resp['notifications'][0]['status'] == sample_notification_with_job.status -def test_get_job_by_id(notify_api, sample_job): +def test_get_job_by_id(admin_request, sample_job): job_id = str(sample_job.id) service_id = sample_job.service.id - with notify_api.test_request_context(): - with notify_api.test_client() as client: - path = '/service/{}/job/{}'.format(service_id, job_id) - auth_header = create_authorization_header() - response = client.get(path, headers=[auth_header]) - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) - assert resp_json['data']['id'] == job_id - assert resp_json['data']['statistics'] == [] - assert resp_json['data']['created_by']['name'] == 'Test User' + resp_json = admin_request.get('job.get_job_by_service_and_job_id', service_id=service_id, job_id=job_id) -def test_get_job_by_id_should_return_statistics(client, notify_db, notify_db_session, notify_api, sample_job): - job_id = str(sample_job.id) - service_id = sample_job.service.id - partial_notification = partial( - create_notification, notify_db, notify_db_session, service=sample_job.service, job=sample_job - ) - partial_notification(status='created') - partial_notification(status='sending') - partial_notification(status='delivered') - partial_notification(status='pending') - partial_notification(status='failed') - partial_notification(status='technical-failure') # noqa - partial_notification(status='temporary-failure') # noqa - partial_notification(status='permanent-failure') # noqa - - path = '/service/{}/job/{}'.format(service_id, job_id) - auth_header = create_authorization_header() - response = client.get(path, headers=[auth_header]) - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) assert resp_json['data']['id'] == job_id - assert {'status': 'created', 'count': 1} in resp_json['data']['statistics'] - assert {'status': 'sending', 'count': 1} in resp_json['data']['statistics'] - assert {'status': 'delivered', 'count': 1} in resp_json['data']['statistics'] - assert {'status': 'pending', 'count': 1} in resp_json['data']['statistics'] - assert {'status': 'failed', 'count': 1} in resp_json['data']['statistics'] - assert {'status': 'technical-failure', 'count': 1} in resp_json['data']['statistics'] - assert {'status': 'temporary-failure', 'count': 1} in resp_json['data']['statistics'] - assert {'status': 'permanent-failure', 'count': 1} in resp_json['data']['statistics'] + assert resp_json['data']['statistics'] == [] assert resp_json['data']['created_by']['name'] == 'Test User' -def test_get_job_by_id_should_return_summed_statistics(client, notify_db, notify_db_session, notify_api, sample_job): +def test_get_job_by_id_should_return_summed_statistics(admin_request, sample_job): job_id = str(sample_job.id) service_id = sample_job.service.id - partial_notification = partial( - create_notification, notify_db, notify_db_session, service=sample_job.service, job=sample_job - ) - partial_notification(status='created') - partial_notification(status='created') - partial_notification(status='created') - partial_notification(status='sending') - partial_notification(status='failed') - partial_notification(status='failed') - partial_notification(status='failed') - partial_notification(status='technical-failure') - partial_notification(status='temporary-failure') - partial_notification(status='temporary-failure') - path = '/service/{}/job/{}'.format(service_id, job_id) - auth_header = create_authorization_header() - response = client.get(path, headers=[auth_header]) - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) + create_notification(job=sample_job, status='created') + create_notification(job=sample_job, status='created') + create_notification(job=sample_job, status='created') + create_notification(job=sample_job, status='sending') + create_notification(job=sample_job, status='failed') + create_notification(job=sample_job, status='failed') + create_notification(job=sample_job, status='failed') + create_notification(job=sample_job, status='technical-failure') + create_notification(job=sample_job, status='temporary-failure') + create_notification(job=sample_job, status='temporary-failure') + + resp_json = admin_request.get('job.get_job_by_service_and_job_id', service_id=service_id, job_id=job_id) + assert resp_json['data']['id'] == job_id assert {'status': 'created', 'count': 3} in resp_json['data']['statistics'] assert {'status': 'sending', 'count': 1} in resp_json['data']['statistics'] @@ -635,32 +547,23 @@ def test_get_job_by_id_should_return_summed_statistics(client, notify_db, notify assert resp_json['data']['created_by']['name'] == 'Test User' -def test_get_jobs(client, notify_db, notify_db_session, sample_template): - _setup_jobs(notify_db, notify_db_session, sample_template) +def test_get_jobs(admin_request, sample_template): + _setup_jobs(sample_template) service_id = sample_template.service.id - path = '/service/{}/job'.format(service_id) - auth_header = create_authorization_header() - response = client.get(path, headers=[auth_header]) - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) + resp_json = admin_request.get('job.get_jobs_by_service', service_id=service_id) assert len(resp_json['data']) == 5 -def test_get_jobs_with_limit_days(admin_request, notify_db, notify_db_session, sample_template): +def test_get_jobs_with_limit_days(admin_request, sample_template): for time in [ 'Sunday 1st July 2018 22:59', 'Sunday 2nd July 2018 23:00', # beginning of monday morning 'Monday 3rd July 2018 12:00' ]: with freeze_time(time): - create_job( - notify_db, - notify_db_session, - service=sample_template.service, - template=sample_template, - ) + create_job(template=sample_template) with freeze_time('Monday 9th July 2018 12:00'): resp_json = admin_request.get('job.get_jobs_by_service', service_id=sample_template.service_id, limit_days=7) @@ -668,51 +571,35 @@ def test_get_jobs_with_limit_days(admin_request, notify_db, notify_db_session, s assert len(resp_json['data']) == 2 -def test_get_jobs_should_return_statistics(client, notify_db, notify_db_session, notify_api, sample_service): +def test_get_jobs_should_return_statistics(admin_request, sample_template): now = datetime.utcnow() earlier = datetime.utcnow() - timedelta(days=1) - job_1 = create_job(notify_db, notify_db_session, service=sample_service, created_at=earlier) - job_2 = create_job(notify_db, notify_db_session, service=sample_service, created_at=now) - partial_notification = partial(create_notification, notify_db, notify_db_session, service=sample_service) - partial_notification(job=job_1, status='created') - partial_notification(job=job_1, status='created') - partial_notification(job=job_1, status='created') - partial_notification(job=job_2, status='sending') - partial_notification(job=job_2, status='sending') - partial_notification(job=job_2, status='sending') + job_1 = create_job(sample_template, processing_started=earlier) + job_2 = create_job(sample_template, processing_started=now) + create_notification(job=job_1, status='created') + create_notification(job=job_1, status='created') + create_notification(job=job_1, status='created') + create_notification(job=job_2, status='sending') + create_notification(job=job_2, status='sending') + create_notification(job=job_2, status='sending') - with notify_api.test_request_context(): - with notify_api.test_client() as client: - path = '/service/{}/job'.format(sample_service.id) - auth_header = create_authorization_header() - response = client.get(path, headers=[auth_header]) - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) - assert len(resp_json['data']) == 2 - assert resp_json['data'][0]['id'] == str(job_2.id) - assert {'status': 'sending', 'count': 3} in resp_json['data'][0]['statistics'] - assert resp_json['data'][1]['id'] == str(job_1.id) - assert {'status': 'created', 'count': 3} in resp_json['data'][1]['statistics'] + resp_json = admin_request.get('job.get_jobs_by_service', service_id=sample_template.service_id) + + assert len(resp_json['data']) == 2 + assert resp_json['data'][0]['id'] == str(job_2.id) + assert {'status': 'sending', 'count': 3} in resp_json['data'][0]['statistics'] + assert resp_json['data'][1]['id'] == str(job_1.id) + assert {'status': 'created', 'count': 3} in resp_json['data'][1]['statistics'] -def test_get_jobs_should_return_no_stats_if_no_rows_in_notifications( - client, - notify_db, - notify_db_session, - notify_api, - sample_service, -): - +def test_get_jobs_should_return_no_stats_if_no_rows_in_notifications(admin_request, sample_template): now = datetime.utcnow() earlier = datetime.utcnow() - timedelta(days=1) - job_1 = create_job(notify_db, notify_db_session, service=sample_service, created_at=earlier) - job_2 = create_job(notify_db, notify_db_session, service=sample_service, created_at=now) + job_1 = create_job(sample_template, created_at=earlier) + job_2 = create_job(sample_template, created_at=now) + + resp_json = admin_request.get('job.get_jobs_by_service', service_id=sample_template.service_id) - path = '/service/{}/job'.format(sample_service.id) - auth_header = create_authorization_header() - response = client.get(path, headers=[auth_header]) - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) assert len(resp_json['data']) == 2 assert resp_json['data'][0]['id'] == str(job_2.id) assert resp_json['data'][0]['statistics'] == [] @@ -720,23 +607,12 @@ def test_get_jobs_should_return_no_stats_if_no_rows_in_notifications( assert resp_json['data'][1]['statistics'] == [] -def test_get_jobs_should_paginate( - notify_db, - notify_db_session, - client, - sample_template -): - create_10_jobs(notify_db, notify_db_session, sample_template.service, sample_template) +def test_get_jobs_should_paginate(admin_request, sample_template): + create_10_jobs(sample_template) - path = '/service/{}/job'.format(sample_template.service_id) - auth_header = create_authorization_header() + with set_config(admin_request.app, 'PAGE_SIZE', 2): + resp_json = admin_request.get('job.get_jobs_by_service', service_id=sample_template.service_id) - with set_config(client.application, 'PAGE_SIZE', 2): - response = client.get(path, headers=[auth_header]) - - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) - assert len(resp_json['data']) == 2 assert resp_json['data'][0]['created_at'] == '2015-01-01T10:00:00+00:00' assert resp_json['data'][1]['created_at'] == '2015-01-01T09:00:00+00:00' assert resp_json['page_size'] == 2 @@ -745,23 +621,12 @@ def test_get_jobs_should_paginate( assert set(resp_json['links'].keys()) == {'next', 'last'} -def test_get_jobs_accepts_page_parameter( - notify_db, - notify_db_session, - client, - sample_template -): - create_10_jobs(notify_db, notify_db_session, sample_template.service, sample_template) +def test_get_jobs_accepts_page_parameter(admin_request, sample_template): + create_10_jobs(sample_template) - path = '/service/{}/job'.format(sample_template.service_id) - auth_header = create_authorization_header() + with set_config(admin_request.app, 'PAGE_SIZE', 2): + resp_json = admin_request.get('job.get_jobs_by_service', service_id=sample_template.service_id, page=2) - with set_config(client.application, 'PAGE_SIZE', 2): - response = client.get(path, headers=[auth_header], query_string={'page': 2}) - - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) - assert len(resp_json['data']) == 2 assert resp_json['data'][0]['created_at'] == '2015-01-01T08:00:00+00:00' assert resp_json['data'][1]['created_at'] == '2015-01-01T07:00:00+00:00' assert resp_json['page_size'] == 2 @@ -778,71 +643,85 @@ def test_get_jobs_accepts_page_parameter( # bad statuses are accepted, just return no data ('foo', []) ]) -def test_get_jobs_can_filter_on_statuses( - notify_db, - notify_db_session, - client, - sample_service, - statuses_filter, - expected_statuses -): - create_job(notify_db, notify_db_session, job_status='pending') - create_job(notify_db, notify_db_session, job_status='in progress') - create_job(notify_db, notify_db_session, job_status='finished') - create_job(notify_db, notify_db_session, job_status='sending limits exceeded') - create_job(notify_db, notify_db_session, job_status='scheduled') - create_job(notify_db, notify_db_session, job_status='cancelled') - create_job(notify_db, notify_db_session, job_status='ready to send') - create_job(notify_db, notify_db_session, job_status='sent to dvla') - create_job(notify_db, notify_db_session, job_status='error') +def test_get_jobs_can_filter_on_statuses(admin_request, sample_template, statuses_filter, expected_statuses): + create_job(sample_template, job_status='pending') + create_job(sample_template, job_status='in progress') + create_job(sample_template, job_status='finished') + create_job(sample_template, job_status='sending limits exceeded') + create_job(sample_template, job_status='scheduled') + create_job(sample_template, job_status='cancelled') + create_job(sample_template, job_status='ready to send') + create_job(sample_template, job_status='sent to dvla') + create_job(sample_template, job_status='error') - path = '/service/{}/job'.format(sample_service.id) - response = client.get( - path, - headers=[create_authorization_header()], - query_string={'statuses': statuses_filter} + resp_json = admin_request.get( + 'job.get_jobs_by_service', + service_id=sample_template.service_id, + statuses=statuses_filter ) - assert response.status_code == 200 - resp_json = json.loads(response.get_data(as_text=True)) - from pprint import pprint - pprint(resp_json) assert {x['job_status'] for x in resp_json['data']} == set(expected_statuses) -def create_10_jobs(db, session, service, template): +def create_10_jobs(template): with freeze_time('2015-01-01T00:00:00') as the_time: for _ in range(10): the_time.tick(timedelta(hours=1)) - create_job(db, session, service, template) + create_job(template) -def test_get_all_notifications_for_job_returns_csv_format( - client, - notify_db, - notify_db_session, -): - job = create_job(notify_db, notify_db_session) - notification = create_notification( - notify_db, - notify_db_session, - job=job, - job_row_number=1, - created_at=datetime.utcnow(), +def test_get_all_notifications_for_job_returns_csv_format(admin_request, sample_notification_with_job): + resp = admin_request.get( + 'job.get_all_notifications_for_service_job', + service_id=sample_notification_with_job.service_id, + job_id=sample_notification_with_job.job_id, + format_for_csv=True ) - path = '/service/{}/job/{}/notifications'.format(notification.service.id, job.id) - - response = client.get( - path=path, - headers=[create_authorization_header()], - query_string={'format_for_csv': True} - ) - assert response.status_code == 200 - - resp = json.loads(response.get_data(as_text=True)) assert len(resp['notifications']) == 1 - notification = resp['notifications'][0] - assert set(notification.keys()) == \ - set(['created_at', 'created_by_name', 'template_type', - 'template_name', 'job_name', 'status', 'row_number', 'recipient']) + assert set(resp['notifications'][0].keys()) == { + 'created_at', + 'created_by_name', + 'created_by_email_address', + 'template_type', + 'template_name', + 'job_name', + 'status', + 'row_number', + 'recipient' + } + + +@freeze_time('2017-06-10 12:00') +def test_get_jobs_should_retrieve_from_ft_notification_status_for_old_jobs(admin_request, sample_template): + # it's the 10th today, so 3 days should include all of 7th, 8th, 9th, and some of 10th. + just_three_days_ago = datetime(2017, 6, 6, 22, 59, 59) + not_quite_three_days_ago = just_three_days_ago + timedelta(seconds=1) + + job_1 = create_job(sample_template, created_at=just_three_days_ago, processing_started=just_three_days_ago) + job_2 = create_job(sample_template, created_at=just_three_days_ago, processing_started=not_quite_three_days_ago) + # is old but hasn't started yet (probably a scheduled job). We don't have any stats for this job yet. + job_3 = create_job(sample_template, created_at=just_three_days_ago, processing_started=None) + + # some notifications created more than three days ago, some created after the midnight cutoff + create_ft_notification_status(date(2017, 6, 6), job=job_1, notification_status='delivered', count=2) + create_ft_notification_status(date(2017, 6, 7), job=job_1, notification_status='delivered', count=4) + # job2's new enough + create_notification(job=job_2, status='created', created_at=not_quite_three_days_ago) + + # this isn't picked up because the job is too new + create_ft_notification_status(date(2017, 6, 7), job=job_2, notification_status='delivered', count=8) + # this isn't picked up - while the job is old, it started in last 3 days so we look at notification table instead + create_ft_notification_status(date(2017, 6, 7), job=job_3, notification_status='delivered', count=16) + + # this isn't picked up because we're using the ft status table for job_1 as it's old + create_notification(job=job_1, status='created', created_at=not_quite_three_days_ago) + + resp_json = admin_request.get('job.get_jobs_by_service', service_id=sample_template.service_id) + + assert resp_json['data'][0]['id'] == str(job_3.id) + assert resp_json['data'][0]['statistics'] == [] + assert resp_json['data'][1]['id'] == str(job_2.id) + assert resp_json['data'][1]['statistics'] == [{'status': 'created', 'count': 1}] + assert resp_json['data'][2]['id'] == str(job_1.id) + assert resp_json['data'][2]['statistics'] == [{'status': 'delivered', 'count': 6}] diff --git a/tests/app/letters/test_letter_utils.py b/tests/app/letters/test_letter_utils.py index 32f3f9df9..7b989a582 100644 --- a/tests/app/letters/test_letter_utils.py +++ b/tests/app/letters/test_letter_utils.py @@ -10,6 +10,7 @@ from app.letters.utils import ( get_bucket_name_and_prefix_for_notification, get_letter_pdf_filename, get_letter_pdf, + letter_print_day, upload_letter_pdf, ScanErrorType, move_failed_pdf, get_folder_name ) @@ -274,3 +275,27 @@ def test_get_folder_name_in_british_summer_time(notify_api, freeze_date, expecte def test_get_folder_name_returns_empty_string_for_test_letter(): assert '' == get_folder_name(datetime.utcnow(), is_test_or_scan_letter=True) + + +@freeze_time('2017-07-07 20:00:00') +def test_letter_print_day_returns_today_if_letter_was_printed_after_1730_yesterday(): + created_at = datetime(2017, 7, 6, 17, 30) + assert letter_print_day(created_at) == 'today' + + +@freeze_time('2017-07-07 16:30:00') +def test_letter_print_day_returns_today_if_letter_was_printed_today(): + created_at = datetime(2017, 7, 7, 12, 0) + assert letter_print_day(created_at) == 'today' + + +@pytest.mark.parametrize('created_at, formatted_date', [ + (datetime(2017, 7, 5, 16, 30), 'on 6 July'), + (datetime(2017, 7, 6, 16, 29), 'on 6 July'), + (datetime(2016, 8, 8, 10, 00), 'on 8 August'), + (datetime(2016, 12, 12, 17, 29), 'on 12 December'), + (datetime(2016, 12, 12, 17, 30), 'on 13 December'), +]) +@freeze_time('2017-07-07 16:30:00') +def test_letter_print_day_returns_formatted_date_if_letter_printed_before_1730_yesterday(created_at, formatted_date): + assert letter_print_day(created_at) == formatted_date diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py index 609a863d8..61d177176 100644 --- a/tests/app/notifications/test_process_notification.py +++ b/tests/app/notifications/test_process_notification.py @@ -1,19 +1,19 @@ import datetime import uuid -from unittest.mock import call import pytest from boto3.exceptions import Boto3Error from sqlalchemy.exc import SQLAlchemyError from freezegun import freeze_time from collections import namedtuple -from flask import current_app from app.models import ( Notification, NotificationHistory, ScheduledNotification, - Template + Template, + LETTER_TYPE, + CHOOSE_POSTAGE ) from app.notifications.process_notifications import ( create_content_for_notification, @@ -23,10 +23,11 @@ from app.notifications.process_notifications import ( simulated_recipient ) from notifications_utils.recipients import validate_and_format_phone_number, validate_and_format_email_address -from app.utils import cache_key_for_service_template_counter from app.v2.errors import BadRequestError from tests.app.conftest import sample_api_key as create_api_key +from tests.app.db import create_service, create_template + def test_create_content_for_notification_passes(sample_email_template): template = Template.query.get(sample_email_template.id) @@ -168,8 +169,6 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker) assert Notification.query.count() == 0 assert NotificationHistory.query.count() == 0 mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.get') - mock_service_template_cache = mocker.patch( - 'app.notifications.process_notifications.redis_store.get_all_from_hash') n_id = uuid.uuid4() created_at = datetime.datetime(2016, 11, 11, 16, 8, 18) persist_notification( @@ -196,7 +195,6 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker) assert persisted_notification.job_row_number == 10 assert persisted_notification.created_at == created_at mocked_redis.assert_called_once_with(str(sample_job.service_id) + "-2016-01-01-count") - mock_service_template_cache.assert_called_once_with(cache_key_for_service_template_counter(sample_job.service_id)) assert persisted_notification.client_reference == "ref from client" assert persisted_notification.reference is None assert persisted_notification.international is False @@ -209,7 +207,6 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker) @freeze_time("2016-01-01 11:09:00.061258") def test_persist_notification_doesnt_touch_cache_for_old_keys_that_dont_exist(sample_template, sample_api_key, mocker): mock_incr = mocker.patch('app.notifications.process_notifications.redis_store.incr') - mock_incr_hash_value = mocker.patch('app.notifications.process_notifications.redis_store.increment_hash_value') mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=None) mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash', return_value=None) @@ -225,16 +222,11 @@ def test_persist_notification_doesnt_touch_cache_for_old_keys_that_dont_exist(sa reference="ref" ) mock_incr.assert_not_called() - mock_incr_hash_value.assert_called_once_with( - "service-{}-template-usage-2016-01-01".format(sample_template.service_id), - sample_template.id - ) @freeze_time("2016-01-01 11:09:00.061258") def test_persist_notification_increments_cache_if_key_exists(sample_template, sample_api_key, mocker): mock_incr = mocker.patch('app.notifications.process_notifications.redis_store.incr') - mock_incr_hash_value = mocker.patch('app.notifications.process_notifications.redis_store.increment_hash_value') mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=1) mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash', return_value={sample_template.id, 1}) @@ -251,10 +243,6 @@ def test_persist_notification_increments_cache_if_key_exists(sample_template, sa reference="ref2") mock_incr.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count", ) - assert mock_incr_hash_value.mock_calls == [ - call("{}-template-counter-limit-7-days".format(sample_template.service_id), sample_template.id), - call("service-{}-template-usage-2016-01-01".format(sample_template.service_id), sample_template.id), - ] @pytest.mark.parametrize(( @@ -477,42 +465,40 @@ def test_persist_email_notification_stores_normalised_email( assert persisted_notification.normalised_to == expected_recipient_normalised -@pytest.mark.parametrize('utc_time, day_in_key', [ - ('2016-01-01 23:00:00', '2016-01-01'), - ('2016-06-01 22:59:00', '2016-06-01'), - ('2016-06-01 23:00:00', '2016-06-02'), -]) -def test_persist_notification_increments_and_expires_redis_template_usage( - utc_time, - day_in_key, - sample_template, - sample_api_key, - mocker +@pytest.mark.parametrize( + "service_permissions, template_postage, expected_postage", + [ + ([LETTER_TYPE], "first", "second"), + ([LETTER_TYPE, CHOOSE_POSTAGE], "first", "first"), + ([LETTER_TYPE, CHOOSE_POSTAGE], None, "second"), + ] +) +def test_persist_letter_notification_finds_correct_postage( + mocker, + notify_db, + notify_db_session, + service_permissions, + template_postage, + expected_postage ): - mock_incr_hash_value = mocker.patch('app.notifications.process_notifications.redis_store.increment_hash_value') - mock_expire = mocker.patch('app.notifications.process_notifications.redis_store.expire') - mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=None) - mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash', return_value=None) + service = create_service(service_permissions=service_permissions, postage="second") + api_key = create_api_key(notify_db, notify_db_session, service=service) + template = create_template(service, template_type=LETTER_TYPE, postage=template_postage) + mocker.patch('app.dao.templates_dao.dao_get_template_by_id', return_value=template) + persist_notification( + template_id=template.id, + template_version=template.version, + template_postage=template.postage, + recipient="Jane Doe, 10 Downing Street, London", + service=service, + personalisation=None, + notification_type=LETTER_TYPE, + api_key_id=api_key.id, + key_type=api_key.key_type, + ) + persisted_notification = Notification.query.all()[0] - with freeze_time(utc_time): - persist_notification( - template_id=sample_template.id, - template_version=sample_template.version, - recipient='+447111111122', - service=sample_template.service, - personalisation={}, - notification_type='sms', - api_key_id=sample_api_key.id, - key_type=sample_api_key.key_type, - ) - mock_incr_hash_value.assert_called_once_with( - 'service-{}-template-usage-{}'.format(str(sample_template.service_id), day_in_key), - sample_template.id - ) - mock_expire.assert_called_once_with( - 'service-{}-template-usage-{}'.format(str(sample_template.service_id), day_in_key), - current_app.config['EXPIRE_CACHE_EIGHT_DAYS'] - ) + assert persisted_notification.postage == expected_postage def test_persist_notification_with_billable_units_stores_correct_info( diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 0c9051d90..a035500cb 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -153,7 +153,7 @@ def test_get_all_notifications(client, sample_notification): assert notifications['notifications'][0]['to'] == '+447700900855' assert notifications['notifications'][0]['service'] == str(sample_notification.service_id) - assert notifications['notifications'][0]['body'] == "This is a template:\nwith a newline" + assert notifications['notifications'][0]['body'] == 'Dear Sir/Madam, Hello. Yours Truly, The Government.' def test_normal_api_key_returns_notifications_created_from_jobs_and_from_api( diff --git a/tests/app/platform_stats/test_rest.py b/tests/app/platform_stats/test_rest.py index 5afa7fa43..91474171e 100644 --- a/tests/app/platform_stats/test_rest.py +++ b/tests/app/platform_stats/test_rest.py @@ -2,11 +2,14 @@ from datetime import date, datetime from freezegun import freeze_time +from app.models import SMS_TYPE, EMAIL_TYPE +from tests.app.db import create_service, create_template, create_ft_notification_status, create_notification + @freeze_time('2018-06-01') def test_get_platform_stats_uses_todays_date_if_no_start_or_end_date_is_provided(admin_request, mocker): today = datetime.now().date() - dao_mock = mocker.patch('app.platform_stats.rest.fetch_aggregate_stats_by_date_range_for_all_services') + dao_mock = mocker.patch('app.platform_stats.rest.fetch_notification_status_totals_for_all_services') mocker.patch('app.service.rest.statistics.format_statistics') admin_request.get('platform_stats.get_platform_stats') @@ -17,7 +20,7 @@ def test_get_platform_stats_uses_todays_date_if_no_start_or_end_date_is_provided def test_get_platform_stats_can_filter_by_date(admin_request, mocker): start_date = date(2017, 1, 1) end_date = date(2018, 1, 1) - dao_mock = mocker.patch('app.platform_stats.rest.fetch_aggregate_stats_by_date_range_for_all_services') + dao_mock = mocker.patch('app.platform_stats.rest.fetch_notification_status_totals_for_all_services') mocker.patch('app.service.rest.statistics.format_statistics') admin_request.get('platform_stats.get_platform_stats', start_date=start_date, end_date=end_date) @@ -35,3 +38,37 @@ def test_get_platform_stats_validates_the_date(admin_request): assert response['errors'][0]['message'] == 'start_date time data {} does not match format %Y-%m-%d'.format( start_date) + + +@freeze_time('2018-10-31 14:00') +def test_get_platform_stats_with_real_query(admin_request, notify_db_session): + service_1 = create_service(service_name='service_1') + sms_template = create_template(service=service_1, template_type=SMS_TYPE) + email_template = create_template(service=service_1, template_type=EMAIL_TYPE) + create_ft_notification_status(date(2018, 10, 29), 'sms', service_1, count=10) + create_ft_notification_status(date(2018, 10, 29), 'email', service_1, count=3) + + create_notification(sms_template, created_at=datetime(2018, 10, 31, 11, 0, 0), key_type='test') + create_notification(sms_template, created_at=datetime(2018, 10, 31, 12, 0, 0), status='delivered') + create_notification(email_template, created_at=datetime(2018, 10, 31, 13, 0, 0), status='delivered') + + response = admin_request.get( + 'platform_stats.get_platform_stats', start_date=date(2018, 10, 29), + ) + assert response == { + 'email': { + 'failures': { + 'virus-scan-failed': 0, 'temporary-failure': 0, 'permanent-failure': 0, 'technical-failure': 0}, + 'total': 4, 'test-key': 0 + }, + 'letter': { + 'failures': { + 'virus-scan-failed': 0, 'temporary-failure': 0, 'permanent-failure': 0, 'technical-failure': 0}, + 'total': 0, 'test-key': 0 + }, + 'sms': { + 'failures': { + 'virus-scan-failed': 0, 'temporary-failure': 0, 'permanent-failure': 0, 'technical-failure': 0}, + 'total': 11, 'test-key': 1 + } + } diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 13b0cb422..931149480 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1407,6 +1407,28 @@ def test_get_only_api_created_notifications_for_service( assert resp['notifications'][0]['id'] == str(without_job.id) +def test_get_notifications_for_service_without_page_count( + admin_request, + sample_job, + sample_template, + sample_user, +): + create_notification(sample_template) + without_job = create_notification(sample_template) + + resp = admin_request.get( + 'service.get_all_notifications_for_service', + service_id=sample_template.service_id, + page_size=1, + include_jobs=False, + include_one_off=False, + count_pages=False + ) + assert len(resp['notifications']) == 1 + assert resp['total'] is None + assert resp['notifications'][0]['id'] == str(without_job.id) + + @pytest.mark.parametrize('should_prefix', [ True, False, @@ -1642,31 +1664,37 @@ def test_get_detailed_services_only_includes_todays_notifications(notify_db, not } -@pytest.mark.parametrize( - 'set_time', - ['2017-03-28T12:00:00', '2017-01-28T12:00:00', '2017-01-02T12:00:00', '2017-10-31T12:00:00'] -) -def test_get_detailed_services_for_date_range(notify_db, notify_db_session, set_time): +@pytest.mark.parametrize("start_date_delta, end_date_delta", + [(2, 1), + (3, 2), + (1, 0) + ]) +@freeze_time('2017-03-28T12:00:00') +def test_get_detailed_services_for_date_range(sample_template, start_date_delta, end_date_delta): from app.service.rest import get_detailed_services - with freeze_time(set_time): - create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow() - timedelta(days=3)) - create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow() - timedelta(days=2)) - create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow() - timedelta(days=1)) - create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow()) + create_ft_notification_status(bst_date=(datetime.utcnow() - timedelta(days=3)).date(), + service=sample_template.service, + notification_type='sms') + create_ft_notification_status(bst_date=(datetime.utcnow() - timedelta(days=2)).date(), + service=sample_template.service, + notification_type='sms') + create_ft_notification_status(bst_date=(datetime.utcnow() - timedelta(days=1)).date(), + service=sample_template.service, + notification_type='sms') - start_date = (datetime.utcnow() - timedelta(days=2)).date() - end_date = (datetime.utcnow() - timedelta(days=1)).date() + create_notification(template=sample_template, created_at=datetime.utcnow(), status='delivered') + + start_date = (datetime.utcnow() - timedelta(days=start_date_delta)).date() + end_date = (datetime.utcnow() - timedelta(days=end_date_delta)).date() data = get_detailed_services(only_active=False, include_from_test_key=True, start_date=start_date, end_date=end_date) assert len(data) == 1 - assert data[0]['statistics'] == { - EMAIL_TYPE: {'delivered': 0, 'failed': 0, 'requested': 0}, - SMS_TYPE: {'delivered': 0, 'failed': 0, 'requested': 2}, - LETTER_TYPE: {'delivered': 0, 'failed': 0, 'requested': 0} - } + assert data[0]['statistics'][EMAIL_TYPE] == {'delivered': 0, 'failed': 0, 'requested': 0} + assert data[0]['statistics'][SMS_TYPE] == {'delivered': 2, 'failed': 0, 'requested': 2} + assert data[0]['statistics'][LETTER_TYPE] == {'delivered': 0, 'failed': 0, 'requested': 0} def test_search_for_notification_by_to_field(client, sample_template, sample_email_template): @@ -2790,3 +2818,114 @@ def test_get_organisation_for_service_id_return_empty_dict_if_service_not_in_org service_id=fake_uuid ) assert response == {} + + +def test_cancel_notification_for_service_raises_invalid_request_when_notification_is_not_found( + admin_request, + sample_service, + fake_uuid, +): + response = admin_request.post( + 'service.cancel_notification_for_service', + service_id=sample_service.id, + notification_id=fake_uuid, + _expected_status=404 + ) + assert response['message'] == 'Notification not found' + assert response['result'] == 'error' + + +def test_cancel_notification_for_service_raises_invalid_request_when_notification_is_not_a_letter( + admin_request, + sample_notification, +): + response = admin_request.post( + 'service.cancel_notification_for_service', + service_id=sample_notification.service_id, + notification_id=sample_notification.id, + _expected_status=400 + ) + assert response['message'] == 'Notification cannot be cancelled - only letters can be cancelled' + assert response['result'] == 'error' + + +@pytest.mark.parametrize('notification_status', [ + 'cancelled', + 'sending', + 'sent', + 'delivered', + 'pending', + 'failed', + 'technical-failure', + 'temporary-failure', + 'permanent-failure', + 'validation-failed', + 'virus-scan-failed', + 'returned-letter', +]) +@freeze_time('2018-07-07 12:00:00') +def test_cancel_notification_for_service_raises_invalid_request_when_letter_is_in_wrong_state_to_be_cancelled( + admin_request, + sample_letter_notification, + notification_status, +): + sample_letter_notification.status = notification_status + + response = admin_request.post( + 'service.cancel_notification_for_service', + service_id=sample_letter_notification.service_id, + notification_id=sample_letter_notification.id, + _expected_status=400 + ) + assert response['message'] == 'It’s too late to cancel this letter. Printing started today at 5.30pm' + assert response['result'] == 'error' + + +@pytest.mark.parametrize('notification_status', ['created', 'pending-virus-check']) +@freeze_time('2018-07-07 16:00:00') +def test_cancel_notification_for_service_updates_letter_if_letter_is_in_cancellable_state( + admin_request, + sample_letter_notification, + notification_status, +): + sample_letter_notification.status = notification_status + sample_letter_notification.created_at = datetime.now() + + response = admin_request.post( + 'service.cancel_notification_for_service', + service_id=sample_letter_notification.service_id, + notification_id=sample_letter_notification.id, + ) + assert response['status'] == 'cancelled' + + +@freeze_time('2017-12-12 17:30:00') +def test_cancel_notification_for_service_raises_error_if_its_too_late_to_cancel( + admin_request, + sample_letter_notification, +): + sample_letter_notification.created_at = datetime(2017, 12, 11, 17, 0) + + response = admin_request.post( + 'service.cancel_notification_for_service', + service_id=sample_letter_notification.service_id, + notification_id=sample_letter_notification.id, + _expected_status=400 + ) + assert response['message'] == 'It’s too late to cancel this letter. Printing started on 11 December at 5.30pm' + assert response['result'] == 'error' + + +@freeze_time('2018-7-7 16:00:00') +def test_cancel_notification_for_service_updates_letter_if_still_time_to_cancel( + admin_request, + sample_letter_notification, +): + sample_letter_notification.created_at = datetime(2018, 7, 7, 10, 0) + + response = admin_request.post( + 'service.cancel_notification_for_service', + service_id=sample_letter_notification.service_id, + notification_id=sample_letter_notification.id, + ) + assert response['status'] == 'cancelled' diff --git a/tests/app/service/test_send_one_off_notification.py b/tests/app/service/test_send_one_off_notification.py index f8ffdf28a..b70459fc8 100644 --- a/tests/app/service/test_send_one_off_notification.py +++ b/tests/app/service/test_send_one_off_notification.py @@ -10,7 +10,9 @@ from app.config import QueueNames from app.dao.service_whitelist_dao import dao_add_and_commit_whitelisted_contacts from app.service.send_notification import send_one_off_notification from app.models import ( + EMAIL_TYPE, KEY_TYPE_NORMAL, + LETTER_TYPE, MOBILE_TYPE, PRIORITY, SMS_TYPE, @@ -64,13 +66,17 @@ def test_send_one_off_notification_calls_celery_correctly(persist_mock, celery_m ) -def test_send_one_off_notification_calls_persist_correctly( +def test_send_one_off_notification_calls_persist_correctly_for_sms( persist_mock, celery_mock, notify_db_session ): service = create_service() - template = create_template(service=service, content="Hello (( Name))\nYour thing is due soon") + template = create_template( + service=service, + template_type=SMS_TYPE, + content="Hello (( Name))\nYour thing is due soon", + ) post_data = { 'template_id': str(template.id), @@ -84,6 +90,7 @@ def test_send_one_off_notification_calls_persist_correctly( persist_mock.assert_called_once_with( template_id=template.id, template_version=template.version, + template_postage=None, recipient=post_data['to'], service=template.service, personalisation={'name': 'foo'}, @@ -91,7 +98,95 @@ def test_send_one_off_notification_calls_persist_correctly( api_key_id=None, key_type=KEY_TYPE_NORMAL, created_by_id=str(service.created_by_id), - reply_to_text='testing' + reply_to_text='testing', + reference=None, + ) + + +def test_send_one_off_notification_calls_persist_correctly_for_email( + persist_mock, + celery_mock, + notify_db_session +): + service = create_service() + template = create_template( + service=service, + template_type=EMAIL_TYPE, + subject="Test subject", + content="Hello (( Name))\nYour thing is due soon", + ) + + post_data = { + 'template_id': str(template.id), + 'to': 'test@example.com', + 'personalisation': {'name': 'foo'}, + 'created_by': str(service.created_by_id) + } + + send_one_off_notification(service.id, post_data) + + persist_mock.assert_called_once_with( + template_id=template.id, + template_version=template.version, + template_postage=None, + recipient=post_data['to'], + service=template.service, + personalisation={'name': 'foo'}, + notification_type=EMAIL_TYPE, + api_key_id=None, + key_type=KEY_TYPE_NORMAL, + created_by_id=str(service.created_by_id), + reply_to_text=None, + reference=None, + ) + + +def test_send_one_off_notification_calls_persist_correctly_for_letter( + mocker, + persist_mock, + celery_mock, + notify_db_session +): + mocker.patch( + 'app.service.send_notification.create_random_identifier', + return_value='this-is-random-in-real-life', + ) + service = create_service() + template = create_template( + service=service, + template_type=LETTER_TYPE, + postage='first', + subject="Test subject", + content="Hello (( Name))\nYour thing is due soon", + ) + + post_data = { + 'template_id': str(template.id), + 'to': 'First Last', + 'personalisation': { + 'name': 'foo', + 'address line 1': 'First Last', + 'address line 2': '1 Example Street', + 'postcode': 'SW1A 1AA', + }, + 'created_by': str(service.created_by_id) + } + + send_one_off_notification(service.id, post_data) + + persist_mock.assert_called_once_with( + template_id=template.id, + template_version=template.version, + template_postage='first', + recipient=post_data['to'], + service=template.service, + personalisation=post_data['personalisation'], + notification_type=LETTER_TYPE, + api_key_id=None, + key_type=KEY_TYPE_NORMAL, + created_by_id=str(service.created_by_id), + reply_to_text=None, + reference='this-is-random-in-real-life', ) diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index 715553359..07a5d5c09 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -38,12 +38,17 @@ NewStatsRow = collections.namedtuple('row', ('notification_type', 'status', 'key StatsRow('letter', 'validation-failed', 1), StatsRow('letter', 'virus-scan-failed', 1), StatsRow('letter', 'permanent-failure', 1), + StatsRow('letter', 'cancelled', 1), ], [4, 0, 4], [0, 0, 0], [3, 0, 3]), 'convert_sent_to_delivered': ([ StatsRow('sms', 'sending', 1), StatsRow('sms', 'delivered', 1), StatsRow('sms', 'sent', 1), ], [0, 0, 0], [3, 2, 0], [0, 0, 0]), + 'handles_none_rows': ([ + StatsRow('sms', 'sending', 1), + StatsRow(None, None, None) + ], [0, 0, 0], [1, 0, 0], [0, 0, 0]) }) def test_format_statistics(stats, email_counts, sms_counts, letter_counts): diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 5b9719637..519612de5 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -4,7 +4,6 @@ from datetime import datetime, date import pytest from freezegun import freeze_time -from app.celery.scheduled_tasks import daily_stats_template_usage_by_month from app.models import ( EMAIL_TYPE, SMS_TYPE, @@ -28,13 +27,7 @@ def test_get_template_usage_by_month_returns_correct_data( admin_request, sample_template ): - create_notification(sample_template, created_at=datetime(2016, 4, 1), status='created') - create_notification(sample_template, created_at=datetime(2017, 4, 1), status='sending') - create_notification(sample_template, created_at=datetime(2017, 4, 1), status='permanent-failure') - create_notification(sample_template, created_at=datetime(2017, 4, 1), status='temporary-failure') - - daily_stats_template_usage_by_month() - + create_ft_notification_status(bst_date=date(2017, 4, 2), template=sample_template, count=3) create_notification(sample_template, created_at=datetime.utcnow()) resp_json = admin_request.get( @@ -61,22 +54,6 @@ def test_get_template_usage_by_month_returns_correct_data( assert resp_json[1]["count"] == 1 -@freeze_time('2017-11-11 02:00') -def test_get_template_usage_by_month_returns_no_data(admin_request, sample_template): - create_notification(sample_template, created_at=datetime(2016, 4, 1), status='created') - - daily_stats_template_usage_by_month() - - create_notification(sample_template, created_at=datetime.utcnow()) - - resp_json = admin_request.get( - 'service.get_monthly_template_usage', - service_id=sample_template.service_id, - year=2015 - ) - assert resp_json['stats'] == [] - - @freeze_time('2017-11-11 02:00') def test_get_template_usage_by_month_returns_two_templates(admin_request, sample_template, sample_service): template_one = create_template( @@ -85,14 +62,8 @@ def test_get_template_usage_by_month_returns_two_templates(admin_request, sample template_name=PRECOMPILED_TEMPLATE_NAME, hidden=True ) - - create_notification(template_one, created_at=datetime(2017, 4, 1), status='created') - create_notification(sample_template, created_at=datetime(2017, 4, 1), status='sending') - create_notification(sample_template, created_at=datetime(2017, 4, 1), status='permanent-failure') - create_notification(sample_template, created_at=datetime(2017, 4, 1), status='temporary-failure') - - daily_stats_template_usage_by_month() - + create_ft_notification_status(bst_date=datetime(2017, 4, 1), template=template_one, count=1) + create_ft_notification_status(bst_date=datetime(2017, 4, 1), template=sample_template, count=3) create_notification(sample_template, created_at=datetime.utcnow()) resp_json = admin_request.get( diff --git a/tests/app/template/test_rest.py b/tests/app/template/test_rest.py index 7cf46aa80..64c51becc 100644 --- a/tests/app/template/test_rest.py +++ b/tests/app/template/test_rest.py @@ -18,7 +18,8 @@ from app.models import ( LETTER_TYPE, SMS_TYPE, Template, - TemplateHistory + TemplateHistory, + CHOOSE_POSTAGE ) from app.dao.templates_dao import dao_get_template_by_id, dao_redact_template @@ -43,7 +44,7 @@ from tests.conftest import set_config_values def test_should_create_a_new_template_for_a_service( client, sample_user, template_type, subject ): - service = create_service(service_permissions=[template_type]) + service = create_service(service_permissions=[template_type, CHOOSE_POSTAGE]) data = { 'name': 'my template', 'template_type': template_type, @@ -53,6 +54,8 @@ def test_should_create_a_new_template_for_a_service( } if subject: data.update({'subject': subject}) + if template_type == LETTER_TYPE: + data.update({'postage': 'first'}) data = json.dumps(data) auth_header = create_authorization_header() @@ -76,6 +79,11 @@ def test_should_create_a_new_template_for_a_service( else: assert not json_resp['data']['subject'] + if template_type == LETTER_TYPE: + assert json_resp['data']['postage'] == 'first' + else: + assert not json_resp['data']['postage'] + template = Template.query.get(json_resp['data']['id']) from app.schemas import template_schema assert sorted(json_resp['data']) == sorted(template_schema.dump(template).data) @@ -210,6 +218,34 @@ def test_should_raise_error_on_create_if_no_permission( assert json_resp['message'] == expected_error +def test_should_raise_error_on_create_if_no_choose_postage_permission(client, sample_user): + service = create_service(service_permissions=[LETTER_TYPE]) + data = { + 'name': 'my template', + 'template_type': LETTER_TYPE, + 'content': 'template content', + 'service': str(service.id), + 'created_by': str(sample_user.id), + 'subject': "Some letter", + 'postage': 'first', + } + + data = json.dumps(data) + auth_header = create_authorization_header() + + response = client.post( + '/service/{}/template'.format(service.id), + headers=[('Content-Type', 'application/json'), auth_header], + data=data + ) + json_resp = json.loads(response.get_data(as_text=True)) + assert response.status_code == 403 + assert json_resp['result'] == 'error' + assert json_resp['message'] == { + "template_postage": ["Setting postage on templates is not enabled for this service."] + } + + @pytest.mark.parametrize('template_factory, expected_error', [ (sample_template_without_sms_permission, {'template_type': ['Updating text message templates is not allowed']}), (sample_template_without_email_permission, {'template_type': ['Updating email templates is not allowed']}), @@ -239,6 +275,33 @@ def test_should_be_error_on_update_if_no_permission( assert json_resp['message'] == expected_error +def test_should_be_error_on_update_if_no_choose_postage_permission(client, sample_user): + service = create_service(service_name='some_service', service_permissions=[LETTER_TYPE]) + template = create_template(service, template_type=LETTER_TYPE) + data = { + 'content': 'new template content', + 'created_by': str(sample_user.id), + 'postage': 'first' + } + + data = json.dumps(data) + auth_header = create_authorization_header() + + update_response = client.post( + '/service/{}/template/{}'.format( + template.service_id, template.id), + headers=[('Content-Type', 'application/json'), auth_header], + data=data + ) + + json_resp = json.loads(update_response.get_data(as_text=True)) + assert update_response.status_code == 403 + assert json_resp['result'] == 'error' + assert json_resp['message'] == { + "template_postage": ["Setting postage on templates is not enabled for this service."] + } + + def test_should_error_if_created_by_missing(client, sample_user, sample_service): service_id = str(sample_service.id) data = { @@ -301,16 +364,19 @@ def test_must_have_a_subject_on_an_email_or_letter_template(client, sample_user, assert json_resp['errors'][0]["message"] == 'subject is a required property' -def test_update_should_update_a_template(client, sample_user, sample_template): +def test_update_should_update_a_template(client, sample_user): + service = create_service(service_permissions=[LETTER_TYPE, CHOOSE_POSTAGE]) + template = create_template(service, template_type="letter", postage="second") data = { - 'content': 'my template has new content ', - 'created_by': str(sample_user.id) + 'content': 'my template has new content, swell!', + 'created_by': str(sample_user.id), + 'postage': 'first' } data = json.dumps(data) auth_header = create_authorization_header() update_response = client.post( - '/service/{}/template/{}'.format(sample_template.service_id, sample_template.id), + '/service/{}/template/{}'.format(service.id, template.id), headers=[('Content-Type', 'application/json'), auth_header], data=data ) @@ -318,10 +384,11 @@ def test_update_should_update_a_template(client, sample_user, sample_template): assert update_response.status_code == 200 update_json_resp = json.loads(update_response.get_data(as_text=True)) assert update_json_resp['data']['content'] == ( - 'my template has new content ' + 'my template has new content, swell!' ) - assert update_json_resp['data']['name'] == sample_template.name - assert update_json_resp['data']['template_type'] == sample_template.template_type + assert update_json_resp['data']['postage'] == 'first' + assert update_json_resp['data']['name'] == template.name + assert update_json_resp['data']['template_type'] == template.template_type assert update_json_resp['data']['version'] == 2 diff --git a/tests/app/template_folder/test_template_folder_rest.py b/tests/app/template_folder/test_template_folder_rest.py index bc9058aca..ff861f0de 100644 --- a/tests/app/template_folder/test_template_folder_rest.py +++ b/tests/app/template_folder/test_template_folder_rest.py @@ -325,9 +325,7 @@ def test_move_to_folder_rejects_if_it_would_cause_folder_loop(admin_request, sam }, _expected_status=400 ) - assert response['message'] == 'Could not move to folder: {} is an ancestor of target folder {}'.format( - f1.id, target_folder.id - ) + assert response['message'] == 'You cannot move a folder to one of its subfolders' def test_move_to_folder_itself_is_rejected(admin_request, sample_service): @@ -343,7 +341,7 @@ def test_move_to_folder_itself_is_rejected(admin_request, sample_service): }, _expected_status=400 ) - assert response['message'] == 'Could not move folder to itself' + assert response['message'] == 'You cannot move a folder to itself' def test_move_to_folder_skips_archived_templates(admin_request, sample_service): diff --git a/tests/app/template_statistics/test_rest.py b/tests/app/template_statistics/test_rest.py index b9559e134..45659a712 100644 --- a/tests/app/template_statistics/test_rest.py +++ b/tests/app/template_statistics/test_rest.py @@ -1,15 +1,10 @@ import uuid -from datetime import datetime -from unittest.mock import Mock, call, ANY +from unittest.mock import Mock import pytest -from flask import current_app from freezegun import freeze_time -from tests.app.db import ( - create_notification, - create_template, -) +from tests.app.db import create_notification def set_up_get_all_from_hash(mock_redis, side_effect): @@ -59,7 +54,7 @@ def test_get_template_statistics_for_service_by_day_returns_template_info(admin_ assert json_resp['data'][0]['count'] == 1 assert json_resp['data'][0]['template_id'] == str(sample_notification.template_id) - assert json_resp['data'][0]['template_name'] == 'Template Name' + assert json_resp['data'][0]['template_name'] == 'sms Template Name' assert json_resp['data'][0]['template_type'] == 'sms' assert json_resp['data'][0]['is_precompiled_letter'] is False @@ -80,169 +75,46 @@ def test_get_template_statistics_for_service_by_day_accepts_old_query_string( assert len(json_resp['data']) == 1 -@freeze_time('2018-01-01 12:00:00') -def test_get_template_statistics_for_service_by_day_gets_out_of_redis_if_available( - admin_request, - mocker, - sample_template -): - mock_redis = mocker.patch('app.template_statistics.rest.redis_store') - set_up_get_all_from_hash(mock_redis, [ - {sample_template.id: 3} - ]) - - json_resp = admin_request.get( - 'template_statistics.get_template_statistics_for_service_by_day', - service_id=sample_template.service_id, - whole_days=0 - ) - - assert len(json_resp['data']) == 1 - assert json_resp['data'][0]['count'] == 3 - assert json_resp['data'][0]['template_id'] == str(sample_template.id) - mock_redis.get_all_from_hash.assert_called_once_with( - 'service-{}-template-usage-{}'.format(sample_template.service_id, '2018-01-01') - ) - - @freeze_time('2018-01-02 12:00:00') -def test_get_template_statistics_for_service_by_day_goes_to_db_if_not_in_redis( +def test_get_template_statistics_for_service_by_day_goes_to_db( admin_request, mocker, sample_template ): - mock_redis = mocker.patch('app.template_statistics.rest.redis_store') # first time it is called redis returns data, second time returns none - set_up_get_all_from_hash(mock_redis, [ - {sample_template.id: 2}, - None - ]) mock_dao = mocker.patch( - 'app.template_statistics.rest.dao_get_template_usage', + 'app.template_statistics.rest.fetch_notification_status_for_service_for_today_and_7_previous_days', return_value=[ - Mock(id=sample_template.id, count=3) + Mock( + template_id=sample_template.id, + count=3, + template_name=sample_template.name, + notification_type=sample_template.template_type, + status='created', + is_precompiled_letter=False + ) ] ) - json_resp = admin_request.get( 'template_statistics.get_template_statistics_for_service_by_day', service_id=sample_template.service_id, whole_days=1 ) - assert len(json_resp['data']) == 1 - assert json_resp['data'][0]['count'] == 5 - assert json_resp['data'][0]['template_id'] == str(sample_template.id) - # first redis call - assert mock_redis.get_all_from_hash.mock_calls == [ - call('service-{}-template-usage-{}'.format(sample_template.service_id, '2018-01-01')), - call('service-{}-template-usage-{}'.format(sample_template.service_id, '2018-01-02')) - ] + assert json_resp['data'] == [{ + "template_id": str(sample_template.id), + "count": 3, + "template_name": sample_template.name, + "template_type": sample_template.template_type, + "status": "created", + "is_precompiled_letter": False + + }] # dao only called for 2nd, since redis returned values for first call mock_dao.assert_called_once_with( - str(sample_template.service_id), day=datetime(2018, 1, 2) + str(sample_template.service_id), limit_days=1, by_template=True ) - mock_redis.set_hash_and_expire.assert_called_once_with( - 'service-{}-template-usage-{}'.format(sample_template.service_id, '2018-01-02'), - # sets the data that the dao returned - {str(sample_template.id): 3}, - current_app.config['EXPIRE_CACHE_EIGHT_DAYS'] - ) - - -def test_get_template_statistics_for_service_by_day_combines_templates_correctly( - admin_request, - mocker, - sample_service -): - t1 = create_template(sample_service, template_name='1') - t2 = create_template(sample_service, template_name='2') - t3 = create_template(sample_service, template_name='3') # noqa - mock_redis = mocker.patch('app.template_statistics.rest.redis_store') - - # first time it is called redis returns data, second time returns none - set_up_get_all_from_hash(mock_redis, [ - {t1.id: 2}, - None, - {t1.id: 1, t2.id: 4}, - ]) - mock_dao = mocker.patch( - 'app.template_statistics.rest.dao_get_template_usage', - return_value=[ - Mock(id=t1.id, count=8) - ] - ) - - json_resp = admin_request.get( - 'template_statistics.get_template_statistics_for_service_by_day', - service_id=sample_service.id, - whole_days=2 - ) - - assert len(json_resp['data']) == 2 - assert json_resp['data'][0]['template_id'] == str(t1.id) - assert json_resp['data'][0]['count'] == 11 - assert json_resp['data'][1]['template_id'] == str(t2.id) - assert json_resp['data'][1]['count'] == 4 - - assert mock_redis.get_all_from_hash.call_count == 3 - # dao only called for 2nd day - assert mock_dao.call_count == 1 - - -@freeze_time('2018-03-28 00:00:00') -def test_get_template_statistics_for_service_by_day_gets_stats_for_correct_days( - admin_request, - mocker, - sample_template -): - mock_redis = mocker.patch('app.template_statistics.rest.redis_store') - - # first time it is called redis returns data, second time returns none - set_up_get_all_from_hash(mock_redis, [ - {sample_template.id: 1}, # last weds - None, - {sample_template.id: 1}, - {sample_template.id: 1}, - {sample_template.id: 1}, - {sample_template.id: 1}, - None, - None, # current day - ]) - mock_dao = mocker.patch( - 'app.template_statistics.rest.dao_get_template_usage', - return_value=[ - Mock(id=sample_template.id, count=2) - ] - ) - - json_resp = admin_request.get( - 'template_statistics.get_template_statistics_for_service_by_day', - service_id=sample_template.service_id, - whole_days=7 - ) - - assert len(json_resp['data']) == 1 - assert json_resp['data'][0]['count'] == 11 - assert json_resp['data'][0]['template_id'] == str(sample_template.id) - - assert mock_redis.get_all_from_hash.call_count == 8 - - assert '2018-03-21' in mock_redis.get_all_from_hash.mock_calls[0][1][0] # last wednesday - assert '2018-03-22' in mock_redis.get_all_from_hash.mock_calls[1][1][0] - assert '2018-03-23' in mock_redis.get_all_from_hash.mock_calls[2][1][0] - assert '2018-03-24' in mock_redis.get_all_from_hash.mock_calls[3][1][0] - assert '2018-03-25' in mock_redis.get_all_from_hash.mock_calls[4][1][0] - assert '2018-03-26' in mock_redis.get_all_from_hash.mock_calls[5][1][0] - assert '2018-03-27' in mock_redis.get_all_from_hash.mock_calls[6][1][0] - assert '2018-03-28' in mock_redis.get_all_from_hash.mock_calls[7][1][0] # current day (wednesday) - - mock_dao.mock_calls == [ - call(ANY, day=datetime(2018, 3, 22)), - call(ANY, day=datetime(2018, 3, 27)), - call(ANY, day=datetime(2018, 3, 28)) - ] def test_get_template_statistics_for_service_by_day_returns_empty_list_if_no_templates( @@ -250,7 +122,6 @@ def test_get_template_statistics_for_service_by_day_returns_empty_list_if_no_tem mocker, sample_service ): - mock_redis = mocker.patch('app.template_statistics.rest.redis_store') json_resp = admin_request.get( 'template_statistics.get_template_statistics_for_service_by_day', @@ -259,9 +130,7 @@ def test_get_template_statistics_for_service_by_day_returns_empty_list_if_no_tem ) assert len(json_resp['data']) == 0 - assert mock_redis.get_all_from_hash.call_count == 8 - # make sure we don't try and set any empty hashes in redis - assert mock_redis.set_hash_and_expire.call_count == 0 + # get_template_statistics_for_template diff --git a/tests/app/test_cronitor.py b/tests/app/test_cronitor.py new file mode 100644 index 000000000..8e1aaa6b4 --- /dev/null +++ b/tests/app/test_cronitor.py @@ -0,0 +1,101 @@ +from urllib import parse + +import requests +import pytest + +from app.cronitor import cronitor + +from tests.conftest import set_config_values + + +def _cronitor_url(key, command): + return parse.urlunparse(parse.ParseResult( + scheme='https', + netloc='cronitor.link', + path='{}/{}'.format(key, command), + params='', + query=parse.urlencode({'host': 'http://localhost:6011'}), + fragment='' + )) + + +RUN_LINK = _cronitor_url('secret', 'run') +FAIL_LINK = _cronitor_url('secret', 'fail') +COMPLETE_LINK = _cronitor_url('secret', 'complete') + + +@cronitor('hello') +def successful_task(): + return 1 + + +@cronitor('hello') +def crashing_task(): + raise ValueError + + +def test_cronitor_sends_run_and_complete(notify_api, rmock): + rmock.get(RUN_LINK, status_code=200) + rmock.get(COMPLETE_LINK, status_code=200) + + with set_config_values(notify_api, { + 'CRONITOR_ENABLED': True, + 'CRONITOR_KEYS': {'hello': 'secret'} + }): + assert successful_task() == 1 + + assert rmock.call_count == 2 + assert rmock.request_history[0].url == RUN_LINK + assert rmock.request_history[1].url == COMPLETE_LINK + + +def test_cronitor_sends_run_and_fail_if_exception(notify_api, rmock): + rmock.get(RUN_LINK, status_code=200) + rmock.get(FAIL_LINK, status_code=200) + + with set_config_values(notify_api, { + 'CRONITOR_ENABLED': True, + 'CRONITOR_KEYS': {'hello': 'secret'} + }): + with pytest.raises(ValueError): + crashing_task() + + assert rmock.call_count == 2 + assert rmock.request_history[0].url == RUN_LINK + assert rmock.request_history[1].url == FAIL_LINK + + +def test_cronitor_does_nothing_if_cronitor_not_enabled(notify_api, rmock): + with set_config_values(notify_api, { + 'CRONITOR_ENABLED': False, + 'CRONITOR_KEYS': {'hello': 'secret'} + }): + assert successful_task() == 1 + + assert rmock.called is False + + +def test_cronitor_does_nothing_if_name_not_recognised(notify_api, rmock, caplog): + with set_config_values(notify_api, { + 'CRONITOR_ENABLED': True, + 'CRONITOR_KEYS': {'not-hello': 'other'} + }): + assert successful_task() == 1 + + error_log = caplog.records[0] + assert error_log.levelname == 'ERROR' + assert error_log.msg == 'Cronitor enabled but task_name hello not found in environment' + assert rmock.called is False + + +def test_cronitor_doesnt_crash_if_request_fails(notify_api, rmock): + rmock.get(RUN_LINK, exc=requests.exceptions.ConnectTimeout) + rmock.get(COMPLETE_LINK, status_code=500) + + with set_config_values(notify_api, { + 'CRONITOR_ENABLED': True, + 'CRONITOR_KEYS': {'hello': 'secret'} + }): + assert successful_task() == 1 + + assert rmock.call_count == 2 diff --git a/tests/app/test_model.py b/tests/app/test_model.py index d7e51f7cc..afea955d9 100644 --- a/tests/app/test_model.py +++ b/tests/app/test_model.py @@ -146,7 +146,7 @@ def test_notification_for_csv_returns_bst_correctly(sample_template): notification = create_notification(sample_template) serialized = notification.serialize_for_csv() - assert serialized['created_at'] == 'Monday 27 March 2017 at 00:01' + assert serialized['created_at'] == '2017-03-27 00:01:53' def test_notification_personalisation_getter_returns_empty_dict_from_None(): diff --git a/tests/app/user/test_rest.py b/tests/app/user/test_rest.py index 1db989706..32222581a 100644 --- a/tests/app/user/test_rest.py +++ b/tests/app/user/test_rest.py @@ -631,11 +631,13 @@ def test_get_orgs_and_services_nests_services(admin_request, sample_user): 'services': [ { 'name': service1.name, - 'id': str(service1.id) + 'id': str(service1.id), + 'restricted': False, }, { 'name': service2.name, - 'id': str(service2.id) + 'id': str(service2.id), + 'restricted': False, } ] }, @@ -648,7 +650,8 @@ def test_get_orgs_and_services_nests_services(admin_request, sample_user): 'services_without_organisations': [ { 'name': service3.name, - 'id': str(service3.id) + 'id': str(service3.id), + 'restricted': False, } ] } @@ -683,7 +686,8 @@ def test_get_orgs_and_services_only_returns_active(admin_request, sample_user): 'services': [ { 'name': service1.name, - 'id': str(service1.id) + 'id': str(service1.id), + 'restricted': False, } ] } @@ -691,7 +695,8 @@ def test_get_orgs_and_services_only_returns_active(admin_request, sample_user): 'services_without_organisations': [ { 'name': service4.name, - 'id': str(service4.id) + 'id': str(service4.id), + 'restricted': False, } ] } @@ -727,7 +732,8 @@ def test_get_orgs_and_services_only_shows_users_orgs_and_services(admin_request, 'services_without_organisations': [ { 'name': service1.name, - 'id': str(service1.id) + 'id': str(service1.id), + 'restricted': False, } ] } diff --git a/tests/app/v2/notifications/test_post_letter_notifications.py b/tests/app/v2/notifications/test_post_letter_notifications.py index 4b42af6dc..22232aa73 100644 --- a/tests/app/v2/notifications/test_post_letter_notifications.py +++ b/tests/app/v2/notifications/test_post_letter_notifications.py @@ -20,6 +20,7 @@ from app.models import ( NOTIFICATION_DELIVERED, NOTIFICATION_PENDING_VIRUS_CHECK, SMS_TYPE, + CHOOSE_POSTAGE ) from app.schema_validation import validate from app.v2.errors import RateLimitError @@ -95,12 +96,23 @@ def test_post_letter_notification_returns_201(client, sample_letter_template, mo mock.assert_called_once_with([str(notification.id)], queue=QueueNames.CREATE_LETTERS_PDF) -@pytest.mark.parametrize('postage', ['first', 'second']) -def test_post_letter_notification_sets_postage(client, sample_letter_template, mocker, postage): - sample_letter_template.service.postage = postage +@pytest.mark.parametrize('service_permissions, service_postage, template_postage, expected_postage', [ + ([LETTER_TYPE], "second", "first", "second"), + ([LETTER_TYPE], "first", "second", "first"), + ([LETTER_TYPE], "first", None, "first"), + ([LETTER_TYPE, CHOOSE_POSTAGE], "second", "first", "first"), + ([LETTER_TYPE, CHOOSE_POSTAGE], "second", None, "second"), + ([LETTER_TYPE, CHOOSE_POSTAGE], "second", "second", "second"), + ([LETTER_TYPE, CHOOSE_POSTAGE], "first", "second", "second"), +]) +def test_post_letter_notification_sets_postage( + client, notify_db_session, mocker, service_permissions, service_postage, template_postage, expected_postage +): + service = create_service(service_permissions=service_permissions, postage=service_postage) + template = create_template(service, template_type="letter", postage=template_postage) mocker.patch('app.celery.tasks.letters_pdf_tasks.create_letters_pdf.apply_async') data = { - 'template_id': str(sample_letter_template.id), + 'template_id': str(template.id), 'personalisation': { 'address_line_1': 'Her Royal Highness Queen Elizabeth II', 'address_line_2': 'Buckingham Palace', @@ -110,11 +122,11 @@ def test_post_letter_notification_sets_postage(client, sample_letter_template, m } } - resp_json = letter_request(client, data, service_id=sample_letter_template.service_id) + resp_json = letter_request(client, data, service_id=service.id) assert validate(resp_json, post_letter_response) == resp_json notification = Notification.query.one() - assert notification.postage == postage + assert notification.postage == expected_postage @pytest.mark.parametrize('env', [ @@ -457,16 +469,27 @@ def test_post_precompiled_letter_with_invalid_base64(client, notify_user, mocker assert not Notification.query.first() -@pytest.mark.parametrize('postage', ['first', 'second']) -def test_post_precompiled_letter_notification_returns_201(client, notify_user, mocker, postage): +@pytest.mark.parametrize('service_postage, notification_postage, expected_postage', [ + ('second', 'second', 'second'), + ('second', 'first', 'first'), + ('second', None, 'second'), + ('first', 'first', 'first'), + ('first', 'second', 'second'), + ('first', None, 'first'), +]) +def test_post_precompiled_letter_notification_returns_201( + client, notify_user, mocker, service_postage, notification_postage, expected_postage +): sample_service = create_service(service_permissions=['letter', 'precompiled_letter']) - sample_service.postage = postage + sample_service.postage = service_postage s3mock = mocker.patch('app.v2.notifications.post_notifications.upload_letter_pdf') mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task') data = { "reference": "letter-reference", "content": "bGV0dGVyLWNvbnRlbnQ=" } + if notification_postage: + data["postage"] = notification_postage auth_header = create_authorization_header(service_id=sample_service.id) response = client.post( path="v2/notifications/letter", @@ -481,10 +504,30 @@ def test_post_precompiled_letter_notification_returns_201(client, notify_user, m assert notification.billable_units == 0 assert notification.status == NOTIFICATION_PENDING_VIRUS_CHECK - assert notification.postage == postage + assert notification.postage == expected_postage notification_history = NotificationHistory.query.one() - assert notification_history.postage == postage + assert notification_history.postage == expected_postage resp_json = json.loads(response.get_data(as_text=True)) - assert resp_json == {'id': str(notification.id), 'reference': 'letter-reference'} + assert resp_json == {'id': str(notification.id), 'reference': 'letter-reference', 'postage': expected_postage} + + +def test_post_letter_notification_throws_error_for_invalid_postage(client, notify_user, mocker): + sample_service = create_service(service_permissions=['letter', 'precompiled_letter']) + data = { + "reference": "letter-reference", + "content": "bGV0dGVyLWNvbnRlbnQ=", + "postage": "space unicorn" + } + auth_header = create_authorization_header(service_id=sample_service.id) + response = client.post( + path="v2/notifications/letter", + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + assert response.status_code == 400, response.get_data(as_text=True) + resp_json = json.loads(response.get_data(as_text=True)) + assert resp_json['errors'][0]['message'] == "postage invalid. It must be either first or second." + + assert not Notification.query.first() diff --git a/tests/app/v2/template/test_get_template.py b/tests/app/v2/template/test_get_template.py index 3be2ce889..900621db7 100644 --- a/tests/app/v2/template/test_get_template.py +++ b/tests/app/v2/template/test_get_template.py @@ -41,6 +41,7 @@ def test_get_template_by_id_returns_200(client, sample_service, tmp_type, expect "subject": expected_subject, 'name': expected_name, 'personalisation': {}, + 'postage': None, } assert json_response == expected_response