diff --git a/.gitignore b/.gitignore index d19df2e94..12f77b15c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +queues.csv + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Makefile b/Makefile index 37d876d92..4142722ff 100644 --- a/Makefile +++ b/Makefile @@ -303,3 +303,8 @@ cf-rollback: ## Rollbacks the app to the previous release cf-push: $(if ${CF_APP},,$(error Must specify CF_APP)) cf push ${CF_APP} -f ${CF_MANIFEST_FILE} + +.PHONY: check-if-migrations-to-run +check-if-migrations-to-run: + @echo $(shell python3 scripts/check_if_new_migration.py) + diff --git a/app/__init__.py b/app/__init__.py index c5b509f30..f8fefbe15 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -93,6 +93,7 @@ def register_blueprint(application): from app.organisation.rest import organisation_blueprint from app.dvla_organisation.rest import dvla_organisation_blueprint from app.delivery.rest import delivery_blueprint + from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint from app.notifications.receive_notifications import receive_notifications_blueprint from app.notifications.notifications_ses_callback import ses_callback_blueprint from app.notifications.notifications_sms_callback import sms_callback_blueprint @@ -133,6 +134,9 @@ def register_blueprint(application): delivery_blueprint.before_request(requires_admin_auth) application.register_blueprint(delivery_blueprint) + inbound_sms_blueprint.before_request(requires_admin_auth) + application.register_blueprint(inbound_sms_blueprint) + accept_invite.before_request(requires_admin_auth) application.register_blueprint(accept_invite, url_prefix='/invite') diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py index e00591f09..50d5a31b7 100644 --- a/app/celery/provider_tasks.py +++ b/app/celery/provider_tasks.py @@ -3,37 +3,14 @@ from notifications_utils.recipients import InvalidEmailError from sqlalchemy.orm.exc import NoResultFound from app import notify_celery +from app.config import QueueNames from app.dao import notifications_dao from app.dao.notifications_dao import update_notification_status_by_id from app.statsd_decorators import statsd from app.delivery import send_to_providers -def retry_iteration_to_delay(retry=0): - """ - :param retry times we have performed a retry - Given current retry calculate some delay before retrying - 0: 10 seconds - 1: 60 seconds (1 minutes) - 2: 300 seconds (5 minutes) - 3: 3600 seconds (60 minutes) - 4: 14400 seconds (4 hours) - :param retry (zero indexed): - :return length to retry in seconds, default 10 seconds - """ - - delays = { - 0: 10, - 1: 60, - 2: 300, - 3: 3600, - 4: 14400 - } - - return delays.get(retry, 10) - - -@notify_celery.task(bind=True, name="deliver_sms", max_retries=5, default_retry_delay=5) +@notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300) @statsd(namespace="tasks") def deliver_sms(self, notification_id): try: @@ -46,7 +23,7 @@ def deliver_sms(self, notification_id): current_app.logger.exception( "SMS notification delivery for id: {} failed".format(notification_id) ) - self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) + self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: current_app.logger.exception( "RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id), @@ -54,7 +31,7 @@ def deliver_sms(self, notification_id): update_notification_status_by_id(notification_id, 'technical-failure') -@notify_celery.task(bind=True, name="deliver_email", max_retries=5, default_retry_delay=5) +@notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300) @statsd(namespace="tasks") def deliver_email(self, notification_id): try: @@ -70,7 +47,7 @@ def deliver_email(self, notification_id): current_app.logger.exception( "RETRY: Email notification {} failed".format(notification_id) ) - self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) + self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: current_app.logger.error( "RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e228b159f..636522f01 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -9,6 +9,7 @@ from sqlalchemy.exc import SQLAlchemyError from app.aws import s3 from app import notify_celery from app import performance_platform_client +from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by from app.dao.notifications_dao import ( @@ -26,12 +27,13 @@ from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago from app.notifications.process_notifications import send_notification_to_queue from app.statsd_decorators import statsd from app.celery.tasks import process_job +from app.config import QueueNames @notify_celery.task(name="remove_csv_files") @statsd(namespace="tasks") -def remove_csv_files(): - jobs = dao_get_jobs_older_than_limited_by() +def remove_csv_files(job_types): + jobs = dao_get_jobs_older_than_limited_by(job_types=job_types) for job in jobs: s3.remove_job_from_s3(job.service_id, job.id) current_app.logger.info("Job ID {} has been removed from s3.".format(job.id)) @@ -42,7 +44,7 @@ def remove_csv_files(): def run_scheduled_jobs(): try: for job in dao_set_scheduled_jobs_to_pending(): - process_job.apply_async([str(job.id)], queue="process-job") + process_job.apply_async([str(job.id)], queue=QueueNames.JOBS) current_app.logger.info("Job ID {} added to process job queue".format(job.id)) except SQLAlchemyError: current_app.logger.exception("Failed to run scheduled jobs") @@ -225,3 +227,21 @@ def timeout_job_statistics(): if updated: current_app.logger.info( "Timeout period reached for {} job statistics, failure count has been updated.".format(updated)) + + +@notify_celery.task(name="delete-inbound-sms") +@statsd(namespace="tasks") +def delete_inbound_sms_older_than_seven_days(): + try: + start = datetime.utcnow() + deleted = delete_inbound_sms_created_more_than_a_week_ago() + current_app.logger.info( + "Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format( + start, + datetime.utcnow(), + deleted + ) + ) + except SQLAlchemyError as e: + current_app.logger.exception("Failed to delete inbound sms notifications") + raise diff --git a/app/celery/statistics_tasks.py b/app/celery/statistics_tasks.py index a82a3791f..150fa6aac 100644 --- a/app/celery/statistics_tasks.py +++ b/app/celery/statistics_tasks.py @@ -3,7 +3,6 @@ from sqlalchemy.exc import SQLAlchemyError from app import notify_celery from flask import current_app -from app.models import JobStatistics from app.statsd_decorators import statsd from app.dao.statistics_dao import ( create_or_update_job_sending_statistics, @@ -11,16 +10,17 @@ from app.dao.statistics_dao import ( ) from app.dao.notifications_dao import get_notification_by_id from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED +from app.config import QueueNames def create_initial_notification_statistic_tasks(notification): if notification.job_id and notification.status: - record_initial_job_statistics.apply_async((str(notification.id),), queue="statistics") + record_initial_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS) def create_outcome_notification_statistic_tasks(notification): if notification.job_id and notification.status in NOTIFICATION_STATUS_TYPES_COMPLETED: - record_outcome_job_statistics.apply_async((str(notification.id),), queue="statistics") + record_outcome_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS) @notify_celery.task(bind=True, name='record_initial_job_statistics', max_retries=20, default_retry_delay=10) @@ -35,7 +35,7 @@ def record_initial_job_statistics(self, notification_id): raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id)) except SQLAlchemyError as e: current_app.logger.exception(e) - self.retry(queue="retry") + self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: current_app.logger.error( "RETRY FAILED: task record_initial_job_statistics failed for notification {}".format( @@ -53,12 +53,12 @@ def record_outcome_job_statistics(self, notification_id): if notification: updated_count = update_job_stats_outcome_count(notification) if updated_count == 0: - self.retry(queue="retry") + self.retry(queue=QueueNames.RETRY) else: raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id)) except SQLAlchemyError as e: current_app.logger.exception(e) - self.retry(queue="retry") + self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: current_app.logger.error( "RETRY FAILED: task update_job_stats_outcome_count failed for notification {}".format( diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 5cd860fe3..bb9967ee4 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -16,6 +16,7 @@ from app import ( ) from app.aws import s3 from app.celery import provider_tasks +from app.config import QueueNames from app.dao.jobs_dao import ( dao_update_job, dao_get_job_by_id, @@ -80,7 +81,7 @@ def process_job(job_id): process_row(row_number, recipient, personalisation, template, job, service) if template.template_type == LETTER_TYPE: - build_dvla_file.apply_async([str(job.id)], queue='process-job') + build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS) # temporary logging current_app.logger.info("send job {} to build-dvla-file in the process-job queue".format(job_id)) else: @@ -112,12 +113,6 @@ def process_row(row_number, recipient, personalisation, template, job, service): LETTER_TYPE: persist_letter } - queues = { - SMS_TYPE: 'db-sms', - EMAIL_TYPE: 'db-email', - LETTER_TYPE: 'db-letter', - } - send_fn = send_fns[template_type] send_fn.apply_async( @@ -127,7 +122,7 @@ def process_row(row_number, recipient, personalisation, template, job, service): encrypted, datetime.utcnow().strftime(DATETIME_FORMAT) ), - queue=queues[template_type] if not service.research_mode else 'research-mode' + queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE ) @@ -182,7 +177,7 @@ def send_sms(self, provider_tasks.deliver_sms.apply_async( [str(saved_notification.id)], - queue='send-sms' if not service.research_mode else 'research-mode' + queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE ) current_app.logger.info( @@ -227,7 +222,7 @@ def send_email(self, provider_tasks.deliver_email.apply_async( [str(saved_notification.id)], - queue='send-email' if not service.research_mode else 'research-mode' + queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE ) current_app.logger.info("Email {} created at {}".format(saved_notification.id, created_at)) @@ -285,10 +280,9 @@ def build_dvla_file(self, job_id): file_location="{}-dvla-job.text".format(job_id) ) dao_update_job_status(job_id, JOB_STATUS_READY_TO_SEND) - notify_celery.send_task("aggregrate-dvla-files", ([str(job_id)], ), queue='aggregate-dvla-files') else: current_app.logger.info("All notifications for job {} are not persisted".format(job_id)) - self.retry(queue="retry", exc="All notifications for job {} are not persisted".format(job_id)) + self.retry(queue=QueueNames.RETRY, exc="All notifications for job {} are not persisted".format(job_id)) except Exception as e: current_app.logger.exception("build_dvla_file threw exception") raise e @@ -342,7 +336,7 @@ def handle_exception(task, notification, notification_id, exc): # send to the retry queue. current_app.logger.exception('Retry' + retry_msg) try: - task.retry(queue="retry", exc=exc) + task.retry(queue=QueueNames.RETRY, exc=exc) except task.MaxRetriesExceededError: current_app.logger.exception('Retry' + retry_msg) diff --git a/app/config.py b/app/config.py index 6933ed7d4..63d1f18c3 100644 --- a/app/config.py +++ b/app/config.py @@ -2,7 +2,11 @@ from datetime import timedelta from celery.schedules import crontab from kombu import Exchange, Queue import os -from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST + +from app.models import ( + EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, + KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST +) if os.environ.get('VCAP_SERVICES'): # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate @@ -12,6 +16,34 @@ if os.environ.get('VCAP_SERVICES'): extract_cloudfoundry_config() +class QueueNames(object): + PERIODIC = 'periodic-tasks' + PRIORITY = 'priority-tasks' + DATABASE = 'database-tasks' + SEND = 'send-tasks' + RESEARCH_MODE = 'research-mode-tasks' + STATISTICS = 'statistics-tasks' + JOBS = 'job-tasks' + RETRY = 'retry-tasks' + NOTIFY = 'notify-internal-tasks' + PROCESS_FTP = 'process-ftp-tasks' + + @staticmethod + def all_queues(): + return [ + QueueNames.PRIORITY, + QueueNames.PERIODIC, + QueueNames.DATABASE, + QueueNames.SEND, + QueueNames.RESEARCH_MODE, + QueueNames.STATISTICS, + QueueNames.JOBS, + QueueNames.RETRY, + QueueNames.NOTIFY, + QueueNames.PROCESS_FTP + ] + + class Config(object): # URL of admin app ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL'] @@ -79,6 +111,7 @@ class Config(object): SMS_CHAR_COUNT_LIMIT = 495 BRANDING_PATH = '/images/email-template/crests/' TEST_MESSAGE_FILENAME = 'Test message' + ONE_OFF_MESSAGE_FILENAME = 'Report' MAX_VERIFY_CODE_COUNT = 10 NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' @@ -95,7 +128,7 @@ class Config(object): BROKER_TRANSPORT_OPTIONS = { 'region': AWS_REGION, 'polling_interval': 1, # 1 second - 'visibility_timeout': 14410, # 4 hours 10 seconds. 10 seconds longer than max retry + 'visibility_timeout': 310, 'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX } CELERY_ENABLE_UTC = True, @@ -107,7 +140,7 @@ class Config(object): 'run-scheduled-jobs': { 'task': 'run-scheduled-jobs', 'schedule': crontab(minute=1), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, # 'send-scheduled-notifications': { # 'task': 'send-scheduled-notifications', @@ -117,59 +150,67 @@ class Config(object): 'delete-verify-codes': { 'task': 'delete-verify-codes', 'schedule': timedelta(minutes=63), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, 'delete-invitations': { 'task': 'delete-invitations', 'schedule': timedelta(minutes=66), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, 'delete-sms-notifications': { 'task': 'delete-sms-notifications', 'schedule': crontab(minute=0, hour=0), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, 'delete-email-notifications': { 'task': 'delete-email-notifications', 'schedule': crontab(minute=20, hour=0), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, 'delete-letter-notifications': { 'task': 'delete-letter-notifications', 'schedule': crontab(minute=40, hour=0), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} + }, + 'delete-inbound-sms': { + 'task': 'delete-inbound-sms', + 'schedule': crontab(minute=0, hour=1), + 'options': {'queue': QueueNames.PERIODIC} }, 'send-daily-performance-platform-stats': { 'task': 'send-daily-performance-platform-stats', 'schedule': crontab(minute=0, hour=2), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, 'switch-current-sms-provider-on-slow-delivery': { 'task': 'switch-current-sms-provider-on-slow-delivery', 'schedule': crontab(), # Every minute - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', 'schedule': crontab(minute=0, hour=3), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} }, - 'remove_csv_files': { + 'remove_sms_email_jobs': { 'task': 'remove_csv_files', 'schedule': crontab(minute=0, hour=4), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC}, + 'kwargs': {'job_types': [EMAIL_TYPE, SMS_TYPE]} + }, + 'remove_letter_jobs': { + 'task': 'remove_csv_files', + 'schedule': crontab(minute=20, hour=4), + 'options': {'queue': QueueNames.PERIODIC}, + 'kwargs': {'job_types': [LETTER_TYPE]} }, 'timeout-job-statistics': { 'task': 'timeout-job-statistics', 'schedule': crontab(minute=0, hour=5), - 'options': {'queue': 'periodic'} + 'options': {'queue': QueueNames.PERIODIC} } } - CELERY_QUEUES = [ - Queue('process-job', Exchange('default'), routing_key='process-job'), - Queue('retry', Exchange('default'), routing_key='retry'), - Queue('notify', Exchange('default'), routing_key='notify') - ] + CELERY_QUEUES = [] NOTIFICATIONS_ALERT = 5 # five mins FROM_NUMBER = 'development' @@ -208,6 +249,8 @@ class Config(object): } } + FREE_SMS_TIER_FRAGMENT_COUNT = 250000 + ###################### # Config overrides ### @@ -220,17 +263,12 @@ class Development(Config): NOTIFY_ENVIRONMENT = 'development' NOTIFICATION_QUEUE_PREFIX = 'development' DEBUG = True - CELERY_QUEUES = Config.CELERY_QUEUES + [ - Queue('db-sms', Exchange('default'), routing_key='db-sms'), - Queue('priority', Exchange('default'), routing_key='priority'), - Queue('periodic', Exchange('default'), routing_key='periodic'), - Queue('db-email', Exchange('default'), routing_key='db-email'), - Queue('db-letter', Exchange('default'), routing_key='db-letter'), - Queue('send-sms', Exchange('default'), routing_key='send-sms'), - Queue('send-email', Exchange('default'), routing_key='send-email'), - Queue('research-mode', Exchange('default'), routing_key='research-mode'), - Queue('statistics', Exchange('default'), routing_key='statistics') - ] + + for queue in QueueNames.all_queues(): + Config.CELERY_QUEUES.append( + Queue(queue, Exchange('default'), routing_key=queue) + ) + API_HOST_NAME = "http://localhost:6011" API_RATE_LIMIT_ENABLED = True @@ -244,17 +282,11 @@ class Test(Config): STATSD_ENABLED = True STATSD_HOST = "localhost" STATSD_PORT = 1000 - CELERY_QUEUES = Config.CELERY_QUEUES + [ - Queue('periodic', Exchange('default'), routing_key='periodic'), - Queue('priority', Exchange('default'), routing_key='priority'), - Queue('db-sms', Exchange('default'), routing_key='db-sms'), - Queue('db-email', Exchange('default'), routing_key='db-email'), - Queue('db-letter', Exchange('default'), routing_key='db-letter'), - Queue('send-sms', Exchange('default'), routing_key='send-sms'), - Queue('send-email', Exchange('default'), routing_key='send-email'), - Queue('research-mode', Exchange('default'), routing_key='research-mode'), - Queue('statistics', Exchange('default'), routing_key='statistics') - ] + + for queue in QueueNames.all_queues(): + Config.CELERY_QUEUES.append( + Queue(queue, Exchange('default'), routing_key=queue) + ) API_RATE_LIMIT_ENABLED = True API_HOST_NAME = "http://localhost:6011" diff --git a/app/dao/inbound_sms_dao.py b/app/dao/inbound_sms_dao.py new file mode 100644 index 000000000..18060cced --- /dev/null +++ b/app/dao/inbound_sms_dao.py @@ -0,0 +1,56 @@ +from datetime import ( + timedelta, + datetime +) + + +from app import db +from app.dao.dao_utils import transactional +from app.models import InboundSms +from app.statsd_decorators import statsd + + +@transactional +def dao_create_inbound_sms(inbound_sms): + db.session.add(inbound_sms) + + +def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None): + q = InboundSms.query.filter( + InboundSms.service_id == service_id + ).order_by( + InboundSms.created_at.desc() + ) + + if user_number: + q = q.filter(InboundSms.user_number == user_number) + + if limit: + q = q.limit(limit) + + return q.all() + + +def dao_count_inbound_sms_for_service(service_id): + return InboundSms.query.filter( + InboundSms.service_id == service_id + ).count() + + +@statsd(namespace="dao") +@transactional +def delete_inbound_sms_created_more_than_a_week_ago(): + seven_days_ago = datetime.utcnow() - timedelta(days=7) + + deleted = db.session.query(InboundSms).filter( + InboundSms.created_at < seven_days_ago + ).delete(synchronize_session='fetch') + + return deleted + + +def dao_get_inbound_sms_by_id(service_id, inbound_id): + return InboundSms.query.filter_by( + id=inbound_id, + service_id=service_id + ).one() diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index b712b0e70..86e565619 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -1,17 +1,15 @@ -from datetime import datetime +from datetime import datetime, timedelta from flask import current_app from sqlalchemy import func, desc, asc, cast, Date as sql_date from app import db from app.dao import days_ago -from app.models import (Job, - Notification, - NotificationHistory, - Template, - JOB_STATUS_SCHEDULED, - JOB_STATUS_PENDING, - LETTER_TYPE, JobStatistics) +from app.models import ( + Job, JobStatistics, Notification, NotificationHistory, Template, + JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, + EMAIL_TYPE, SMS_TYPE, LETTER_TYPE +) from app.statsd_decorators import statsd @@ -53,7 +51,8 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id): def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None): query_filter = [ Job.service_id == service_id, - Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'] + Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'], + Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'], ] if limit_days is not None: query_filter.append(cast(Job.created_at, sql_date) >= days_ago(limit_days)) @@ -128,10 +127,14 @@ def dao_update_job_status(job_id, status): db.session.commit() -def dao_get_jobs_older_than_limited_by(older_than=7, limit_days=2): - return Job.query.filter( - cast(Job.created_at, sql_date) < days_ago(older_than), - cast(Job.created_at, sql_date) >= days_ago(older_than + limit_days) +def dao_get_jobs_older_than_limited_by(job_types, older_than=7, limit_days=2): + end_date = datetime.utcnow() - timedelta(days=older_than) + start_date = end_date - timedelta(days=limit_days) + + return Job.query.join(Template).filter( + Job.created_at < end_date, + Job.created_at >= start_date, + Template.template_type.in_(job_types) ).order_by(desc(Job.created_at)).all() diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 4f9e2ee8e..995462509 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta +from flask import current_app from sqlalchemy import Float, Integer from sqlalchemy import func, case, cast from sqlalchemy import literal_column @@ -11,7 +12,7 @@ from app.models import (NotificationHistory, NOTIFICATION_STATUS_TYPES_BILLABLE, KEY_TYPE_TEST, SMS_TYPE, - EMAIL_TYPE) + EMAIL_TYPE, Service) from app.statsd_decorators import statsd from app.utils import get_london_month_from_utc_column @@ -153,3 +154,78 @@ def rate_multiplier(): (NotificationHistory.rate_multiplier == None, literal_column("'1'")), # noqa (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier), # noqa ]), Integer()) + + +@statsd(namespace="dao") +def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date, end_date, service_id): + + free_sms_limit = Service.free_sms_fragment_limit() + + billable_units = 0 + total_cost = 0.0 + + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + for rate_boundary in rate_boundaries: + result = db.session.query( + func.sum( + NotificationHistory.billable_units * func.coalesce(NotificationHistory.rate_multiplier, 1) + ).label('billable_units') + ).filter( + NotificationHistory.service_id == service_id, + NotificationHistory.notification_type == 'sms', + NotificationHistory.created_at >= rate_boundary['start_date'], + NotificationHistory.created_at < rate_boundary['end_date'], + NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE) + ) + billable_units_by_rate_boundry = result.scalar() + if billable_units_by_rate_boundry: + int_billable_units_by_rate_boundry = int(billable_units_by_rate_boundry) + if billable_units >= free_sms_limit: + total_cost += int_billable_units_by_rate_boundry * rate_boundary['rate'] + elif billable_units + int_billable_units_by_rate_boundry > free_sms_limit: + remaining_free_allowance = abs(free_sms_limit - billable_units) + total_cost += ((int_billable_units_by_rate_boundry - remaining_free_allowance) * rate_boundary['rate']) + else: + total_cost += 0 + billable_units += int_billable_units_by_rate_boundry + return billable_units, total_cost + + +def discover_rate_bounds_for_billing_query(start_date, end_date): + bounds = [] + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + + def current_valid_from(index): + return rates[index].valid_from + + def next_valid_from(index): + return rates[index + 1].valid_from + + def current_rate(index): + return rates[index].rate + + def append_rate(rate_start_date, rate_end_date, rate): + bounds.append({ + 'start_date': rate_start_date, + 'end_date': rate_end_date, + 'rate': rate + }) + + if len(rates) == 1: + append_rate(start_date, end_date, current_rate(0)) + return bounds + + for i in range(len(rates)): + # first boundary + if i == 0: + append_rate(start_date, next_valid_from(i), current_rate(i)) + + # last boundary + elif i == (len(rates) - 1): + append_rate(current_valid_from(i), end_date, current_rate(i)) + + # other boundaries + else: + append_rate(current_valid_from(i), next_valid_from(i), current_rate(i)) + + return bounds diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 7ebf9170c..22da11966 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -2,14 +2,16 @@ import functools from datetime import ( datetime, timedelta, - date) + date +) from flask import current_app from notifications_utils.recipients import ( validate_and_format_phone_number, validate_and_format_email_address, - InvalidPhoneError + InvalidPhoneError, + InvalidEmailError, ) from werkzeug.datastructures import MultiDict from sqlalchemy import (desc, func, or_, and_, asc) @@ -25,6 +27,7 @@ from app.models import ( NotificationHistory, NotificationStatistics, Template, + ScheduledNotification, NOTIFICATION_CREATED, NOTIFICATION_DELIVERED, NOTIFICATION_SENDING, @@ -34,7 +37,8 @@ from app.models import ( NOTIFICATION_PERMANENT_FAILURE, KEY_TYPE_NORMAL, KEY_TYPE_TEST, LETTER_TYPE, - NOTIFICATION_SENT, ScheduledNotification) + NOTIFICATION_SENT, +) from app.dao.dao_utils import transactional from app.statsd_decorators import statsd @@ -477,7 +481,10 @@ def dao_get_notifications_by_to_field(service_id, search_term, statuses=None): try: normalised = validate_and_format_phone_number(search_term) except InvalidPhoneError: - normalised = validate_and_format_email_address(search_term) + try: + normalised = validate_and_format_email_address(search_term) + except InvalidEmailError: + normalised = search_term filters = [ Notification.service_id == service_id, diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 572e36e63..24c95567a 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -3,6 +3,7 @@ from datetime import date, datetime, timedelta from sqlalchemy import asc, func from sqlalchemy.orm import joinedload +from flask import current_app from app import db from app.dao.dao_utils import ( @@ -66,6 +67,12 @@ def dao_fetch_service_by_id(service_id, only_active=False): return query.one() +def dao_fetch_services_by_sms_sender(sms_sender): + return Service.query.filter( + Service.sms_sender == sms_sender + ).all() + + def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): query = Service.query.filter_by( id=service_id @@ -131,6 +138,12 @@ def dao_fetch_service_by_id_and_user(service_id, user_id): @transactional @version_class(Service) def dao_create_service(service, user, service_id=None, service_permissions=[SMS_TYPE, EMAIL_TYPE]): + # the default property does not appear to work when there is a difference between the sqlalchemy schema and the + # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender + # migration is completed, this code should be able to be removed. + if not service.sms_sender: + service.sms_sender = current_app.config['FROM_NUMBER'] + from app.dao.permissions_dao import permission_dao service.users.append(user) permission_dao.add_default_service_permissions_for_user(user, service) diff --git a/app/dao/statistics_dao.py b/app/dao/statistics_dao.py index baff82ba4..48d62e71c 100644 --- a/app/dao/statistics_dao.py +++ b/app/dao/statistics_dao.py @@ -60,7 +60,10 @@ def timeout_job_counts(notifications_type, timeout_start): ).update({ sent: sent_count, failed: failed_count, - delivered: delivered_count + delivered: delivered_count, + 'sent': sent_count, + 'delivered': delivered_count, + 'failed': failed_count }, synchronize_session=False) return total_updated @@ -87,11 +90,13 @@ def create_or_update_job_sending_statistics(notification): @transactional def __update_job_stats_sent_count(notification): column = columns(notification.notification_type, 'sent') + new_column = 'sent' return db.session.query(JobStatistics).filter_by( job_id=notification.job_id, ).update({ - column: column + 1 + column: column + 1, + new_column: column + 1 }) @@ -102,7 +107,8 @@ def __insert_job_stats(notification): emails_sent=1 if notification.notification_type == EMAIL_TYPE else 0, sms_sent=1 if notification.notification_type == SMS_TYPE else 0, letters_sent=1 if notification.notification_type == LETTER_TYPE else 0, - updated_at=datetime.utcnow() + updated_at=datetime.utcnow(), + sent=1 ) db.session.add(stats) @@ -131,10 +137,12 @@ def columns(notification_type, status): def update_job_stats_outcome_count(notification): if notification.status in NOTIFICATION_STATUS_TYPES_FAILED: column = columns(notification.notification_type, 'failed') + new_column = 'failed' elif notification.status in [NOTIFICATION_DELIVERED, NOTIFICATION_SENT] and notification.notification_type != LETTER_TYPE: column = columns(notification.notification_type, 'delivered') + new_column = 'delivered' else: column = None @@ -143,7 +151,8 @@ def update_job_stats_outcome_count(notification): return db.session.query(JobStatistics).filter_by( job_id=notification.job_id, ).update({ - column: column + 1 + column: column + 1, + new_column: column + 1 }) else: return 0 diff --git a/app/delivery/rest.py b/app/delivery/rest.py index 0bacb43bb..489a5fcda 100644 --- a/app/delivery/rest.py +++ b/app/delivery/rest.py @@ -1,5 +1,6 @@ from flask import Blueprint, jsonify +from app.config import QueueNames from app.delivery import send_to_providers from app.models import EMAIL_TYPE from app.celery import provider_tasks @@ -23,18 +24,16 @@ def send_notification_to_provider(notification_id): send_response( send_to_providers.send_email_to_provider, provider_tasks.deliver_email, - notification, - 'send-email') + notification) else: send_response( send_to_providers.send_sms_to_provider, provider_tasks.deliver_sms, - notification, - 'send-sms') + notification) return jsonify({}), 204 -def send_response(send_call, task_call, notification, queue): +def send_response(send_call, task_call, notification): try: send_call(notification) except Exception as e: @@ -43,4 +42,4 @@ def send_response(send_call, task_call, notification, queue): notification.id, notification.notification_type), e) - task_call.apply_async((str(notification.id)), queue=queue) + task_call.apply_async((str(notification.id)), queue=QueueNames.SEND) diff --git a/app/inbound_sms/__init__.py b/app/inbound_sms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/inbound_sms/rest.py b/app/inbound_sms/rest.py new file mode 100644 index 000000000..1eca4b089 --- /dev/null +++ b/app/inbound_sms/rest.py @@ -0,0 +1,54 @@ +from flask import ( + Blueprint, + jsonify, + request +) + +from notifications_utils.recipients import validate_and_format_phone_number + +from app.dao.inbound_sms_dao import ( + dao_get_inbound_sms_for_service, + dao_count_inbound_sms_for_service, + dao_get_inbound_sms_by_id +) +from app.errors import register_errors + +inbound_sms = Blueprint( + 'inbound_sms', + __name__, + url_prefix='/service//inbound-sms' +) + +register_errors(inbound_sms) + + +@inbound_sms.route('') +def get_inbound_sms_for_service(service_id): + limit = request.args.get('limit') + user_number = request.args.get('user_number') + + if user_number: + # we use this to normalise to an international phone number + user_number = validate_and_format_phone_number(user_number, international=True) + + results = dao_get_inbound_sms_for_service(service_id, limit, user_number) + + return jsonify(data=[row.serialize() for row in results]) + + +@inbound_sms.route('/summary') +def get_inbound_sms_summary_for_service(service_id): + count = dao_count_inbound_sms_for_service(service_id) + most_recent = dao_get_inbound_sms_for_service(service_id, limit=1) + + return jsonify( + count=count, + most_recent=most_recent[0].created_at.isoformat() if most_recent else None + ) + + +@inbound_sms.route('/', methods=['GET']) +def get_inbound_by_id(service_id, inbound_sms_id): + inbound_sms = dao_get_inbound_sms_by_id(service_id, inbound_sms_id) + + return jsonify(inbound_sms.serialize()), 200 diff --git a/app/invite/rest.py b/app/invite/rest.py index 2361629d1..8105b171f 100644 --- a/app/invite/rest.py +++ b/app/invite/rest.py @@ -4,6 +4,7 @@ from flask import ( jsonify, current_app) +from app.config import QueueNames from app.dao.invited_user_dao import ( save_invited_user, get_invited_user, @@ -44,7 +45,7 @@ def create_invited_user(service_id): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify(data=invited_user_schema.dump(invited_user).data), 201 diff --git a/app/job/rest.py b/app/job/rest.py index 8195ca87a..b4882945f 100644 --- a/app/job/rest.py +++ b/app/job/rest.py @@ -34,6 +34,8 @@ from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANC from app.utils import pagination_links +from app.config import QueueNames + job_blueprint = Blueprint('job', __name__, url_prefix='/service//job') from app.errors import ( @@ -143,7 +145,7 @@ def create_job(service_id): dao_create_job(job) if job.job_status == JOB_STATUS_PENDING: - process_job.apply_async([str(job.id)], queue="process-job") + process_job.apply_async([str(job.id)], queue=QueueNames.JOBS) job_json = job_schema.dump(job).data job_json['statistics'] = [] diff --git a/app/letters/send_letter_jobs.py b/app/letters/send_letter_jobs.py index 7030b9bc5..91c39615a 100644 --- a/app/letters/send_letter_jobs.py +++ b/app/letters/send_letter_jobs.py @@ -2,6 +2,7 @@ from flask import Blueprint, jsonify from flask import request from app import notify_celery +from app.config import QueueNames from app.dao.jobs_dao import dao_get_all_letter_jobs from app.schemas import job_schema from app.v2.errors import register_errors @@ -15,7 +16,7 @@ register_errors(letter_job) @letter_job.route('/send-letter-jobs', methods=['POST']) def send_letter_jobs(): job_ids = validate(request.get_json(), letter_job_ids) - notify_celery.send_task(name="send-files-to-dvla", args=(job_ids['job_ids'],), queue="process-ftp") + notify_celery.send_task(name="send-files-to-dvla", args=(job_ids['job_ids'],), queue=QueueNames.PROCESS_FTP) return jsonify(data={"response": "Task created to send files to DVLA"}), 201 diff --git a/app/models.py b/app/models.py index 8c1e22e79..129ce3b1f 100644 --- a/app/models.py +++ b/app/models.py @@ -146,8 +146,10 @@ class DVLAOrganisation(db.Model): INTERNATIONAL_SMS_TYPE = 'international_sms' INBOUND_SMS_TYPE = 'inbound_sms' +SCHEDULE_NOTIFICATIONS = 'schedule_notifications' -SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE] +SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE, + SCHEDULE_NOTIFICATIONS] class ServicePermissionTypes(db.Model): @@ -188,7 +190,7 @@ class Service(db.Model, Versioned): created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False) reply_to_email_address = db.Column(db.Text, index=False, unique=False, nullable=True) letter_contact_block = db.Column(db.Text, index=False, unique=False, nullable=True) - sms_sender = db.Column(db.String(11), nullable=True, default=lambda: current_app.config['FROM_NUMBER']) + sms_sender = db.Column(db.String(11), nullable=False, default=lambda: current_app.config['FROM_NUMBER']) organisation_id = db.Column(UUID(as_uuid=True), db.ForeignKey('organisation.id'), index=True, nullable=True) organisation = db.relationship('Organisation') dvla_organisation_id = db.Column( @@ -215,6 +217,10 @@ class Service(db.Model, Versioned): self.can_send_letters = LETTER_TYPE in [p.permission for p in self.permissions] self.can_send_international_sms = INTERNATIONAL_SMS_TYPE in [p.permission for p in self.permissions] + @staticmethod + def free_sms_fragment_limit(): + return current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] + @classmethod def from_json(cls, data): """ @@ -681,6 +687,8 @@ NOTIFICATION_STATUS_TYPES = [ NOTIFICATION_PERMANENT_FAILURE, ] +NOTIFICATION_STATUS_TYPES_NON_BILLABLE = list(set(NOTIFICATION_STATUS_TYPES) - set(NOTIFICATION_STATUS_TYPES_BILLABLE)) + NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notify_status_type') @@ -977,7 +985,7 @@ INVITED_USER_STATUS_TYPES = ['pending', 'accepted', 'cancelled'] class ScheduledNotification(db.Model): __tablename__ = 'scheduled_notifications' - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4()) + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) notification_id = db.Column(UUID(as_uuid=True), db.ForeignKey('notifications.id'), index=True, nullable=False) notification = db.relationship('Notification', uselist=False) scheduled_for = db.Column(db.DateTime, index=False, nullable=False) @@ -1097,6 +1105,12 @@ class Rate(db.Model): rate = db.Column(db.Float(asdecimal=False), nullable=False) notification_type = db.Column(notification_types, index=True, nullable=False) + def __str__(self): + the_string = "{}".format(self.rate) + the_string += " {}".format(self.notification_type) + the_string += " {}".format(self.valid_from) + return the_string + class JobStatistics(db.Model): __tablename__ = 'job_statistics' @@ -1112,6 +1126,9 @@ class JobStatistics(db.Model): sms_failed = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) letters_sent = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) letters_failed = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) + sent = db.Column(db.BigInteger, index=False, unique=False, nullable=True, default=0) + delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True, default=0) + failed = db.Column(db.BigInteger, index=False, unique=False, nullable=True, default=0) created_at = db.Column( db.DateTime, index=False, @@ -1141,3 +1158,56 @@ class JobStatistics(db.Model): ) the_string += "created at {}".format(self.created_at) return the_string + + +class InboundSms(db.Model): + __tablename__ = 'inbound_sms' + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) + service = db.relationship('Service', backref='inbound_sms') + + notify_number = db.Column(db.String, nullable=False) # the service's number, that the msg was sent to + user_number = db.Column(db.String, nullable=False) # the end user's number, that the msg was sent from + provider_date = db.Column(db.DateTime) + provider_reference = db.Column(db.String) + provider = db.Column(db.String, nullable=True) + _content = db.Column('content', db.String, nullable=False) + + @property + def content(self): + return encryption.decrypt(self._content) + + @content.setter + def content(self, content): + self._content = encryption.encrypt(content) + + def serialize(self): + return { + 'id': str(self.id), + 'created_at': self.created_at.isoformat(), + 'service_id': str(self.service_id), + 'notify_number': self.notify_number, + 'user_number': self.user_number, + 'content': self.content, + 'provider_date': self.provider_date and self.provider_date.isoformat(), + 'provider_reference': self.provider_reference + } + + +class LetterRate(db.Model): + __tablename__ = 'letter_rates' + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + valid_from = valid_from = db.Column(db.DateTime, nullable=False) + + +class LetterRateDetail(db.Model): + __tablename__ = 'letter_rate_details' + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + letter_rate_id = db.Column(UUID(as_uuid=True), db.ForeignKey('letter_rates.id'), index=True, nullable=False) + letter_rate = db.relationship('LetterRate', backref='letter_rates') + page_total = db.Column(db.Integer, nullable=False) + rate = db.Column(db.Numeric(), nullable=False) diff --git a/app/notifications/notifications_letter_callback.py b/app/notifications/notifications_letter_callback.py index cfcaf28df..ac2de9e5c 100644 --- a/app/notifications/notifications_letter_callback.py +++ b/app/notifications/notifications_letter_callback.py @@ -13,7 +13,7 @@ from app.celery.tasks import update_letter_notifications_statuses from app.v2.errors import register_errors from app.notifications.utils import autoconfirm_subscription from app.schema_validation import validate - +from app.config import QueueNames letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__) register_errors(letter_callback_blueprint) @@ -54,7 +54,7 @@ def process_letter_response(): filename = message['Records'][0]['s3']['object']['key'] current_app.logger.info('Received file from DVLA: {}'.format(filename)) current_app.logger.info('DVLA callback: Calling task to update letter notifications') - update_letter_notifications_statuses.apply_async([filename], queue='notify') + update_letter_notifications_statuses.apply_async([filename], queue=QueueNames.NOTIFY) return jsonify( result="success", message="DVLA callback succeeded" diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index a6d1137ce..86291b46f 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -11,10 +11,12 @@ from notifications_utils.recipients import ( from app import redis_store from app.celery import provider_tasks from notifications_utils.clients import redis + +from app.config import QueueNames +from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE, ScheduledNotification from app.dao.notifications_dao import (dao_create_notification, dao_delete_notifications_and_history_by_id, dao_created_scheduled_notification) -from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE, ScheduledNotification from app.v2.errors import BadRequestError, SendNotificationToQueueError from app.utils import get_template_instance, cache_key_for_service_template_counter, convert_bst_to_utc @@ -96,12 +98,9 @@ def persist_notification( def send_notification_to_queue(notification, research_mode, queue=None): if research_mode or notification.key_type == KEY_TYPE_TEST: - queue = 'research-mode' + queue = QueueNames.RESEARCH_MODE elif not queue: - if notification.notification_type == SMS_TYPE: - queue = 'send-sms' - if notification.notification_type == EMAIL_TYPE: - queue = 'send-email' + queue = QueueNames.SEND if notification.notification_type == SMS_TYPE: deliver_task = provider_tasks.deliver_sms diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index 08122fb41..fdd1f95ec 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -1,8 +1,15 @@ -from flask import Blueprint -from flask import current_app -from flask import request +from urllib.parse import unquote +import iso8601 +from flask import jsonify, Blueprint, current_app, request +from notifications_utils.recipients import validate_and_format_phone_number + +from app import statsd_client, firetext_client, mmg_client +from app.dao.services_dao import dao_fetch_services_by_sms_sender +from app.dao.inbound_sms_dao import dao_create_inbound_sms +from app.models import InboundSms from app.errors import register_errors +from app.utils import convert_bst_to_utc receive_notifications_blueprint = Blueprint('receive_notifications', __name__) register_errors(receive_notifications_blueprint) @@ -10,8 +17,117 @@ register_errors(receive_notifications_blueprint) @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) def receive_mmg_sms(): + """ + { + 'MSISDN': '447123456789' + 'Number': '40604', + 'Message': 'some+uri+encoded+message%3A', + 'ID': 'SOME-MMG-SPECIFIC-ID', + 'DateRecieved': '2017-05-21+11%3A56%3A11' + } + """ post_data = request.get_json() - post_data.pop('MSISDN', None) - current_app.logger.info("Recieve notification form data: {}".format(post_data)) - return "RECEIVED" + inbound_number = strip_leading_forty_four(post_data['Number']) + + potential_services = dao_fetch_services_by_sms_sender(inbound_number) + + if len(potential_services) != 1: + current_app.logger.error('Inbound number "{}" from MMG not associated with exactly one service'.format( + post_data['Number'] + )) + statsd_client.incr('inbound.mmg.failed') + # since this is an issue with our service <-> number mapping, we should still tell MMG that we received + # succesfully + return 'RECEIVED', 200 + + statsd_client.incr('inbound.mmg.successful') + + service = potential_services[0] + + inbound = create_inbound_mmg_sms_object(service, post_data) + + current_app.logger.info('{} received inbound SMS with reference {}'.format(service.id, inbound.provider_reference)) + + return 'RECEIVED', 200 + + +def format_mmg_message(message): + return unquote(message.replace('+', ' ')) + + +def format_mmg_datetime(date): + """ + We expect datetimes in format 2017-05-21+11%3A56%3A11 - ie, spaces replaced with pluses, and URI encoded + (the same as UTC) + """ + orig_date = format_mmg_message(date) + parsed_datetime = iso8601.parse_date(orig_date).replace(tzinfo=None) + return convert_bst_to_utc(parsed_datetime) + + +def create_inbound_mmg_sms_object(service, json): + message = format_mmg_message(json['Message']) + user_number = validate_and_format_phone_number(json['MSISDN'], international=True) + + provider_date = json.get('DateRecieved') + if provider_date: + provider_date = format_mmg_datetime(provider_date) + + inbound = InboundSms( + service=service, + notify_number=service.sms_sender, + user_number=user_number, + provider_date=provider_date, + provider_reference=json.get('ID'), + content=message, + provider=mmg_client.name + ) + dao_create_inbound_sms(inbound) + return inbound + + +@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) +def receive_firetext_sms(): + post_data = request.form + + inbound_number = strip_leading_forty_four(post_data['destination']) + + potential_services = dao_fetch_services_by_sms_sender(inbound_number) + if len(potential_services) != 1: + current_app.logger.error('Inbound number "{}" from firetext not associated with exactly one service'.format( + post_data['destination'] + )) + statsd_client.incr('inbound.firetext.failed') + return jsonify({ + "status": "ok" + }), 200 + + service = potential_services[0] + + user_number = validate_and_format_phone_number(post_data['source'], international=True) + message = post_data['message'] + timestamp = post_data['time'] + + dao_create_inbound_sms( + InboundSms( + service=service, + notify_number=service.sms_sender, + user_number=user_number, + provider_date=timestamp, + content=message, + provider=firetext_client.name + ) + ) + + statsd_client.incr('inbound.firetext.successful') + + return jsonify({ + "status": "ok" + }), 200 + + +def strip_leading_forty_four(number): + if number.startswith('44'): + return number.replace('44', '0', 1) + return number diff --git a/app/notifications/rest.py b/app/notifications/rest.py index faa086e32..bb136f8c8 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -6,6 +6,7 @@ from flask import ( ) from app import api_user, authenticated_service +from app.config import QueueNames from app.dao import ( templates_dao, notifications_dao @@ -134,7 +135,7 @@ def send_notification(notification_type): key_type=api_user.key_type, simulated=simulated) if not simulated: - queue_name = 'priority' if template.process_type == PRIORITY else None + queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None send_notification_to_queue(notification=notification_model, research_mode=authenticated_service.research_mode, queue=queue_name) diff --git a/app/notifications/validators.py b/app/notifications/validators.py index 1193d4015..5663a6046 100644 --- a/app/notifications/validators.py +++ b/app/notifications/validators.py @@ -6,7 +6,7 @@ from notifications_utils.recipients import ( ) from app.dao import services_dao -from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE +from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE, SCHEDULE_NOTIFICATIONS from app.service.utils import service_allowed_to_send_to from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError from app import redis_store @@ -92,6 +92,7 @@ def check_sms_content_char_count(content_count): raise BadRequestError(message=message) -def service_can_schedule_notification(service): - # TODO: implement once the service permission works. - raise BadRequestError(message="Your service must be invited to schedule notifications via the API.") +def service_can_schedule_notification(service, scheduled_for): + if scheduled_for: + if SCHEDULE_NOTIFICATIONS not in [p.permission for p in service.permissions]: + raise BadRequestError(message="Cannot schedule notifications (this feature is invite-only)") diff --git a/app/schemas.py b/app/schemas.py index ff4f83a95..7971300b7 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -25,9 +25,8 @@ from notifications_utils.recipients import ( from app import ma from app import models -from app.models import ServicePermission, INTERNATIONAL_SMS_TYPE, SMS_TYPE, LETTER_TYPE, EMAIL_TYPE +from app.models import ServicePermission, INTERNATIONAL_SMS_TYPE, LETTER_TYPE from app.dao.permissions_dao import permission_dao -from app.dao.service_permissions_dao import dao_fetch_service_permissions from app.utils import get_template_instance @@ -176,6 +175,7 @@ class ProviderDetailsHistorySchema(BaseSchema): class ServiceSchema(BaseSchema): + free_sms_fragment_limit = fields.Method(method_name='get_free_sms_fragment_limit') created_by = field_for(models.Service, 'created_by', required=True) organisation = field_for(models.Service, 'organisation') branding = field_for(models.Service, 'branding') @@ -183,43 +183,15 @@ class ServiceSchema(BaseSchema): permissions = fields.Method("service_permissions") override_flag = False + def get_free_sms_fragment_limit(selfs, service): + return service.free_sms_fragment_limit() + def service_permissions(self, service): - permissions = [] - str_permissions = [] - - perms = dao_fetch_service_permissions(service.id) - for p in perms: - permission = { - "service_id": service.id, - "permission": p.permission - } - permissions.append(permission) - str_permissions.append(p.permission) - - def deprecate_convert_flags_to_permissions(): - def convert_flags(flag, notify_type): - if flag and notify_type not in str_permissions: - permission = { - "service_id": service.id, - "permission": notify_type - } - permissions.append(permission) - elif flag is False and notify_type in str_permissions: - permission = { - "service_id": service.id, - "permission": notify_type - } - permissions.remove(permission) - - convert_flags(service.can_send_international_sms, INTERNATIONAL_SMS_TYPE) - convert_flags(service.can_send_letters, LETTER_TYPE) - - deprecate_convert_flags_to_permissions() - - return permissions + return [p.permission for p in service.permissions] class Meta: model = models.Service + dump_only = ['free_sms_fragment_limit'] exclude = ( 'updated_at', 'created_at', @@ -254,14 +226,13 @@ class ServiceSchema(BaseSchema): if isinstance(in_data, dict) and 'permissions' in in_data: str_permissions = in_data['permissions'] permissions = [] - for p in in_data['permissions']: + for p in str_permissions: permission = ServicePermission(service_id=in_data["id"], permission=p) permissions.append(permission) - in_data['permissions'] = permissions def deprecate_override_flags(): - in_data['can_send_letters'] = LETTER_TYPE in [p.permission for p in permissions] - in_data['can_send_international_sms'] = INTERNATIONAL_SMS_TYPE in [p.permission for p in permissions] + in_data['can_send_letters'] = LETTER_TYPE in str_permissions + in_data['can_send_international_sms'] = INTERNATIONAL_SMS_TYPE in str_permissions def deprecate_convert_flags_to_permissions(): def convert_flags(flag, notify_type): @@ -280,12 +251,7 @@ class ServiceSchema(BaseSchema): deprecate_override_flags() else: deprecate_convert_flags_to_permissions() - - @post_dump - def format_as_string_array(self, in_data): - if isinstance(in_data, dict) and 'permissions' in in_data: - in_data['permissions'] = [p.get("permission") for p in in_data['permissions']] - return in_data + in_data['permissions'] = permissions def set_override_flag(self, flag): self.override_flag = flag @@ -294,6 +260,11 @@ class ServiceSchema(BaseSchema): class DetailedServiceSchema(BaseSchema): statistics = fields.Dict() + free_sms_fragment_limit = fields.Method(method_name='get_free_sms_fragment_limit') + + def get_free_sms_fragment_limit(selfs, service): + return service.free_sms_fragment_limit() + class Meta: model = models.Service exclude = ( diff --git a/app/service/rest.py b/app/service/rest.py index dfca05c89..1e301d0a7 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -10,6 +10,7 @@ from flask import ( ) from sqlalchemy.orm.exc import NoResultFound +from app import redis_store from app.dao import notification_usage_dao, notifications_dao from app.dao.dao_utils import dao_rollback from app.dao.api_key_dao import ( @@ -17,6 +18,8 @@ from app.dao.api_key_dao import ( get_model_api_keys, get_unsigned_secret, expire_api_key) +from app.dao.date_util import get_financial_year +from app.dao.notification_usage_dao import get_total_billable_units_for_sent_sms_notifications_in_date_range from app.dao.services_dao import ( dao_fetch_service_by_id, dao_fetch_all_services, @@ -60,6 +63,7 @@ from app.schemas import ( detailed_service_schema ) from app.utils import pagination_links +from notifications_utils.clients.redis import sms_billable_units_cache_key service_blueprint = Blueprint('service', __name__) @@ -296,6 +300,19 @@ def get_all_notifications_for_service(service_id): ), 200 +@service_blueprint.route('//notifications/', methods=['GET']) +def get_notification_for_service(service_id, notification_id): + + notification = notifications_dao.get_notification_with_personalisation( + service_id, + notification_id, + key_type=None, + ) + return jsonify( + notification_with_template_schema.dump(notification).data, + ), 200 + + def search_for_notification_by_to_field(service_id, search_term, statuses): results = notifications_dao.dao_get_notifications_by_to_field(service_id, search_term, statuses) return jsonify( @@ -447,6 +464,39 @@ def get_monthly_template_stats(service_id): raise InvalidRequest('Year must be a number', status_code=400) +@service_blueprint.route('//yearly-sms-billable-units') +def get_yearly_sms_billable_units(service_id): + cache_key = sms_billable_units_cache_key(service_id) + cached_billable_sms_units = redis_store.get_all_from_hash(cache_key) + if cached_billable_sms_units: + return jsonify({ + 'billable_sms_units': int(cached_billable_sms_units[b'billable_units']), + 'total_cost': float(cached_billable_sms_units[b'total_cost']) + }) + else: + try: + start_date, end_date = get_financial_year(int(request.args.get('year'))) + except (ValueError, TypeError) as e: + current_app.logger.exception(e) + return jsonify(result='error', message='No valid year provided'), 400 + + billable_units, total_cost = get_total_billable_units_for_sent_sms_notifications_in_date_range( + start_date, + end_date, + service_id) + + cached_values = { + 'billable_units': billable_units, + 'total_cost': total_cost + } + + redis_store.set_hash_and_expire(cache_key, cached_values, expire_in_seconds=60) + return jsonify({ + 'billable_sms_units': billable_units, + 'total_cost': total_cost + }) + + @service_blueprint.route('//yearly-usage') def get_yearly_billing_usage(service_id): try: diff --git a/app/service/sender.py b/app/service/sender.py index 3c6a6a03e..4919a93bf 100644 --- a/app/service/sender.py +++ b/app/service/sender.py @@ -1,5 +1,6 @@ from flask import current_app +from app.config import QueueNames from app.dao.services_dao import dao_fetch_service_by_id, dao_fetch_active_users_for_service from app.dao.templates_dao import dao_get_template_by_id from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL @@ -24,7 +25,7 @@ def send_notification_to_service_users(service_id, template_id, personalisation= api_key_id=None, key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(notification, False, queue='notify') + send_notification_to_queue(notification, False, queue=QueueNames.NOTIFY) def _add_user_fields(user, personalisation, fields): diff --git a/app/user/rest.py b/app/user/rest.py index b40c3a31c..9a9cf3832 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -4,6 +4,7 @@ from datetime import datetime from flask import (jsonify, request, Blueprint, current_app) +from app.config import QueueNames from app.dao.users_dao import ( get_user_by_id, save_model_user, @@ -182,7 +183,7 @@ def send_user_sms_code(user_id): # Assume that we never want to observe the Notify service's research mode # setting for this notification - we still need to be able to log into the # admin even if we're doing user research using this service: - send_notification_to_queue(saved_notification, False, queue='notify') + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -205,14 +206,14 @@ def send_user_confirm_new_email(user_id): personalisation={ 'name': user_to_send_to.name, 'url': _create_confirmation_url(user=user_to_send_to, email_address=email['email']), - 'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/feedback' + 'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/support' }, notification_type=EMAIL_TYPE, api_key_id=None, key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue='notify') + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -239,7 +240,7 @@ def send_user_email_verification(user_id): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -258,14 +259,14 @@ def send_already_registered_email(user_id): personalisation={ 'signin_url': current_app.config['ADMIN_BASE_URL'] + '/sign-in', 'forgot_password_url': current_app.config['ADMIN_BASE_URL'] + '/forgot-password', - 'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/feedback' + 'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/support' }, notification_type=EMAIL_TYPE, api_key_id=None, key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -327,7 +328,7 @@ def send_user_reset_password(): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 diff --git a/app/v2/notifications/notification_schemas.py b/app/v2/notifications/notification_schemas.py index 6aa4dcb77..69372c4e1 100644 --- a/app/v2/notifications/notification_schemas.py +++ b/app/v2/notifications/notification_schemas.py @@ -113,7 +113,7 @@ post_sms_request = { "phone_number": {"type": "string", "format": "phone_number"}, "template_id": uuid, "personalisation": personalisation, - "scheduled_for": {"type": "string", "format": "datetime"} + "scheduled_for": {"type": ["string", "null"], "format": "datetime"} }, "required": ["phone_number", "template_id"] } @@ -141,7 +141,7 @@ post_sms_response = { "content": sms_content, "uri": {"type": "string", "format": "uri"}, "template": template, - "scheduled_for": {"type": "string"} + "scheduled_for": {"type": ["string", "null"]} }, "required": ["id", "content", "uri", "template"] } @@ -157,7 +157,7 @@ post_email_request = { "email_address": {"type": "string", "format": "email_address"}, "template_id": uuid, "personalisation": personalisation, - "scheduled_for": {"type": "string", "format": "datetime"} + "scheduled_for": {"type": ["string", "null"], "format": "datetime"} }, "required": ["email_address", "template_id"] } @@ -186,7 +186,7 @@ post_email_response = { "content": email_content, "uri": {"type": "string", "format": "uri"}, "template": template, - "scheduled_for": {"type": "string"} + "scheduled_for": {"type": ["string", "null"]} }, "required": ["id", "content", "uri", "template"] } diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index 6cbcbf0f0..47a40a72e 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -2,7 +2,8 @@ from flask import request, jsonify, current_app from sqlalchemy.orm.exc import NoResultFound from app import api_user, authenticated_service -from app.dao import services_dao, templates_dao +from app.config import QueueNames +from app.dao import templates_dao from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY from app.notifications.process_notifications import ( create_content_for_notification, @@ -34,9 +35,8 @@ def post_notification(notification_type): form = validate(request.get_json(), post_sms_request) scheduled_for = form.get("scheduled_for", None) - if scheduled_for: - if not service_can_schedule_notification(authenticated_service): - return + service_can_schedule_notification(authenticated_service, scheduled_for) + check_rate_limiting(authenticated_service, api_user) form_send_to = form['phone_number'] if notification_type == SMS_TYPE else form['email_address'] @@ -65,13 +65,12 @@ def post_notification(notification_type): persist_scheduled_notification(notification.id, form["scheduled_for"]) else: if not simulated: - queue_name = 'priority' if template.process_type == PRIORITY else None + queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None send_notification_to_queue( notification=notification, research_mode=authenticated_service.research_mode, queue=queue_name ) - else: current_app.logger.info("POST simulated notification for id: {}".format(notification.id)) diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml index 04b396388..eccae21b1 100644 --- a/manifest-api-preview.yml +++ b/manifest-api-preview.yml @@ -6,3 +6,6 @@ routes: - route: notify-api-preview.cloudapps.digital - route: api-paas.notify.works - route: api.notify.works + +instances: 1 +memory: 1G diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 73457b3c9..7f73a4b54 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -23,33 +23,32 @@ applications: NOTIFY_APP_NAME: delivery-celery-beat - name: notify-delivery-worker-database - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks env: NOTIFY_APP_NAME: delivery-worker-database - name: notify-delivery-worker-research - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks env: NOTIFY_APP_NAME: delivery-worker-research - name: notify-delivery-worker-sender - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-tasks env: NOTIFY_APP_NAME: delivery-worker-sender - name: notify-delivery-worker-periodic - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic,statistics + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks instances: 1 - memory: 2G env: NOTIFY_APP_NAME: delivery-worker-periodic - name: notify-delivery-worker-priority - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks env: NOTIFY_APP_NAME: delivery-worker-priority - name: notify-delivery-worker - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,notify-internal-tasks env: NOTIFY_APP_NAME: delivery-worker diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml index d628e5fc9..492bc6c55 100644 --- a/manifest-delivery-preview.yml +++ b/manifest-delivery-preview.yml @@ -1,3 +1,4 @@ --- inherit: manifest-delivery-base.yml +memory: 1G diff --git a/migrations/versions/0088_add_schedule_serv_perm.py b/migrations/versions/0088_add_schedule_serv_perm.py new file mode 100644 index 000000000..0882c7c94 --- /dev/null +++ b/migrations/versions/0088_add_schedule_serv_perm.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 0088_add_schedule_serv_perm +Revises: 0087_scheduled_notifications +Create Date: 2017-05-26 14:53:18.581320 + +""" + +# revision identifiers, used by Alembic. +revision = '0088_add_schedule_serv_perm' +down_revision = '0087_scheduled_notifications' + +from alembic import op + + +def upgrade(): + op.get_bind() + op.execute("insert into service_permission_types values('schedule_notifications')") + + +def downgrade(): + op.get_bind() + op.execute("delete from service_permissions where permission = 'schedule_notifications'") + op.execute("delete from service_permission_types where name = 'schedule_notifications'") diff --git a/migrations/versions/0089_govuk_sms_sender.py b/migrations/versions/0089_govuk_sms_sender.py new file mode 100644 index 000000000..b69701abd --- /dev/null +++ b/migrations/versions/0089_govuk_sms_sender.py @@ -0,0 +1,25 @@ +"""empty message + +Revision ID: 0089_govuk_sms_sender +Revises: 0088_add_schedule_serv_perm +Create Date: 2017-05-22 13:46:09.584801 + +""" + +# revision identifiers, used by Alembic. +revision = '0089_govuk_sms_sender' +down_revision = '0088_add_schedule_serv_perm' + +from alembic import op + + +def upgrade(): + op.execute("UPDATE services SET sms_sender = 'GOVUK' where sms_sender is null") + op.execute("UPDATE services_history SET sms_sender = 'GOVUK' where sms_sender is null") + op.alter_column('services', 'sms_sender', nullable=False) + op.alter_column('services_history', 'sms_sender', nullable=False) + + +def downgrade(): + op.alter_column('services_history', 'sms_sender', nullable=True) + op.alter_column('services', 'sms_sender', nullable=True) diff --git a/migrations/versions/0090_inbound_sms.py b/migrations/versions/0090_inbound_sms.py new file mode 100644 index 000000000..d02690379 --- /dev/null +++ b/migrations/versions/0090_inbound_sms.py @@ -0,0 +1,37 @@ +"""empty message + +Revision ID: 0090_inbound_sms +Revises: 0089_govuk_sms_sender +Create Date: 2017-05-22 11:28:53.471004 + +""" + +# revision identifiers, used by Alembic. +revision = '0090_inbound_sms' +down_revision = '0089_govuk_sms_sender' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + op.create_table( + 'inbound_sms', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('content', sa.String, nullable=False), + sa.Column('notify_number', sa.String, nullable=False), + sa.Column('user_number', sa.String, nullable=False), + sa.Column('created_at', sa.DateTime, nullable=False), + sa.Column('provider_date', sa.DateTime, nullable=True), + sa.Column('provider_reference', sa.String, nullable=True), + + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_inbound_sms_service_id'), 'inbound_sms', ['service_id'], unique=False) + op.create_index(op.f('ix_inbound_sms_user_number'), 'inbound_sms', ['user_number'], unique=False) + + +def downgrade(): + op.drop_table('inbound_sms') diff --git a/migrations/versions/0091_letter_billing.py b/migrations/versions/0091_letter_billing.py new file mode 100644 index 000000000..7dda9bb70 --- /dev/null +++ b/migrations/versions/0091_letter_billing.py @@ -0,0 +1,50 @@ +"""empty message + +Revision ID: 0091_letter_billing +Revises: 0090_inbound_sms +Create Date: 2017-05-31 11:43:55.744631 + +""" +import uuid +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0091_letter_billing' +down_revision = '0090_inbound_sms' + + +def upgrade(): + op.create_table('letter_rates', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('valid_from', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('letter_rate_details', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('letter_rate_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('page_total', sa.Integer(), nullable=False), + sa.Column('rate', sa.Numeric(), nullable=False), + sa.ForeignKeyConstraint(['letter_rate_id'], ['letter_rates.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_letter_rate_details_letter_rate_id'), 'letter_rate_details', ['letter_rate_id'], + unique=False) + + op.get_bind() + letter_id = uuid.uuid4() + op.execute("insert into letter_rates(id, valid_from) values('{}', '2017-03-31 23:00:00')".format(letter_id)) + insert_details = "insert into letter_rate_details(id, letter_rate_id, page_total, rate) values('{}', '{}', {}, {})" + op.execute( + insert_details.format(uuid.uuid4(), letter_id, 1, 29.3)) + op.execute( + insert_details.format(uuid.uuid4(), letter_id, 2, 32)) + op.execute( + insert_details.format(uuid.uuid4(), letter_id, 3, 35)) + + +def downgrade(): + op.get_bind() + op.drop_index('ix_letter_rate_details_letter_rate_id') + op.drop_table('letter_rate_details') + op.drop_table('letter_rates') diff --git a/migrations/versions/0092_add_inbound_provider.py b/migrations/versions/0092_add_inbound_provider.py new file mode 100644 index 000000000..f7e5f510e --- /dev/null +++ b/migrations/versions/0092_add_inbound_provider.py @@ -0,0 +1,22 @@ +"""empty message + +Revision ID: 0092_add_inbound_provider +Revises: 0091_letter_billing +Create Date: 2017-06-02 16:07:35.445423 + +""" + +# revision identifiers, used by Alembic. +revision = '0092_add_inbound_provider' +down_revision = '0091_letter_billing' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + op.add_column('inbound_sms', sa.Column('provider', sa.String(), nullable=True)) + + +def downgrade(): + op.drop_column('inbound_sms', 'provider') diff --git a/migrations/versions/0093_data_gov_uk.py b/migrations/versions/0093_data_gov_uk.py new file mode 100644 index 000000000..fbe22d38a --- /dev/null +++ b/migrations/versions/0093_data_gov_uk.py @@ -0,0 +1,32 @@ +"""empty message + +Revision ID: 0093_data_gov_uk +Revises: 0092_add_inbound_provider +Create Date: 2017-06-05 16:15:17.744908 + +""" + +# revision identifiers, used by Alembic. +revision = '0093_data_gov_uk' +down_revision = '0092_add_inbound_provider' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +DATA_GOV_UK_ID = '123496d4-44cb-4324-8e0a-4187101f4bdc' + + +def upgrade(): + op.execute("""INSERT INTO organisation VALUES ( + '{}', + '', + 'data_gov_uk_x2.png', + '' + )""".format(DATA_GOV_UK_ID)) + + +def downgrade(): + op.execute(""" + DELETE FROM organisation WHERE "id" = '{}' + """.format(DATA_GOV_UK_ID)) diff --git a/migrations/versions/0094_job_stats_update.py b/migrations/versions/0094_job_stats_update.py new file mode 100644 index 000000000..6a7f7db2a --- /dev/null +++ b/migrations/versions/0094_job_stats_update.py @@ -0,0 +1,25 @@ +"""empty message + +Revision ID: 0094_job_stats_update +Revises: 0093_data_gov_uk +Create Date: 2017-06-06 14:37:30.051647 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '0094_job_stats_update' +down_revision = '0093_data_gov_uk' + + +def upgrade(): + op.add_column('job_statistics', sa.Column('sent', sa.BigInteger(), nullable=True)) + op.add_column('job_statistics', sa.Column('delivered', sa.BigInteger(), nullable=True)) + op.add_column('job_statistics', sa.Column('failed', sa.BigInteger(), nullable=True)) + + +def downgrade(): + op.drop_column('job_statistics', 'sent') + op.drop_column('job_statistics', 'failed') + op.drop_column('job_statistics', 'delivered') diff --git a/requirements.txt b/requirements.txt index 3551937fa..46b02daa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,6 @@ notifications-python-client>=3.1,<3.2 awscli>=1.11,<1.12 awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@17.1.0#egg=notifications-utils==17.1.0 +git+https://github.com/alphagov/notifications-utils.git@17.1.3#egg=notifications-utils==17.1.3 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/scripts/check_if_new_migration.py b/scripts/check_if_new_migration.py new file mode 100644 index 000000000..ba0519ea1 --- /dev/null +++ b/scripts/check_if_new_migration.py @@ -0,0 +1,34 @@ +import os +from os.path import dirname, abspath +import requests +import sys + + +def get_latest_db_migration_to_apply(): + project_dir = dirname(dirname(abspath(__file__))) # Get the main project directory + migrations_dir = '{}/migrations/versions/'.format(project_dir) + migration_files = [migration_file for migration_file in os.listdir(migrations_dir) if migration_file.endswith('py')] + latest_file = sorted(migration_files, reverse=True)[0].replace('.py', '') + return latest_file + + +def get_current_db_version(): + api_status_url = '{}/_status'.format(os.getenv('API_HOST_NAME')) + response = requests.get(api_status_url) + + if response.status_code != 200: + sys.exit('Could not make a request to the API: {}'.format()) + + current_db_version = response.json()['db_version'] + return current_db_version + + +def run(): + if get_current_db_version() == get_latest_db_migration_to_apply(): + print('no') + else: + print('yes') + + +if __name__ == "__main__": + run() diff --git a/scripts/delete_sqs_queues.py b/scripts/delete_sqs_queues.py old mode 100644 new mode 100755 index bdbf6ff65..b167ce392 --- a/scripts/delete_sqs_queues.py +++ b/scripts/delete_sqs_queues.py @@ -1,10 +1,33 @@ +""" + +Script to manage SQS queues. Can list or delete queues. + +Uses boto, so relies on correctly set up AWS access keys and tokens. + +In principle use this script to dump details of all queues in a gievn environment, and then +manipulate the resultant CSV file so that it contains the queues you want to delete. + +Very hands on. Starter for a more automagic process. + +Usage: + scripts/delete_sqs_queues.py + + options are: + - list: dumps queue details to local file queues.csv in current directory. + - delete: delete queues from local file queues.csv in current directory. + +Example: + scripts/delete_sqs_queues.py list delete +""" + +from docopt import docopt import boto3 import csv from datetime import datetime -from pprint import pprint -import os -client = boto3.client('sqs', region_name=os.getenv('AWS_REGION')) +FILE_NAME = "/tmp/queues.csv" + +client = boto3.client('sqs', region_name='eu-west-1') def _formatted_date_from_timestamp(timestamp): @@ -27,15 +50,18 @@ def get_queue_attributes(queue_name): ] ) queue_attributes = response['Attributes'] + queue_attributes.update({'QueueUrl': queue_name}) return queue_attributes -def delete_queue(queue_name): +def delete_queue(queue_url): + # Note that deleting a queue returns 200 OK if it doesn't exist + print("DELETEING {}".format(queue_url)) response = client.delete_queue( - QueueUrl=queue_name + QueueUrl=queue_url ) if response['ResponseMetadata']['HTTPStatusCode'] == 200: - print('Deleted queue successfully') + print('Deleted queue successfully {}'.format(response['ResponseMetadata'])) else: print('Error occured when attempting to delete queue') pprint(response) @@ -43,10 +69,10 @@ def delete_queue(queue_name): def output_to_csv(queue_attributes): - csv_name = 'queues.csv' - with open(csv_name, 'w') as csvfile: + with open(FILE_NAME, 'w') as csvfile: fieldnames = [ 'Queue Name', + 'Queue URL', 'Number of Messages', 'Number of Messages Delayed', 'Number of Messages Not Visible', @@ -55,23 +81,19 @@ def output_to_csv(queue_attributes): writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() for queue_attr in queue_attributes: - queue_url = client.get_queue_url( - QueueName=queue_attr['QueueArn'] - )['QueueUrl'] writer.writerow({ 'Queue Name': queue_attr['QueueArn'], - 'Queue URL': queue_url, + 'Queue URL': queue_attr['QueueUrl'], 'Number of Messages': queue_attr['ApproximateNumberOfMessages'], 'Number of Messages Delayed': queue_attr['ApproximateNumberOfMessagesDelayed'], 'Number of Messages Not Visible': queue_attr['ApproximateNumberOfMessagesNotVisible'], 'Created': _formatted_date_from_timestamp(queue_attr['CreatedTimestamp']) }) - return csv_name -def read_from_csv(csv_name): +def read_from_csv(): queue_urls = [] - with open(csv_name, 'r') as csvfile: + with open(FILE_NAME, 'r') as csvfile: next(csvfile) rows = csv.reader(csvfile, delimiter=',') for row in rows: @@ -79,6 +101,19 @@ def read_from_csv(csv_name): return queue_urls -queues = get_queues() -for queue in queues: - delete_queue(queue) +if __name__ == "__main__": + arguments = docopt(__doc__) + + if arguments[''] == 'list': + queues = get_queues() + queue_attributes = [] + for queue in queues: + queue_attributes.append(get_queue_attributes(queue)) + output_to_csv(queue_attributes) + elif arguments[''] == 'delete': + queues_to_delete = read_from_csv() + for queue in queues_to_delete: + delete_queue(queue) + else: + print("UNKNOWN COMMAND") + exit(1) diff --git a/tests/app/celery/test_provider_tasks.py b/tests/app/celery/test_provider_tasks.py index d2f815d9d..b5dbc999a 100644 --- a/tests/app/celery/test_provider_tasks.py +++ b/tests/app/celery/test_provider_tasks.py @@ -11,34 +11,6 @@ def test_should_have_decorated_tasks_functions(): assert deliver_email.__wrapped__.__name__ == 'deliver_email' -def test_should_by_10_second_delay_as_default(): - assert provider_tasks.retry_iteration_to_delay() == 10 - - -def test_should_by_10_second_delay_on_unmapped_retry_iteration(): - assert provider_tasks.retry_iteration_to_delay(99) == 10 - - -def test_should_by_10_second_delay_on_retry_one(): - assert provider_tasks.retry_iteration_to_delay(0) == 10 - - -def test_should_by_1_minute_delay_on_retry_two(): - assert provider_tasks.retry_iteration_to_delay(1) == 60 - - -def test_should_by_5_minute_delay_on_retry_two(): - assert provider_tasks.retry_iteration_to_delay(2) == 300 - - -def test_should_by_60_minute_delay_on_retry_two(): - assert provider_tasks.retry_iteration_to_delay(3) == 3600 - - -def test_should_by_240_minute_delay_on_retry_two(): - assert provider_tasks.retry_iteration_to_delay(4) == 14400 - - def test_should_call_send_sms_to_provider_from_deliver_sms_task( notify_db, notify_db_session, @@ -61,7 +33,7 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task deliver_sms(notification_id) app.delivery.send_to_providers.send_sms_to_provider.assert_not_called() - app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry", countdown=10) + app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks") def test_should_call_send_email_to_provider_from_deliver_email_task( @@ -83,7 +55,7 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_ta deliver_email(notification_id) app.delivery.send_to_providers.send_email_to_provider.assert_not_called() - app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry", countdown=10) + app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks") # DO THESE FOR THE 4 TYPES OF TASK @@ -94,7 +66,7 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(s deliver_sms(sample_notification.id) - provider_tasks.deliver_sms.retry.assert_called_with(queue='retry', countdown=10) + provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks") assert sample_notification.status == 'technical-failure' @@ -105,7 +77,7 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_email_task deliver_email(sample_notification.id) - provider_tasks.deliver_email.retry.assert_called_with(queue='retry', countdown=10) + provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks") assert sample_notification.status == 'technical-failure' diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 194d764fd..b3ff5d478 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -5,19 +5,23 @@ from functools import partial from flask import current_app from freezegun import freeze_time -from app.celery.scheduled_tasks import s3, timeout_job_statistics, delete_sms_notifications_older_than_seven_days, \ - delete_letter_notifications_older_than_seven_days, delete_email_notifications_older_than_seven_days, \ - send_scheduled_notifications from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( + delete_email_notifications_older_than_seven_days, + delete_inbound_sms_older_than_seven_days, + delete_invitations, + delete_notifications_created_more_than_a_week_ago_by_type, + delete_letter_notifications_older_than_seven_days, + delete_sms_notifications_older_than_seven_days, delete_verify_codes, remove_csv_files, - delete_notifications_created_more_than_a_week_ago_by_type, - delete_invitations, - timeout_notifications, run_scheduled_jobs, + s3, send_daily_performance_platform_stats, - switch_current_sms_provider_on_slow_delivery + send_scheduled_notifications, + switch_current_sms_provider_on_slow_delivery, + timeout_job_statistics, + timeout_notifications ) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.dao.jobs_dao import dao_get_job_by_id @@ -26,9 +30,12 @@ from app.dao.provider_details_dao import ( dao_update_provider_details, get_current_provider ) -from app.models import Service, Template +from app.models import ( + Service, Template, + SMS_TYPE, LETTER_TYPE +) from app.utils import get_london_midnight_in_utc -from tests.app.db import create_notification, create_service +from tests.app.db import create_notification, create_service, create_template, create_job from tests.app.conftest import ( sample_job as create_sample_job, sample_notification_history as create_notification_history, @@ -71,7 +78,8 @@ def prepare_current_provider(restore_provider_details): def test_should_have_decorated_tasks_functions(): assert delete_verify_codes.__wrapped__.__name__ == 'delete_verify_codes' - assert delete_notifications_created_more_than_a_week_ago_by_type.__wrapped__.__name__ == 'delete_notifications_created_more_than_a_week_ago_by_type' # noqa + assert delete_notifications_created_more_than_a_week_ago_by_type.__wrapped__.__name__ == \ + 'delete_notifications_created_more_than_a_week_ago_by_type' assert timeout_notifications.__wrapped__.__name__ == 'timeout_notifications' assert delete_invitations.__wrapped__.__name__ == 'delete_invitations' assert run_scheduled_jobs.__wrapped__.__name__ == 'run_scheduled_jobs' @@ -79,6 +87,8 @@ def test_should_have_decorated_tasks_functions(): assert send_daily_performance_platform_stats.__wrapped__.__name__ == 'send_daily_performance_platform_stats' assert switch_current_sms_provider_on_slow_delivery.__wrapped__.__name__ == \ 'switch_current_sms_provider_on_slow_delivery' + assert delete_inbound_sms_older_than_seven_days.__wrapped__.__name__ == \ + 'delete_inbound_sms_older_than_seven_days' def test_should_call_delete_sms_notifications_more_than_week_in_task(notify_api, mocker): @@ -166,7 +176,7 @@ def test_should_update_scheduled_jobs_and_put_on_queue(notify_db, notify_db_sess updated_job = dao_get_job_by_id(job.id) assert updated_job.job_status == 'pending' - mocked.assert_called_with([str(job.id)], queue='process-job') + mocked.assert_called_with([str(job.id)], queue="job-tasks") def test_should_update_all_scheduled_jobs_and_put_on_queue(notify_db, notify_db_session, mocker): @@ -201,28 +211,39 @@ def test_should_update_all_scheduled_jobs_and_put_on_queue(notify_db, notify_db_ assert dao_get_job_by_id(job_2.id).job_status == 'pending' mocked.assert_has_calls([ - call([str(job_3.id)], queue='process-job'), - call([str(job_2.id)], queue='process-job'), - call([str(job_1.id)], queue='process-job') + call([str(job_3.id)], queue="job-tasks"), + call([str(job_2.id)], queue="job-tasks"), + call([str(job_1.id)], queue="job-tasks") ]) -def test_will_remove_csv_files_for_jobs_older_than_seven_days(notify_db, notify_db_session, mocker): +@freeze_time('2016-10-18T10:00:00') +def test_will_remove_csv_files_for_jobs_older_than_seven_days( + notify_db, notify_db_session, mocker, sample_template +): mocker.patch('app.celery.scheduled_tasks.s3.remove_job_from_s3') + """ + Jobs older than seven days are deleted, but only two day's worth (two-day window) + """ + seven_days_ago = datetime.utcnow() - timedelta(days=7) + just_under_seven_days = seven_days_ago + timedelta(seconds=1) + eight_days_ago = seven_days_ago - timedelta(days=1) + nine_days_ago = eight_days_ago - timedelta(days=1) + just_under_nine_days = nine_days_ago + timedelta(seconds=1) + nine_days_one_second_ago = nine_days_ago - timedelta(seconds=1) - eligible_job_1 = datetime(2016, 10, 10, 23, 59, 59, 000) - eligible_job_2 = datetime(2016, 10, 9, 00, 00, 00, 000) - in_eligible_job_too_new = datetime(2016, 10, 11, 00, 00, 00, 000) - in_eligible_job_too_old = datetime(2016, 10, 8, 23, 59, 59, 999) + create_sample_job(notify_db, notify_db_session, created_at=nine_days_one_second_ago) + job1_to_delete = create_sample_job(notify_db, notify_db_session, created_at=eight_days_ago) + job2_to_delete = create_sample_job(notify_db, notify_db_session, created_at=just_under_nine_days) + create_sample_job(notify_db, notify_db_session, created_at=seven_days_ago) + create_sample_job(notify_db, notify_db_session, created_at=just_under_seven_days) - job_1 = create_sample_job(notify_db, notify_db_session, created_at=eligible_job_1) - job_2 = create_sample_job(notify_db, notify_db_session, created_at=eligible_job_2) - create_sample_job(notify_db, notify_db_session, created_at=in_eligible_job_too_new) - create_sample_job(notify_db, notify_db_session, created_at=in_eligible_job_too_old) + remove_csv_files(job_types=[sample_template.template_type]) - with freeze_time('2016-10-18T10:00:00'): - remove_csv_files() - assert s3.remove_job_from_s3.call_args_list == [call(job_1.service_id, job_1.id), call(job_2.service_id, job_2.id)] + assert s3.remove_job_from_s3.call_args_list == [ + call(job1_to_delete.service_id, job1_to_delete.id), + call(job2_to_delete.service_id, job2_to_delete.id) + ] def test_send_daily_performance_stats_calls_does_not_send_if_inactive( @@ -430,7 +451,7 @@ def test_should_send_all_scheduled_notifications_to_deliver_queue(sample_templat send_scheduled_notifications() - mocked.apply_async.assert_called_once_with([str(message_to_deliver.id)], queue='send-sms') + mocked.apply_async.assert_called_once_with([str(message_to_deliver.id)], queue='send-tasks') scheduled_notifications = dao_get_scheduled_notifications() assert not scheduled_notifications @@ -440,3 +461,30 @@ def test_timeout_job_statistics_called_with_notification_timeout(notify_api, moc dao_mock = mocker.patch('app.celery.scheduled_tasks.dao_timeout_job_statistics') timeout_job_statistics() dao_mock.assert_called_once_with(999) + + +def test_should_call_delete_inbound_sms_older_than_seven_days(notify_api, mocker): + mocker.patch('app.celery.scheduled_tasks.delete_inbound_sms_created_more_than_a_week_ago') + delete_inbound_sms_older_than_seven_days() + assert scheduled_tasks.delete_inbound_sms_created_more_than_a_week_ago.call_count == 1 + + +@freeze_time('2017-01-01 10:00:00') +def test_remove_csv_files_filters_by_type(mocker, sample_service): + mocker.patch('app.celery.scheduled_tasks.s3.remove_job_from_s3') + """ + Jobs older than seven days are deleted, but only two day's worth (two-day window) + """ + letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) + sms_template = create_template(service=sample_service, template_type=SMS_TYPE) + + eight_days_ago = datetime.utcnow() - timedelta(days=8) + + job_to_delete = create_job(template=letter_template, created_at=eight_days_ago) + create_job(template=sms_template, created_at=eight_days_ago) + + remove_csv_files(job_types=[LETTER_TYPE]) + + assert s3.remove_job_from_s3.call_args_list == [ + call(job_to_delete.service_id, job_to_delete.id), + ] diff --git a/tests/app/celery/test_statistics_tasks.py b/tests/app/celery/test_statistics_tasks.py index 24aaba97d..40d20117d 100644 --- a/tests/app/celery/test_statistics_tasks.py +++ b/tests/app/celery/test_statistics_tasks.py @@ -17,7 +17,7 @@ def test_should_create_initial_job_task_if_notification_is_related_to_a_job( mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job) create_initial_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics") + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") @pytest.mark.parametrize('status', [ @@ -29,7 +29,7 @@ def test_should_create_intial_job_task_if_notification_is_not_in_completed_state mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status) create_initial_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics") + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") def test_should_not_create_initial_job_task_if_notification_is_not_related_to_a_job( @@ -47,7 +47,7 @@ def test_should_create_outcome_job_task_if_notification_is_related_to_a_job( mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=NOTIFICATION_DELIVERED) create_outcome_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics") + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") @pytest.mark.parametrize('status', NOTIFICATION_STATUS_TYPES_COMPLETED) @@ -57,7 +57,7 @@ def test_should_create_outcome_job_task_if_notification_is_in_completed_state( mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status) create_outcome_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue='statistics') + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") @pytest.mark.parametrize('status', [ @@ -100,7 +100,7 @@ def test_should_retry_if_persisting_the_job_stats_has_a_sql_alchemy_exception( record_initial_job_statistics(str(sample_notification.id)) dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_call_update_job_stats_dao_outcome_methods(notify_db, notify_db_session, sample_notification, mocker): @@ -123,7 +123,7 @@ def test_should_retry_if_persisting_the_job_outcome_stats_has_a_sql_alchemy_exce record_outcome_job_statistics(str(sample_notification.id)) dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_retry_if_persisting_the_job_outcome_stats_updates_zero_rows( @@ -136,7 +136,7 @@ def test_should_retry_if_persisting_the_job_outcome_stats_updates_zero_rows( record_outcome_job_statistics(str(sample_notification.id)) dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_retry_if_persisting_the_job_stats_creation_cant_find_notification_by_id( @@ -148,7 +148,7 @@ def test_should_retry_if_persisting_the_job_stats_creation_cant_find_notificatio record_initial_job_statistics(str(create_uuid())) dao_mock.assert_not_called() - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_retry_if_persisting_the_job_stats_outcome_cant_find_notification_by_id( @@ -161,4 +161,4 @@ def test_should_retry_if_persisting_the_job_stats_outcome_cant_find_notification record_outcome_job_statistics(str(create_uuid())) dao_mock.assert_not_called() - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 27c9833a4..8cb5a13d4 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -108,7 +108,7 @@ def test_should_process_sms_job(sample_job, mocker): "uuid", "something_encrypted", "2016-01-01T11:09:00.061258Z"), - queue="db-sms" + queue="database-tasks" ) job = jobs_dao.dao_get_job_by_id(sample_job.id) assert job.job_status == 'finished' @@ -237,7 +237,7 @@ def test_should_process_email_job_if_exactly_on_send_limits(notify_db, "something_encrypted", "2016-01-01T11:09:00.061258Z" ), - queue="db-email" + queue="database-tasks" ) @@ -283,7 +283,7 @@ def test_should_process_email_job(email_job_with_placeholders, mocker): "something_encrypted", "2016-01-01T11:09:00.061258Z" ), - queue="db-email" + queue="database-tasks" ) job = jobs_dao.dao_get_job_by_id(email_job_with_placeholders.id) assert job.job_status == 'finished' @@ -324,7 +324,7 @@ def test_should_process_letter_job(sample_letter_job, mocker): assert process_row_mock.call_count == 1 assert sample_letter_job.job_status == 'in progress' - tasks.build_dvla_file.apply_async.assert_called_once_with([str(sample_letter_job.id)], queue="process-job") + tasks.build_dvla_file.apply_async.assert_called_once_with([str(sample_letter_job.id)], queue="job-tasks") def test_should_process_all_sms_job(sample_job_with_placeholdered_template, @@ -355,12 +355,12 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template, @freeze_time('2001-01-01T12:00:00') @pytest.mark.parametrize('template_type, research_mode, expected_function, expected_queue', [ - (SMS_TYPE, False, 'send_sms', 'db-sms'), - (SMS_TYPE, True, 'send_sms', 'research-mode'), - (EMAIL_TYPE, False, 'send_email', 'db-email'), - (EMAIL_TYPE, True, 'send_email', 'research-mode'), - (LETTER_TYPE, False, 'persist_letter', 'db-letter'), - (LETTER_TYPE, True, 'persist_letter', 'research-mode'), + (SMS_TYPE, False, 'send_sms', 'database-tasks'), + (SMS_TYPE, True, 'send_sms', 'research-mode-tasks'), + (EMAIL_TYPE, False, 'send_email', 'database-tasks'), + (EMAIL_TYPE, True, 'send_email', 'research-mode-tasks'), + (LETTER_TYPE, False, 'persist_letter', 'database-tasks'), + (LETTER_TYPE, True, 'persist_letter', 'research-mode-tasks'), ]) def test_process_row_sends_letter_task(template_type, research_mode, expected_function, expected_queue, mocker): mocker.patch('app.celery.tasks.create_uuid', return_value='noti_uuid') @@ -420,7 +420,7 @@ def test_should_send_template_to_correct_sms_task_and_persist(sample_template_wi assert persisted_notification.notification_type == 'sms' mocked_deliver_sms.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -446,7 +446,7 @@ def test_should_put_send_sms_task_in_research_mode_queue_if_research_mode_servic persisted_notification = Notification.query.one() provider_tasks.deliver_sms.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="research-mode" + queue="research-mode-tasks" ) assert mocked_deliver_sms.called @@ -481,7 +481,7 @@ def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notif assert persisted_notification.notification_type == 'sms' provider_tasks.deliver_sms.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -507,7 +507,7 @@ def test_should_send_sms_if_restricted_service_and_non_team_number_with_test_key persisted_notification = Notification.query.one() mocked_deliver_sms.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -535,7 +535,7 @@ def test_should_send_email_if_restricted_service_and_non_team_email_address_with persisted_notification = Notification.query.one() mocked_deliver_email.assert_called_once_with( [str(persisted_notification.id)], - queue="send-email" + queue="send-tasks" ) @@ -602,7 +602,7 @@ def test_should_put_send_email_task_in_research_mode_queue_if_research_mode_serv persisted_notification = Notification.query.one() provider_tasks.deliver_email.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="research-mode" + queue="research-mode-tasks" ) @@ -639,7 +639,7 @@ def test_should_send_sms_template_to_and_persist_with_job_id(sample_job, sample_ provider_tasks.deliver_sms.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -736,7 +736,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue='send-email') + [str(persisted_notification.id)], queue='send-tasks') def test_send_email_should_use_template_version_from_job_not_latest(sample_email_template, mocker): @@ -767,7 +767,7 @@ def test_send_email_should_use_template_version_from_job_not_latest(sample_email assert not persisted_notification.sent_by assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)], - queue='send-email') + queue='send-tasks') def test_should_use_email_template_subject_placeholders(sample_email_template_with_placeholders, mocker): @@ -793,7 +793,7 @@ def test_should_use_email_template_subject_placeholders(sample_email_template_wi assert not persisted_notification.reference assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue='send-email' + [str(persisted_notification.id)], queue='send-tasks' ) @@ -821,7 +821,7 @@ def test_should_use_email_template_and_persist_without_personalisation(sample_em assert not persisted_notification.reference assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)], - queue='send-email') + queue='send-tasks') def test_send_sms_should_go_to_retry_queue_if_database_errors(sample_template, mocker): @@ -844,7 +844,7 @@ def test_send_sms_should_go_to_retry_queue_if_database_errors(sample_template, m now.strftime(DATETIME_FORMAT) ) assert provider_tasks.deliver_sms.apply_async.called is False - tasks.send_sms.retry.assert_called_with(exc=expected_exception, queue='retry') + tasks.send_sms.retry.assert_called_with(exc=expected_exception, queue="retry-tasks") assert Notification.query.count() == 0 @@ -869,7 +869,7 @@ def test_send_email_should_go_to_retry_queue_if_database_errors(sample_email_tem now.strftime(DATETIME_FORMAT) ) assert not provider_tasks.deliver_email.apply_async.called - tasks.send_email.retry.assert_called_with(exc=expected_exception, queue='retry') + tasks.send_email.retry.assert_called_with(exc=expected_exception, queue="retry-tasks") assert Notification.query.count() == 0 @@ -1002,7 +1002,6 @@ def test_build_dvla_file(sample_letter_template, mocker): file_location="{}-dvla-job.text".format(job.id) ) assert Job.query.get(job.id).job_status == 'ready to send' - mocked_send_task.assert_called_once_with("aggregrate-dvla-files", ([str(job.id)], ), queue='aggregate-dvla-files') def test_build_dvla_file_retries_if_all_notifications_are_not_created(sample_letter_template, mocker): @@ -1016,7 +1015,7 @@ def test_build_dvla_file_retries_if_all_notifications_are_not_created(sample_let build_dvla_file(job.id) mocked.assert_not_called() - tasks.build_dvla_file.retry.assert_called_with(queue='retry', + tasks.build_dvla_file.retry.assert_called_with(queue="retry-tasks", exc="All notifications for job {} are not persisted".format(job.id)) assert Job.query.get(job.id).job_status == 'in progress' mocked_send_task.assert_not_called() diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 5b1d32144..91eff00fc 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -1,11 +1,12 @@ -import requests_mock -import pytest +import json import uuid from datetime import (datetime, date, timedelta) +import requests_mock +import pytest from sqlalchemy import asc from sqlalchemy.orm.session import make_transient -from flask import current_app +from flask import current_app, url_for from app import db from app.models import ( @@ -35,6 +36,7 @@ from app.dao.invited_user_dao import save_invited_user from app.dao.provider_rates_dao import create_provider_rates from app.clients.sms.firetext import FiretextClient +from tests import create_authorization_header from tests.app.db import create_user, create_template, create_notification @@ -278,16 +280,18 @@ def sample_team_api_key(notify_db, notify_db_session, service=None): @pytest.fixture(scope='function') -def sample_job(notify_db, - notify_db_session, - service=None, - template=None, - notification_count=1, - created_at=None, - job_status='pending', - scheduled_for=None, - processing_started=None, - original_file_name='some.csv'): +def sample_job( + notify_db, + notify_db_session, + service=None, + template=None, + notification_count=1, + created_at=None, + job_status='pending', + scheduled_for=None, + processing_started=None, + original_file_name='some.csv' +): if service is None: service = sample_service(notify_db, notify_db_session) if template is None: @@ -976,3 +980,41 @@ def restore_provider_details(notify_db, notify_db_session): notify_db.session.add_all(existing_provider_details) notify_db.session.add_all(existing_provider_details_history) notify_db.session.commit() + + +@pytest.fixture +def admin_request(client): + class AdminRequest: + + @staticmethod + def get(endpoint, endpoint_kwargs=None, expected_status=200): + resp = client.get( + url_for(endpoint, **(endpoint_kwargs or {})), + headers=[create_authorization_header()] + ) + json_resp = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == expected_status + return json_resp + + @staticmethod + def post(endpoint, endpoint_kwargs=None, data=None, expected_status=200): + resp = client.post( + url_for(endpoint, **(endpoint_kwargs or {})), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), create_authorization_header()] + ) + json_resp = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == expected_status + return json_resp + + @staticmethod + def delete(endpoint, endpoint_kwargs=None, expected_status=204): + resp = client.delete( + url_for(endpoint, **(endpoint_kwargs or {})), + headers=[create_authorization_header()] + ) + json_resp = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == expected_status + return json_resp + + return AdminRequest diff --git a/tests/app/dao/test_inbound_sms_dao.py b/tests/app/dao/test_inbound_sms_dao.py new file mode 100644 index 000000000..b26dc913e --- /dev/null +++ b/tests/app/dao/test_inbound_sms_dao.py @@ -0,0 +1,97 @@ +from datetime import datetime, timedelta + +from freezegun import freeze_time + +from app.dao.inbound_sms_dao import ( + dao_get_inbound_sms_for_service, + dao_count_inbound_sms_for_service, + delete_inbound_sms_created_more_than_a_week_ago, + dao_get_inbound_sms_by_id +) +from tests.app.db import create_inbound_sms, create_service + +from app.models import InboundSms + + +def test_get_all_inbound_sms(sample_service): + inbound = create_inbound_sms(sample_service) + + res = dao_get_inbound_sms_for_service(sample_service.id) + assert len(res) == 1 + assert res[0] == inbound + + +def test_get_all_inbound_sms_when_none_exist(sample_service): + res = dao_get_inbound_sms_for_service(sample_service.id) + assert len(res) == 0 + + +def test_get_all_inbound_sms_limits_and_orders(sample_service): + with freeze_time('2017-01-01'): + one = create_inbound_sms(sample_service) + with freeze_time('2017-01-03'): + three = create_inbound_sms(sample_service) + with freeze_time('2017-01-02'): + two = create_inbound_sms(sample_service) + + res = dao_get_inbound_sms_for_service(sample_service.id, limit=2) + assert len(res) == 2 + assert res[0] == three + assert res[0].created_at == datetime(2017, 1, 3) + assert res[1] == two + assert res[1].created_at == datetime(2017, 1, 2) + + +def test_get_all_inbound_sms_filters_on_service(notify_db_session): + service_one = create_service(service_name='one') + service_two = create_service(service_name='two') + + sms_one = create_inbound_sms(service_one) + sms_two = create_inbound_sms(service_two) + + res = dao_get_inbound_sms_for_service(service_one.id) + assert len(res) == 1 + assert res[0] == sms_one + + +def test_count_inbound_sms_for_service(notify_db_session): + service_one = create_service(service_name='one') + service_two = create_service(service_name='two') + + create_inbound_sms(service_one) + create_inbound_sms(service_one) + create_inbound_sms(service_two) + + assert dao_count_inbound_sms_for_service(service_one.id) == 2 + + +@freeze_time("2017-01-01 12:00:00") +def test_should_delete_inbound_sms_older_than_seven_days(sample_service): + older_than_seven_days = datetime.utcnow() - timedelta(days=7, seconds=1) + create_inbound_sms(sample_service, created_at=older_than_seven_days) + delete_inbound_sms_created_more_than_a_week_ago() + + assert len(InboundSms.query.all()) == 0 + + +@freeze_time("2017-01-01 12:00:00") +def test_should_not_delete_inbound_sms_before_seven_days(sample_service): + yesterday = datetime.utcnow() - timedelta(days=1) + just_before_seven_days = datetime.utcnow() - timedelta(days=6, hours=23, minutes=59, seconds=59) + older_than_seven_days = datetime.utcnow() - timedelta(days=7, seconds=1) + + create_inbound_sms(sample_service, created_at=yesterday) + create_inbound_sms(sample_service, created_at=just_before_seven_days) + create_inbound_sms(sample_service, created_at=older_than_seven_days) + + delete_inbound_sms_created_more_than_a_week_ago() + + assert len(InboundSms.query.all()) == 2 + + +def test_get_inbound_sms_by_id_returns(sample_service): + inbound = create_inbound_sms(sample_service) + + inbound_from_db = dao_get_inbound_sms_by_id(sample_service.id, inbound.id) + + assert inbound == inbound_from_db diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index 034048a32..42f93d230 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta from functools import partial +import pytest import uuid from freezegun import freeze_time @@ -16,7 +17,10 @@ from app.dao.jobs_dao import ( dao_update_job_status, dao_get_all_notifications_for_job, dao_get_jobs_older_than_limited_by) -from app.models import Job, JobStatistics +from app.models import ( + Job, JobStatistics, + EMAIL_TYPE, SMS_TYPE, LETTER_TYPE +) from tests.app.conftest import sample_notification as create_notification from tests.app.conftest import sample_job as create_job @@ -284,33 +288,30 @@ def test_get_future_scheduled_job_gets_a_job_yet_to_send(sample_scheduled_job): assert result.id == sample_scheduled_job.id -def test_should_get_jobs_seven_days_old(notify_db, notify_db_session): - # job runs at some point on each day - # shouldn't matter when, we are deleting things 7 days ago - job_run_time = '2016-10-31T10:00:00' +@freeze_time('2016-10-31 10:00:00') +def test_should_get_jobs_seven_days_old(notify_db, notify_db_session, sample_template): + """ + Jobs older than seven days are deleted, but only two day's worth (two-day window) + """ + seven_days_ago = datetime.utcnow() - timedelta(days=7) + within_seven_days = seven_days_ago + timedelta(seconds=1) - # running on the 31st means the previous 7 days are ignored + eight_days_ago = seven_days_ago - timedelta(days=1) - # 2 day window for delete jobs - # 7 days of files to skip includes the 30,29,28,27,26,25,24th, so the.... - last_possible_time_for_eligible_job = '2016-10-23T23:59:59' - first_possible_time_for_eligible_job = '2016-10-22T00:00:00' + nine_days_ago = eight_days_ago - timedelta(days=2) + nine_days_one_second_ago = nine_days_ago - timedelta(seconds=1) - job_1 = create_job(notify_db, notify_db_session, created_at=last_possible_time_for_eligible_job) - job_2 = create_job(notify_db, notify_db_session, created_at=first_possible_time_for_eligible_job) + job = partial(create_job, notify_db, notify_db_session) + job(created_at=seven_days_ago) + job(created_at=within_seven_days) + job_to_delete = job(created_at=eight_days_ago) + job(created_at=nine_days_ago) + job(created_at=nine_days_one_second_ago) - # bookmarks for jobs that should be ignored - last_possible_time_for_ineligible_job = '2016-10-24T00:00:00' - create_job(notify_db, notify_db_session, created_at=last_possible_time_for_ineligible_job) + jobs = dao_get_jobs_older_than_limited_by(job_types=[sample_template.template_type]) - first_possible_time_for_ineligible_job = '2016-10-21T23:59:59' - create_job(notify_db, notify_db_session, created_at=first_possible_time_for_ineligible_job) - - with freeze_time(job_run_time): - jobs = dao_get_jobs_older_than_limited_by() - assert len(jobs) == 2 - assert jobs[0].id == job_1.id - assert jobs[1].id == job_2.id + assert len(jobs) == 1 + assert jobs[0].id == job_to_delete.id def test_get_jobs_for_service_is_paginated(notify_db, notify_db_session, sample_service, sample_template): @@ -334,13 +335,24 @@ def test_get_jobs_for_service_is_paginated(notify_db, notify_db_session, sample_ assert res.items[1].created_at == datetime(2015, 1, 1, 7) -def test_get_jobs_for_service_doesnt_return_test_messages(notify_db, notify_db_session, sample_template, sample_job): +@pytest.mark.parametrize('file_name', [ + 'Test message', + 'Report', +]) +def test_get_jobs_for_service_doesnt_return_test_messages( + notify_db, + notify_db_session, + sample_template, + sample_job, + file_name, +): test_job = create_job( notify_db, notify_db_session, sample_template.service, sample_template, - original_file_name='Test message') + original_file_name=file_name, + ) jobs = dao_get_jobs_by_service_id(sample_job.service_id).items @@ -379,3 +391,23 @@ def test_dao_update_job_status(sample_job): updated_job = Job.query.get(sample_job.id) assert updated_job.job_status == 'sent to dvla' assert updated_job.updated_at + + +@freeze_time('2016-10-31 10:00:00') +def test_should_get_jobs_seven_days_old_filters_type(notify_db, notify_db_session): + eight_days_ago = datetime.utcnow() - timedelta(days=8) + letter_template = create_template(notify_db, notify_db_session, template_type=LETTER_TYPE) + sms_template = create_template(notify_db, notify_db_session, template_type=SMS_TYPE) + email_template = create_template(notify_db, notify_db_session, template_type=EMAIL_TYPE) + + job = partial(create_job, notify_db, notify_db_session, created_at=eight_days_ago) + job_to_remain = job(template=letter_template) + job(template=sms_template) + job(template=email_template) + + jobs = dao_get_jobs_older_than_limited_by( + job_types=[EMAIL_TYPE, SMS_TYPE] + ) + + assert len(jobs) == 2 + assert job_to_remain.id not in [job.id for job in jobs] diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index a6b69cd24..dc0f3fc34 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -1772,6 +1772,20 @@ def test_dao_get_notifications_by_to_field_search_is_not_case_sensitive(sample_t assert notification.id in notification_ids +@pytest.mark.parametrize('to', [ + 'not@email', '123' +]) +def test_dao_get_notifications_by_to_field_accepts_invalid_phone_numbers_and_email_addresses( + sample_template, + to, +): + notification = create_notification( + template=sample_template, to_field='test@example.com', normalised_to='test@example.com' + ) + results = dao_get_notifications_by_to_field(notification.service_id, to) + assert len(results) == 0 + + def test_dao_get_notifications_by_to_field_search_ignores_spaces(sample_template): notification1 = create_notification( template=sample_template, to_field='+447700900855', normalised_to='447700900855' diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py index 5252b0f90..83bfa264c 100644 --- a/tests/app/dao/test_notification_usage_dao.py +++ b/tests/app/dao/test_notification_usage_dao.py @@ -1,11 +1,27 @@ import uuid -from datetime import datetime +from datetime import datetime, timedelta + +import pytest +from flask import current_app from app.dao.date_util import get_financial_year -from app.dao.notification_usage_dao import (get_rates_for_year, get_yearly_billing_data, - get_monthly_billing_data) -from app.models import Rate +from app.dao.notification_usage_dao import ( + get_rates_for_year, + get_yearly_billing_data, + get_monthly_billing_data, + get_total_billable_units_for_sent_sms_notifications_in_date_range, + discover_rate_bounds_for_billing_query +) +from app.models import ( + Rate, + NOTIFICATION_DELIVERED, + NOTIFICATION_STATUS_TYPES_BILLABLE, + NOTIFICATION_STATUS_TYPES_NON_BILLABLE) +from tests.app.conftest import sample_notification, sample_email_template, sample_letter_template, sample_service from tests.app.db import create_notification +from freezegun import freeze_time + +from tests.conftest import set_config def test_get_rates_for_year(notify_db, notify_db_session): @@ -248,3 +264,449 @@ def test_get_monthly_billing_data_with_no_notifications_for_year(notify_db, noti def set_up_rate(notify_db, start_date, value): rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms') notify_db.session.add(rate) + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications(notify_db, notify_db_session, sample_service): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + + set_up_rate(notify_db, datetime(2016, 1, 1), 0.016) + + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=1, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=2, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=3, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=4, status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id)[0] == 10 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id)[1] == 0.16 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_multiplied_by_multipler_for_sms_notifications( + notify_db, notify_db_session, sample_service +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=1.0, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=2.0, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=5.0, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=10.0, status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 18 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 45 + + +def test_returns_total_billable_units_multiplied_by_multipler_for_sms_notifications_for_several_rates( + notify_db, notify_db_session, sample_service +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + + set_up_rate(notify_db, datetime(2016, 1, 1), 2) + set_up_rate(notify_db, datetime(2016, 10, 1), 4) + set_up_rate(notify_db, datetime(2017, 1, 1), 6) + + eligble_rate_1 = datetime(2016, 2, 1) + eligble_rate_2 = datetime(2016, 11, 1) + eligble_rate_3 = datetime(2017, 2, 1) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=1.0, + status=NOTIFICATION_DELIVERED, + created_at=eligble_rate_1) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=2.0, + status=NOTIFICATION_DELIVERED, + created_at=eligble_rate_2) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=5.0, + status=NOTIFICATION_DELIVERED, + created_at=eligble_rate_3) + + start = datetime(2016, 1, 1) + end = datetime(2018, 1, 1) + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 8 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 40 + + +def test_returns_total_billable_units_for_sms_notifications_for_several_rates_where_dates_match_rate_boundary( + notify_db, notify_db_session, sample_service +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + + set_up_rate(notify_db, datetime(2016, 1, 1), 2) + set_up_rate(notify_db, datetime(2016, 10, 1), 4) + set_up_rate(notify_db, datetime(2017, 1, 1), 6) + + eligble_rate_1_start = datetime(2016, 1, 1, 0, 0, 0, 0) + eligble_rate_1_end = datetime(2016, 9, 30, 23, 59, 59, 999) + eligble_rate_2_start = datetime(2016, 10, 1, 0, 0, 0, 0) + eligble_rate_2_end = datetime(2016, 12, 31, 23, 59, 59, 999) + eligble_rate_3_start = datetime(2017, 1, 1, 0, 0, 0, 0) + eligble_rate_3_whenever = datetime(2017, 12, 12, 0, 0, 0, 0) + + def make_notification(created_at): + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=1.0, + status=NOTIFICATION_DELIVERED, + created_at=created_at) + + make_notification(eligble_rate_1_start) + make_notification(eligble_rate_1_end) + make_notification(eligble_rate_2_start) + make_notification(eligble_rate_2_end) + make_notification(eligble_rate_3_start) + make_notification(eligble_rate_3_whenever) + + start = datetime(2016, 1, 1) + end = datetime(2018, 1, 1) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id)[0] == 6 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id)[1] == 24.0 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications_ignoring_letters_and_emails( + notify_db, notify_db_session, sample_service +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + email_template = sample_email_template(notify_db, notify_db_session, service=sample_service) + letter_template = sample_letter_template(sample_service) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + template=email_template, + service=sample_service, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + template=letter_template, + service=sample_service, + billable_units=2, + status=NOTIFICATION_DELIVERED + ) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 2 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 5 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications_for_only_requested_service( + notify_db, notify_db_session +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + service_1 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4())) + service_2 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4())) + service_3 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4())) + + sample_notification( + notify_db, + notify_db_session, + service=service_1, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + service=service_2, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + service=service_3, + billable_units=2, + status=NOTIFICATION_DELIVERED + ) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[0] == 2 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[1] == 5 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications_handling_null_values( + notify_db, notify_db_session, sample_service +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=2, + rate_multiplier=None, + status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 2 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 5 + + +@pytest.mark.parametrize('billable_units, states', ([ + (len(NOTIFICATION_STATUS_TYPES_BILLABLE), NOTIFICATION_STATUS_TYPES_BILLABLE), + (0, NOTIFICATION_STATUS_TYPES_NON_BILLABLE) +])) +@freeze_time("2016-01-10 12:00:00.000000") +def test_ignores_non_billable_states_when_returning_billable_units_for_sms_notifications( + notify_db, notify_db_session, sample_service, billable_units, states +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + for state in states: + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=1, + rate_multiplier=None, + status=state) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id + )[0] == billable_units + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id + )[1] == billable_units * 2.5 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_restricts_to_time_period_when_returning_billable_units_for_sms_notifications( + notify_db, notify_db_session, sample_service +): + with set_config(current_app, 'FREE_SMS_TIER_FRAGMENT_COUNT', 0): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=1, + rate_multiplier=1.0, + created_at=datetime.utcnow() - timedelta(minutes=100), + status=NOTIFICATION_DELIVERED) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=1, + rate_multiplier=1.0, + created_at=datetime.utcnow() - timedelta(minutes=5), + status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id)[0] == 1 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id)[1] == 2.5 + + +def test_returns_zero_if_no_matching_rows_when_returning_billable_units_for_sms_notifications( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 0 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 0.0 + + +def test_should_calculate_rate_boundaries_for_billing_query_for_single_relevant_rate(notify_db, notify_db_session): + start_date, end_date = get_financial_year(2016) + set_up_rate(notify_db, datetime(2016, 1, 1), 0.016) + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + assert len(rate_boundaries) == 1 + assert rate_boundaries[0]['start_date'] == start_date + assert rate_boundaries[0]['end_date'] == end_date + assert rate_boundaries[0]['rate'] == 0.016 + + +def test_should_calculate_rate_boundaries_for_billing_query_for_two_relevant_rates(notify_db, notify_db_session): + start_date, end_date = get_financial_year(2016) + + rate_1_valid_from = datetime(2016, 1, 1) + rate_2_valid_from = datetime(2017, 1, 1) + + set_up_rate(notify_db, rate_1_valid_from, 0.02) + set_up_rate(notify_db, rate_2_valid_from, 0.04) + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + assert len(rate_boundaries) == 2 + assert rate_boundaries[0]['start_date'] == start_date + assert rate_boundaries[0]['end_date'] == rate_2_valid_from + assert rate_boundaries[0]['rate'] == 0.02 + + assert rate_boundaries[1]['start_date'] == rate_2_valid_from + assert rate_boundaries[1]['end_date'] == end_date + assert rate_boundaries[1]['rate'] == 0.04 + + +def test_should_calculate_rate_boundaries_for_billing_query_for_three_relevant_rates(notify_db, notify_db_session): + start_date, end_date = get_financial_year(2016) + rate_1_valid_from = datetime(2016, 1, 1) + rate_2_valid_from = datetime(2017, 1, 1) + rate_3_valid_from = datetime(2017, 2, 1) + + set_up_rate(notify_db, rate_1_valid_from, 0.02) + set_up_rate(notify_db, rate_2_valid_from, 0.04) + set_up_rate(notify_db, rate_3_valid_from, 0.06) + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + assert len(rate_boundaries) == 3 + + assert rate_boundaries[0]['start_date'] == start_date + assert rate_boundaries[0]['end_date'] == rate_2_valid_from + assert rate_boundaries[0]['rate'] == 0.02 + + assert rate_boundaries[1]['start_date'] == rate_2_valid_from + assert rate_boundaries[1]['end_date'] == rate_3_valid_from + assert rate_boundaries[1]['rate'] == 0.04 + + assert rate_boundaries[2]['start_date'] == rate_3_valid_from + assert rate_boundaries[2]['end_date'] == end_date + assert rate_boundaries[2]['rate'] == 0.06 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_deducts_free_tier_from_bill( + notify_db, notify_db_session +): + start_value = current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] + try: + current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] = 1 + + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + service_1 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4())) + + sample_notification( + notify_db, + notify_db_session, + service=service_1, + billable_units=1, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + service=service_1, + billable_units=1, + status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[0] == 2 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[1] == 2.5 + finally: + current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] = start_value + + +@freeze_time("2016-01-10 12:00:00.000000") +@pytest.mark.parametrize( + 'free_tier, expected_cost', + [(0, 24.0), (1, 22.0), (2, 20.0), (3, 16.0), (4, 12.0), (5, 6.0), (6, 0.0)] +) +def test_deducts_free_tier_from_bill_across_rate_boundaries( + notify_db, notify_db_session, sample_service, free_tier, expected_cost +): + start_value = current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] + try: + current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] = free_tier + set_up_rate(notify_db, datetime(2016, 1, 1), 2) + set_up_rate(notify_db, datetime(2016, 10, 1), 4) + set_up_rate(notify_db, datetime(2017, 1, 1), 6) + + eligble_rate_1_start = datetime(2016, 1, 1, 0, 0, 0, 0) + eligble_rate_1_end = datetime(2016, 9, 30, 23, 59, 59, 999) + eligble_rate_2_start = datetime(2016, 10, 1, 0, 0, 0, 0) + eligble_rate_2_end = datetime(2016, 12, 31, 23, 59, 59, 999) + eligble_rate_3_start = datetime(2017, 1, 1, 0, 0, 0, 0) + eligble_rate_3_whenever = datetime(2017, 12, 12, 0, 0, 0, 0) + + def make_notification(created_at): + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=1.0, + status=NOTIFICATION_DELIVERED, + created_at=created_at) + + make_notification(eligble_rate_1_start) + make_notification(eligble_rate_1_end) + make_notification(eligble_rate_2_start) + make_notification(eligble_rate_2_end) + make_notification(eligble_rate_3_start) + make_notification(eligble_rate_3_whenever) + + start = datetime(2016, 1, 1) + end = datetime(2018, 1, 1) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 6 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id + )[1] == expected_cost + finally: + current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] = start_value diff --git a/tests/app/dao/test_permissionDAO.py b/tests/app/dao/test_permissions_dao.py similarity index 100% rename from tests/app/dao/test_permissionDAO.py rename to tests/app/dao/test_permissions_dao.py diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index b7b3176dd..800852ca2 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -25,7 +25,8 @@ from app.dao.services_dao import ( fetch_stats_by_date_range_for_all_services, dao_suspend_service, dao_resume_service, - dao_fetch_active_users_for_service + dao_fetch_active_users_for_service, + dao_fetch_services_by_sms_sender ) from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission from app.dao.users_dao import save_model_user @@ -948,3 +949,13 @@ def test_dao_fetch_active_users_for_service_returns_active_only(notify_db, notif users = dao_fetch_active_users_for_service(service.id) assert len(users) == 1 + + +def test_dao_fetch_services_by_sms_sender(notify_db_session): + foo1 = create_service(service_name='a', sms_sender='foo') + foo2 = create_service(service_name='b', sms_sender='foo') + bar = create_service(service_name='c', sms_sender='bar') + + services = dao_fetch_services_by_sms_sender('foo') + + assert {foo1.id, foo2.id} == {x.id for x in services} diff --git a/tests/app/dao/test_statistics_dao.py b/tests/app/dao/test_statistics_dao.py index 4425a3409..5a73ea672 100644 --- a/tests/app/dao/test_statistics_dao.py +++ b/tests/app/dao/test_statistics_dao.py @@ -199,6 +199,10 @@ def test_should_update_a_stats_entry_with_its_success_outcome_for_a_job( assert stat.sms_failed == 0 assert stat.letters_failed == 0 + assert stat.sent == email_count + sms_count + letter_count + assert stat.delivered == email_count + sms_count + assert stat.failed == 0 + @pytest.mark.parametrize('notification_type, sms_count, email_count, letter_count, status', [ (SMS_TYPE, 1, 0, 0, NOTIFICATION_TECHNICAL_FAILURE), @@ -264,6 +268,10 @@ def test_should_update_a_stats_entry_with_its_error_outcomes_for_a_job( assert stat.emails_delivered == 0 assert stat.sms_delivered == 0 + assert stat.sent == email_count + sms_count + letter_count + assert stat.delivered == 0 + assert stat.failed == email_count + sms_count + letter_count + @pytest.mark.parametrize('notification_type, sms_count, email_count, letter_count, status', [ (SMS_TYPE, 1, 0, 0, NOTIFICATION_DELIVERED), @@ -326,6 +334,10 @@ def test_should_update_a_stats_entry_with_its_success_outcomes_for_a_job( assert stat.emails_delivered == email_count assert stat.sms_delivered == sms_count + assert stat.sent == email_count + sms_count + letter_count + assert stat.delivered == 0 if notification_type == LETTER_TYPE else 1 + assert stat.failed == 0 + @pytest.mark.parametrize('notification_type, sms_count, email_count, letter_count, status', [ (SMS_TYPE, 1, 0, 0, NOTIFICATION_PENDING), @@ -394,6 +406,10 @@ def test_should_not_update_job_stats_if_irrelevant_status( assert stat.emails_delivered == 0 assert stat.sms_delivered == 0 + assert stat.sent == email_count + sms_count + letter_count + assert stat.delivered == 0 + assert stat.failed == 0 + @pytest.mark.parametrize('notification_type, sms_count, email_count, letter_count', [ (SMS_TYPE, 2, 1, 1), @@ -480,41 +496,52 @@ def test_inserting_one_type_of_notification_maintains_other_counts( assert updated_stats[0].sms_sent == sms_count assert updated_stats[0].letters_sent == letter_count + if notification_type == EMAIL_TYPE: + assert updated_stats[0].sent == email_count + elif notification_type == SMS_TYPE: + assert updated_stats[0].sent == sms_count + elif notification_type == LETTER_TYPE: + assert updated_stats[0].sent == letter_count + def test_updating_one_type_of_notification_to_success_maintains_other_counts( notify_db, notify_db_session, - sample_job, + sample_service, sample_letter_template ): - sms_template = sample_template(notify_db, notify_db_session, service=sample_job.service) - email_template = sample_email_template(notify_db, notify_db_session, service=sample_job.service) + job_1 = sample_job(notify_db, notify_db_session, service=sample_service) + job_2 = sample_job(notify_db, notify_db_session, service=sample_service) + job_3 = sample_job(notify_db, notify_db_session, service=sample_service) + + sms_template = sample_template(notify_db, notify_db_session, service=sample_service) + email_template = sample_email_template(notify_db, notify_db_session, service=sample_service) letter_template = sample_letter_template letter = sample_notification( notify_db, notify_db_session, - service=sample_job.service, + service=sample_service, template=letter_template, - job=sample_job, + job=job_1, status=NOTIFICATION_CREATED ) email = sample_notification( notify_db, notify_db_session, - service=sample_job.service, + service=sample_service, template=email_template, - job=sample_job, + job=job_2, status=NOTIFICATION_CREATED ) sms = sample_notification( notify_db, notify_db_session, - service=sample_job.service, + service=sample_service, template=sms_template, - job=sample_job, + job=job_3, status=NOTIFICATION_CREATED ) @@ -530,49 +557,76 @@ def test_updating_one_type_of_notification_to_success_maintains_other_counts( update_job_stats_outcome_count(email) update_job_stats_outcome_count(sms) - stats = JobStatistics.query.all() - assert len(stats) == 1 - assert stats[0].emails_sent == 1 - assert stats[0].sms_sent == 1 + stats = JobStatistics.query.order_by(JobStatistics.created_at).all() + assert len(stats) == 3 assert stats[0].letters_sent == 1 - assert stats[0].emails_delivered == 1 - assert stats[0].sms_delivered == 1 + assert stats[0].emails_sent == 0 + assert stats[0].sms_sent == 0 + assert stats[0].emails_delivered == 0 + assert stats[0].sms_delivered == 0 + + assert stats[1].letters_sent == 0 + assert stats[1].emails_sent == 1 + assert stats[1].sms_sent == 0 + assert stats[1].emails_delivered == 1 + assert stats[1].sms_delivered == 0 + + assert stats[2].letters_sent == 0 + assert stats[2].emails_sent == 0 + assert stats[2].sms_sent == 1 + assert stats[2].emails_delivered == 0 + assert stats[2].sms_delivered == 1 + + assert stats[0].sent == 1 + assert stats[0].delivered == 0 + assert stats[0].failed == 0 + + assert stats[1].sent == 1 + assert stats[1].delivered == 1 + assert stats[1].failed == 0 + + assert stats[2].sent == 1 + assert stats[2].delivered == 1 + assert stats[2].failed == 0 def test_updating_one_type_of_notification_to_error_maintains_other_counts( notify_db, notify_db_session, - sample_job, + sample_service, sample_letter_template ): - sms_template = sample_template(notify_db, notify_db_session, service=sample_job.service) - email_template = sample_email_template(notify_db, notify_db_session, service=sample_job.service) + job_1 = sample_job(notify_db, notify_db_session, service=sample_service) + job_2 = sample_job(notify_db, notify_db_session, service=sample_service) + job_3 = sample_job(notify_db, notify_db_session, service=sample_service) + sms_template = sample_template(notify_db, notify_db_session, service=sample_service) + email_template = sample_email_template(notify_db, notify_db_session, service=sample_service) letter_template = sample_letter_template letter = sample_notification( notify_db, notify_db_session, - service=sample_job.service, + service=sample_service, template=letter_template, - job=sample_job, + job=job_1, status=NOTIFICATION_CREATED ) email = sample_notification( notify_db, notify_db_session, - service=sample_job.service, + service=sample_service, template=email_template, - job=sample_job, + job=job_2, status=NOTIFICATION_CREATED ) sms = sample_notification( notify_db, notify_db_session, - service=sample_job.service, + service=sample_service, template=sms_template, - job=sample_job, + job=job_3, status=NOTIFICATION_CREATED ) @@ -588,20 +642,50 @@ def test_updating_one_type_of_notification_to_error_maintains_other_counts( update_job_stats_outcome_count(email) update_job_stats_outcome_count(sms) - stats = JobStatistics.query.all() - assert len(stats) == 1 - assert stats[0].emails_sent == 1 - assert stats[0].sms_sent == 1 + stats = JobStatistics.query.order_by(JobStatistics.created_at).all() + assert len(stats) == 3 + assert stats[0].emails_sent == 0 + assert stats[0].sms_sent == 0 assert stats[0].letters_sent == 1 assert stats[0].emails_delivered == 0 assert stats[0].sms_delivered == 0 - assert stats[0].sms_failed == 1 - assert stats[0].emails_failed == 1 + assert stats[0].sms_failed == 0 + assert stats[0].emails_failed == 0 + assert stats[0].letters_failed == 1 + + assert stats[1].emails_sent == 1 + assert stats[1].sms_sent == 0 + assert stats[1].letters_sent == 0 + assert stats[1].emails_delivered == 0 + assert stats[1].sms_delivered == 0 + assert stats[1].sms_failed == 0 + assert stats[1].emails_failed == 1 + assert stats[1].letters_failed == 0 + + assert stats[2].emails_sent == 0 + assert stats[2].sms_sent == 1 + assert stats[2].letters_sent == 0 + assert stats[2].emails_delivered == 0 + assert stats[2].sms_delivered == 0 + assert stats[2].sms_failed == 1 + assert stats[2].emails_failed == 0 + assert stats[1].letters_failed == 0 + + assert stats[0].sent == 1 + assert stats[0].delivered == 0 + assert stats[0].failed == 1 + + assert stats[1].sent == 1 + assert stats[1].delivered == 0 + assert stats[1].failed == 1 + + assert stats[2].sent == 1 + assert stats[2].delivered == 0 + assert stats[2].failed == 1 -def test_will_not_timeout_job_counts_before_notification_timeouts(notify_db, notify_db_session, sample_job): - sms_template = sample_template(notify_db, notify_db_session, service=sample_job.service) - email_template = sample_email_template(notify_db, notify_db_session, service=sample_job.service) +def test_will_not_timeout_job_counts_before_notification_timeouts(notify_db, notify_db_session, + sample_job, sample_template): one_minute_ago = datetime.utcnow() - timedelta(minutes=1) @@ -609,43 +693,51 @@ def test_will_not_timeout_job_counts_before_notification_timeouts(notify_db, not notify_db, notify_db_session, service=sample_job.service, - template=sms_template, + template=sample_template, job=sample_job, status=NOTIFICATION_CREATED ) - email = sample_notification( + sms_2 = sample_notification( notify_db, notify_db_session, service=sample_job.service, - template=email_template, + template=sample_template, job=sample_job, status=NOTIFICATION_CREATED ) - create_or_update_job_sending_statistics(email) create_or_update_job_sending_statistics(sms) + create_or_update_job_sending_statistics(sms_2) JobStatistics.query.update({JobStatistics.created_at: one_minute_ago}) - intial_stats = JobStatistics.query.all() + initial_stats = JobStatistics.query.all() - assert intial_stats[0].emails_sent == 1 - assert intial_stats[0].sms_sent == 1 - assert intial_stats[0].emails_delivered == 0 - assert intial_stats[0].sms_delivered == 0 - assert intial_stats[0].sms_failed == 0 - assert intial_stats[0].emails_failed == 0 + assert initial_stats[0].emails_sent == 0 + assert initial_stats[0].sms_sent == 2 + assert initial_stats[0].emails_delivered == 0 + assert initial_stats[0].sms_delivered == 0 + assert initial_stats[0].sms_failed == 0 + assert initial_stats[0].emails_failed == 0 + + assert initial_stats[0].sent == 2 + assert initial_stats[0].delivered == 0 + assert initial_stats[0].failed == 0 dao_timeout_job_statistics(61) updated_stats = JobStatistics.query.all() - assert updated_stats[0].emails_sent == 1 - assert updated_stats[0].sms_sent == 1 + assert updated_stats[0].emails_sent == 0 + assert updated_stats[0].sms_sent == 2 assert updated_stats[0].emails_delivered == 0 assert updated_stats[0].sms_delivered == 0 assert updated_stats[0].sms_failed == 0 assert updated_stats[0].emails_failed == 0 + assert initial_stats[0].sent == 2 + assert initial_stats[0].delivered == 0 + assert initial_stats[0].failed == 0 + @pytest.mark.parametrize('notification_type, sms_count, email_count', [ (SMS_TYPE, 3, 0), @@ -688,6 +780,9 @@ def test_timeout_job_counts_timesout_multiple_jobs( assert stats.sms_delivered == 0 assert stats.sms_failed == 0 assert stats.emails_failed == 0 + assert stats.sent == email_count + sms_count + assert stats.delivered == 0 + assert stats.failed == 0 dao_timeout_job_statistics(1) updated_stats = JobStatistics.query.all() @@ -698,6 +793,9 @@ def test_timeout_job_counts_timesout_multiple_jobs( assert stats.sms_delivered == 0 assert stats.sms_failed == sms_count assert stats.emails_failed == email_count + assert stats.sent == email_count + sms_count + assert stats.delivered == 0 + assert stats.failed == email_count + sms_count count_notifications = len(NOTIFICATION_STATUS_TYPES) @@ -754,17 +852,13 @@ def test_timeout_job_sets_all_non_delivered_emails_to_error_and_doesnt_affect_sm assert initial_stats[0].sms_failed == 0 assert initial_stats[0].emails_failed == 0 - all = JobStatistics.query.all() - for a in all: - print(a) + assert initial_stats[0].sent == count_notifications + assert initial_stats[0].delivered == 0 + assert initial_stats[0].failed == 0 # timeout the notifications dao_timeout_job_statistics(1) - all = JobStatistics.query.all() - for a in all: - print(a) - # after timeout all delivered states are success and ALL other states are failed updated_stats = JobStatistics.query.filter_by(job_id=email_job.id).all() assert updated_stats[0].emails_sent == count_notifications @@ -774,6 +868,10 @@ def test_timeout_job_sets_all_non_delivered_emails_to_error_and_doesnt_affect_sm assert updated_stats[0].sms_failed == 0 assert updated_stats[0].emails_failed == count_error_notifications + assert initial_stats[0].sent == count_notifications + assert initial_stats[0].delivered == count_success_notifications + assert initial_stats[0].failed == count_error_notifications + sms_stats = JobStatistics.query.filter_by(job_id=sms_job.id).all() assert sms_stats[0].emails_sent == 0 assert sms_stats[0].sms_sent == 1 @@ -781,6 +879,9 @@ def test_timeout_job_sets_all_non_delivered_emails_to_error_and_doesnt_affect_sm assert sms_stats[0].sms_delivered == 0 assert sms_stats[0].sms_failed == 1 assert sms_stats[0].emails_failed == 0 + assert sms_stats[0].sent == 1 + assert sms_stats[0].delivered == 0 + assert sms_stats[0].failed == 1 # this test is as above, but for SMS not email @@ -810,6 +911,10 @@ def test_timeout_job_sets_all_non_delivered_states_to_error( assert stats.sms_failed == 0 assert stats.emails_failed == 0 + assert stats.sent == count_notifications + assert stats.delivered == 0 + assert stats.failed == 0 + dao_timeout_job_statistics(1) updated_stats = JobStatistics.query.all() @@ -820,3 +925,7 @@ def test_timeout_job_sets_all_non_delivered_states_to_error( assert stats.sms_delivered == count_success_notifications assert stats.sms_failed == count_error_notifications assert stats.emails_failed == 0 + + assert stats.sent == count_notifications + assert stats.delivered == count_success_notifications + assert stats.failed == count_error_notifications diff --git a/tests/app/db.py b/tests/app/db.py index 2f637cf1b..2e658023d 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -2,8 +2,10 @@ from datetime import datetime import uuid +from app.dao.inbound_sms_dao import dao_create_inbound_sms from app.dao.jobs_dao import dao_create_job from app.models import ( + InboundSms, Service, User, Template, @@ -11,6 +13,7 @@ from app.models import ( ScheduledNotification, ServicePermission, Job, + InboundSms, EMAIL_TYPE, SMS_TYPE, KEY_TYPE_NORMAL, @@ -20,6 +23,7 @@ from app.dao.notifications_dao import dao_create_notification, dao_created_sched from app.dao.templates_dao import dao_create_template from app.dao.services_dao import dao_create_service from app.dao.service_permissions_dao import dao_add_service_permission +from app.dao.inbound_sms_dao import dao_create_inbound_sms def create_user(mobile_number="+447700900986", email="notify@digital.cabinet-office.gov.uk", state='active'): @@ -39,14 +43,20 @@ def create_user(mobile_number="+447700900986", email="notify@digital.cabinet-off def create_service( - user=None, service_name="Sample service", service_id=None, restricted=False, - service_permissions=[EMAIL_TYPE, SMS_TYPE]): + user=None, + service_name="Sample service", + service_id=None, + restricted=False, + service_permissions=[EMAIL_TYPE, SMS_TYPE], + sms_sender='testing' +): service = Service( name=service_name, message_limit=1000, restricted=restricted, email_from=service_name.lower().replace(' ', '.'), - created_by=user or create_user() + created_by=user or create_user(), + sms_sender=sms_sender ) dao_create_service(service, service.created_by, service_id, service_permissions=service_permissions) return service @@ -143,14 +153,15 @@ def create_notification( return notification -def create_job(template, - notification_count=1, - created_at=None, - job_status='pending', - scheduled_for=None, - processing_started=None, - original_file_name='some.csv'): - +def create_job( + template, + notification_count=1, + created_at=None, + job_status='pending', + scheduled_for=None, + processing_started=None, + original_file_name='some.csv' +): data = { 'id': uuid.uuid4(), 'service_id': template.service_id, @@ -177,3 +188,25 @@ def create_service_permission(service_id, permission=EMAIL_TYPE): service_permissions = ServicePermission.query.all() return service_permissions + + +def create_inbound_sms( + service, + notify_number=None, + user_number='447700900111', + provider_date=None, + provider_reference=None, + content='Hello', + created_at=None +): + inbound = InboundSms( + service=service, + created_at=created_at or datetime.utcnow(), + notify_number=notify_number or service.sms_sender, + user_number=user_number, + provider_date=provider_date or datetime.utcnow(), + provider_reference=provider_reference or 'foo', + content=content, + ) + dao_create_inbound_sms(inbound) + return inbound diff --git a/tests/app/delivery/test_rest.py b/tests/app/delivery/test_rest.py index fc51fb508..984bf90e1 100644 --- a/tests/app/delivery/test_rest.py +++ b/tests/app/delivery/test_rest.py @@ -78,7 +78,7 @@ def test_should_call_deliver_sms_task_if_send_sms_to_provider_fails(notify_api, ) app.delivery.send_to_providers.send_sms_to_provider.assert_called_with(sample_notification) app.celery.provider_tasks.deliver_sms.apply_async.assert_called_with( - (str(sample_notification.id)), queue='send-sms' + (str(sample_notification.id)), queue='send-tasks' ) assert response.status_code == 204 @@ -100,6 +100,6 @@ def test_should_call_deliver_email_task_if_send_email_to_provider_fails( ) app.delivery.send_to_providers.send_email_to_provider.assert_called_with(sample_email_notification) app.celery.provider_tasks.deliver_email.apply_async.assert_called_with( - (str(sample_email_notification.id)), queue='send-email' + (str(sample_email_notification.id)), queue='send-tasks' ) assert response.status_code == 204 diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index 5cb467ac3..e1a4ec4a2 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -627,7 +627,6 @@ def test_should_set_international_phone_number_to_sent_status( # if 40604 is actually in DB then treat that as if entered manually ('40604', '40604', 'bar'), # 'testing' is the FROM_NUMBER during unit tests - (None, 'testing', 'Sample service: bar'), ('testing', 'testing', 'Sample service: bar'), ]) def test_should_handle_sms_sender_and_prefix_message( diff --git a/tests/app/inbound_sms/__init__.py b/tests/app/inbound_sms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/app/inbound_sms/test_rest.py b/tests/app/inbound_sms/test_rest.py new file mode 100644 index 000000000..b4c55cd83 --- /dev/null +++ b/tests/app/inbound_sms/test_rest.py @@ -0,0 +1,151 @@ +from datetime import datetime + +import pytest +from freezegun import freeze_time + +from tests.app.db import create_inbound_sms, create_service + + +def test_get_inbound_sms(admin_request, sample_service): + one = create_inbound_sms(sample_service) + two = create_inbound_sms(sample_service) + + json_resp = admin_request.get( + 'inbound_sms.get_inbound_sms_for_service', + endpoint_kwargs={'service_id': sample_service.id} + ) + + sms = json_resp['data'] + + assert len(sms) == 2 + assert {inbound['id'] for inbound in sms} == {str(one.id), str(two.id)} + assert sms[0]['content'] == 'Hello' + assert set(sms[0].keys()) == { + 'id', + 'created_at', + 'service_id', + 'notify_number', + 'user_number', + 'content', + 'provider_date', + 'provider_reference' + } + + +def test_get_inbound_sms_limits(admin_request, sample_service): + with freeze_time('2017-01-01'): + one = create_inbound_sms(sample_service) + with freeze_time('2017-01-02'): + two = create_inbound_sms(sample_service) + + sms = admin_request.get( + 'inbound_sms.get_inbound_sms_for_service', + endpoint_kwargs={'service_id': sample_service.id, 'limit': 1} + ) + + assert len(sms['data']) == 1 + assert sms['data'][0]['id'] == str(two.id) + + +@pytest.mark.parametrize('user_number', [ + '(07700) 900-001', + '+4407700900001', + '447700900001', +]) +def test_get_inbound_sms_filters_user_number(admin_request, sample_service, user_number): + # user_number in the db is international and normalised + one = create_inbound_sms(sample_service, user_number='447700900001') + two = create_inbound_sms(sample_service, user_number='447700900002') + + sms = admin_request.get( + 'inbound_sms.get_inbound_sms_for_service', + endpoint_kwargs={'service_id': sample_service.id, 'user_number': user_number} + ) + + assert len(sms['data']) == 1 + assert sms['data'][0]['id'] == str(one.id) + assert sms['data'][0]['user_number'] == str(one.user_number) + + +def test_get_inbound_sms_filters_international_user_number(admin_request, sample_service): + # user_number in the db is international and normalised + one = create_inbound_sms(sample_service, user_number='12025550104') + two = create_inbound_sms(sample_service) + + sms = admin_request.get( + 'inbound_sms.get_inbound_sms_for_service', + endpoint_kwargs={'service_id': sample_service.id, 'user_number': '+1 (202) 555-0104'} + ) + + assert len(sms['data']) == 1 + assert sms['data'][0]['id'] == str(one.id) + assert sms['data'][0]['user_number'] == str(one.user_number) + + +def test_get_inbound_sms_summary(admin_request, sample_service): + other_service = create_service(service_name='other_service') + with freeze_time('2017-01-01'): + create_inbound_sms(sample_service) + with freeze_time('2017-01-02'): + create_inbound_sms(sample_service) + with freeze_time('2017-01-03'): + create_inbound_sms(other_service) + + summary = admin_request.get( + 'inbound_sms.get_inbound_sms_summary_for_service', + endpoint_kwargs={'service_id': sample_service.id} + ) + + assert summary == { + 'count': 2, + 'most_recent': datetime(2017, 1, 2).isoformat() + } + + +def test_get_inbound_sms_summary_with_no_inbound(admin_request, sample_service): + summary = admin_request.get( + 'inbound_sms.get_inbound_sms_summary_for_service', + endpoint_kwargs={'service_id': sample_service.id} + ) + + assert summary == { + 'count': 0, + 'most_recent': None + } + + +def test_get_inbound_sms_by_id_returns_200(admin_request, sample_service): + inbound = create_inbound_sms(sample_service, user_number='447700900001') + + response = admin_request.get( + 'inbound_sms.get_inbound_by_id', + endpoint_kwargs={ + 'service_id': sample_service.id, + 'inbound_sms_id': inbound.id + } + ) + + assert response['user_number'] == '447700900001' + assert response['service_id'] == str(sample_service.id) + + +def test_get_inbound_sms_by_id_invalid_id_returns_404(admin_request, sample_service): + assert admin_request.get( + 'inbound_sms.get_inbound_by_id', + endpoint_kwargs={ + 'service_id': sample_service.id, + 'inbound_sms_id': 'bar' + }, + expected_status=404 + ) + + +def test_get_inbound_sms_by_id_with_invalid_service_id_returns_404(admin_request, sample_service): + assert admin_request.get( + 'inbound_sms.get_inbound_by_id', + endpoint_kwargs={ + 'service_id': 'foo', + 'inbound_sms_id': '2cfbd6a1-1575-4664-8969-f27be0ea40d9' + }, + expected_status=404 + ) diff --git a/tests/app/invite/test_invite_rest.py b/tests/app/invite/test_invite_rest.py index 182f4b05b..be1f46d82 100644 --- a/tests/app/invite/test_invite_rest.py +++ b/tests/app/invite/test_invite_rest.py @@ -33,7 +33,7 @@ def test_create_invited_user(client, sample_service, mocker, invitation_email_te assert json_resp['data']['id'] notification = Notification.query.first() - mocked.assert_called_once_with([(str(notification.id))], queue="notify") + mocked.assert_called_once_with([(str(notification.id))], queue="notify-internal-tasks") def test_create_invited_user_invalid_email(client, sample_service, mocker): diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index d583d6b31..6ebcb2e89 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -119,7 +119,7 @@ def test_create_unscheduled_job(notify_api, sample_template, mocker, fake_uuid): app.celery.tasks.process_job.apply_async.assert_called_once_with( ([str(fake_uuid)]), - queue="process-job" + queue="job-tasks" ) resp_json = json.loads(response.get_data(as_text=True)) diff --git a/tests/app/letters/test_send_letter_jobs.py b/tests/app/letters/test_send_letter_jobs.py index 7f1d88c85..1d32baf12 100644 --- a/tests/app/letters/test_send_letter_jobs.py +++ b/tests/app/letters/test_send_letter_jobs.py @@ -21,7 +21,7 @@ def test_send_letter_jobs(client, mocker): mock_celery.assert_called_once_with(name="send-files-to-dvla", args=(job_ids['job_ids'],), - queue="process-ftp") + queue="process-ftp-tasks") def test_send_letter_jobs_throws_validation_error(client, mocker): diff --git a/tests/app/notifications/rest/test_callbacks.py b/tests/app/notifications/rest/test_callbacks.py index 2c2ffd9cf..58bbc8b1f 100644 --- a/tests/app/notifications/rest/test_callbacks.py +++ b/tests/app/notifications/rest/test_callbacks.py @@ -68,7 +68,7 @@ def test_dvla_callback_calls_update_letter_notifications_task(client, mocker): assert response.status_code == 200 assert update_task.called - update_task.assert_called_with(['bar.txt'], queue='notify') + update_task.assert_called_with(['bar.txt'], queue='notify-internal-tasks') def test_dvla_callback_does_not_raise_error_parsing_json_for_plaintext_header(client, mocker): diff --git a/tests/app/notifications/rest/test_send_notification.py b/tests/app/notifications/rest/test_send_notification.py index ab527e06e..b491e3d7f 100644 --- a/tests/app/notifications/rest/test_send_notification.py +++ b/tests/app/notifications/rest/test_send_notification.py @@ -127,7 +127,7 @@ def test_send_notification_with_placeholders_replaced(notify_api, sample_email_t mocked.assert_called_once_with( [notification_id], - queue="send-email" + queue="send-tasks" ) assert response.status_code == 201 assert response_data['body'] == u'Hello Jo\nThis is an email from GOV.\u200BUK' @@ -338,7 +338,7 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker response_data = json.loads(response.data)['data'] notification_id = response_data['notification']['id'] - mocked.assert_called_once_with([notification_id], queue='send-sms') + mocked.assert_called_once_with([notification_id], queue='send-tasks') assert response.status_code == 201 assert notification_id assert 'subject' not in response_data @@ -392,7 +392,7 @@ def test_should_allow_valid_email_notification(notify_api, sample_email_template notification_id = response_data['notification']['id'] app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( [notification_id], - queue="send-email" + queue="send-tasks" ) assert response.status_code == 201 @@ -593,7 +593,7 @@ def test_should_send_email_if_team_api_key_and_a_service_user(notify_api, sample data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='send-email') + app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 201 @@ -626,7 +626,9 @@ def test_should_send_sms_to_anyone_with_test_key( data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] ) - app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='research-mode') + app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( + [fake_uuid], queue='research-mode-tasks' + ) assert response.status_code == 201 @@ -660,7 +662,9 @@ def test_should_send_email_to_anyone_with_test_key( headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] ) - app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='research-mode') + app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( + [fake_uuid], queue='research-mode-tasks' + ) assert response.status_code == 201 @@ -685,7 +689,7 @@ def test_should_send_sms_if_team_api_key_and_a_service_user(notify_api, sample_t data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='send-sms') + app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 201 @@ -718,7 +722,7 @@ def test_should_persist_notification(notify_api, sample_template, data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type)) + mocked.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 201 notification = notifications_dao.get_notification_by_id(fake_uuid) @@ -761,7 +765,7 @@ def test_should_delete_notification_and_return_error_if_sqs_fails( data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type)) + mocked.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 500 assert not notifications_dao.get_notification_by_id(fake_uuid) assert not NotificationHistory.query.get(fake_uuid) @@ -1046,7 +1050,7 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori notification_id = response_data['notification']['id'] assert response.status_code == 201 - mocked.assert_called_once_with([notification_id], queue='priority') + mocked.assert_called_once_with([notification_id], queue='priority-tasks') @pytest.mark.parametrize( @@ -1114,7 +1118,7 @@ def test_should_allow_store_original_number_on_sms_notification(client, sample_t response_data = json.loads(response.data)['data'] notification_id = response_data['notification']['id'] - mocked.assert_called_once_with([notification_id], queue='send-sms') + mocked.assert_called_once_with([notification_id], queue='send-tasks') assert response.status_code == 201 assert notification_id notifications = Notification.query.all() diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py index 2f66ef457..3220ef2ed 100644 --- a/tests/app/notifications/test_process_notification.py +++ b/tests/app/notifications/test_process_notification.py @@ -208,17 +208,17 @@ def test_persist_notification_increments_cache_if_key_exists(sample_template, sa @pytest.mark.parametrize('research_mode, requested_queue, expected_queue, notification_type, key_type', - [(True, None, 'research-mode', 'sms', 'normal'), - (True, None, 'research-mode', 'email', 'normal'), - (True, None, 'research-mode', 'email', 'team'), - (False, None, 'send-sms', 'sms', 'normal'), - (False, None, 'send-email', 'email', 'normal'), - (False, None, 'send-sms', 'sms', 'team'), - (False, None, 'research-mode', 'sms', 'test'), - (True, 'notify', 'research-mode', 'email', 'normal'), - (False, 'notify', 'notify', 'sms', 'normal'), - (False, 'notify', 'notify', 'email', 'normal'), - (False, 'notify', 'research-mode', 'sms', 'test')]) + [(True, None, 'research-mode-tasks', 'sms', 'normal'), + (True, None, 'research-mode-tasks', 'email', 'normal'), + (True, None, 'research-mode-tasks', 'email', 'team'), + (False, None, 'send-tasks', 'sms', 'normal'), + (False, None, 'send-tasks', 'email', 'normal'), + (False, None, 'send-tasks', 'sms', 'team'), + (False, None, 'research-mode-tasks', 'sms', 'test'), + (True, 'notify-internal-tasks', 'research-mode-tasks', 'email', 'normal'), + (False, 'notify-internal-tasks', 'notify-internal-tasks', 'sms', 'normal'), + (False, 'notify-internal-tasks', 'notify-internal-tasks', 'email', 'normal'), + (False, 'notify-internal-tasks', 'research-mode-tasks', 'sms', 'test')]) def test_send_notification_to_queue(notify_db, notify_db_session, research_mode, requested_queue, expected_queue, notification_type, key_type, mocker): diff --git a/tests/app/notifications/test_receive_notification.py b/tests/app/notifications/test_receive_notification.py index f325fe6f9..66f743be4 100644 --- a/tests/app/notifications/test_receive_notification.py +++ b/tests/app/notifications/test_receive_notification.py @@ -1,14 +1,28 @@ +from datetime import datetime +from unittest.mock import call + +import pytest from flask import json +from app.notifications.receive_notifications import ( + format_mmg_message, + format_mmg_datetime, + create_inbound_mmg_sms_object, + strip_leading_forty_four +) -def test_receive_notification_returns_received_to_mmg(client): +from app.models import InboundSms +from tests.app.db import create_service + + +def test_receive_notification_returns_received_to_mmg(client, sample_service): data = {"ID": "1234", "MSISDN": "447700900855", "Message": "Some message to notify", "Trigger": "Trigger?", - "Number": "40604", + "Number": "testing", "Channel": "SMS", - "DateReceived": "2012-06-27-12:33:00" + "DateRecieved": "2012-06-27 12:33:00" } response = client.post(path='/notifications/sms/receive/mmg', data=json.dumps(data), @@ -16,3 +30,166 @@ def test_receive_notification_returns_received_to_mmg(client): assert response.status_code == 200 assert response.get_data(as_text=True) == 'RECEIVED' + + +@pytest.mark.parametrize('message, expected_output', [ + ('abc', 'abc'), + ('', ''), + ('lots+of+words', 'lots of words'), + ('%F0%9F%93%A9+%F0%9F%93%A9+%F0%9F%93%A9', '📩 📩 📩'), + ('x+%2B+y', 'x + y') +]) +def test_format_mmg_message(message, expected_output): + assert format_mmg_message(message) == expected_output + + +@pytest.mark.parametrize('provider_date, expected_output', [ + ('2017-01-21+11%3A56%3A11', datetime(2017, 1, 21, 11, 56, 11)), + ('2017-05-21+11%3A56%3A11', datetime(2017, 5, 21, 10, 56, 11)) +]) +def test_format_mmg_datetime(provider_date, expected_output): + assert format_mmg_datetime(provider_date) == expected_output + + +def test_create_inbound_mmg_sms_object(sample_service): + sample_service.sms_sender = 'foo' + data = { + 'Message': 'hello+there+%F0%9F%93%A9', + 'Number': 'foo', + 'MSISDN': '07700 900 001', + 'DateRecieved': '2017-01-02+03%3A04%3A05', + 'ID': 'bar', + } + + inbound_sms = create_inbound_mmg_sms_object(sample_service, data) + + assert inbound_sms.service_id == sample_service.id + assert inbound_sms.notify_number == 'foo' + assert inbound_sms.user_number == '447700900001' + assert inbound_sms.provider_date == datetime(2017, 1, 2, 3, 4, 5) + assert inbound_sms.provider_reference == 'bar' + assert inbound_sms._content != 'hello there 📩' + assert inbound_sms.content == 'hello there 📩' + assert inbound_sms.provider == 'mmg' + + +@pytest.mark.parametrize('notify_number', ['foo', 'baz'], ids=['two_matching_services', 'no_matching_services']) +def test_receive_notification_error_if_not_single_matching_service(client, notify_db_session, notify_number): + create_service(service_name='a', sms_sender='foo') + create_service(service_name='b', sms_sender='foo') + + data = { + 'Message': 'hello', + 'Number': notify_number, + 'MSISDN': '7700900001', + 'DateRecieved': '2017-01-02 03:04:05', + 'ID': 'bar', + } + response = client.post(path='/notifications/sms/receive/mmg', + data=json.dumps(data), + headers=[('Content-Type', 'application/json')]) + + # we still return 'RECEIVED' to MMG + assert response.status_code == 200 + assert response.get_data(as_text=True) == 'RECEIVED' + assert InboundSms.query.count() == 0 + + +def test_receive_notification_returns_received_to_firetext(notify_db_session, client, mocker): + mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr') + + create_service(service_name='b', sms_sender='07111111111') + + data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00" + + response = client.post( + path='/notifications/sms/receive/firetext', + data=data, + headers=[('Content-Type', 'application/x-www-form-urlencoded')]) + + assert response.status_code == 200 + result = json.loads(response.get_data(as_text=True)) + + mock.assert_has_calls([call('inbound.firetext.successful')]) + + assert result['status'] == 'ok' + + +def test_receive_notification_from_firetext_persists_message(notify_db_session, client, mocker): + mocker.patch('app.notifications.receive_notifications.statsd_client.incr') + + service = create_service(service_name='b', sms_sender='07111111111') + + data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00" + + response = client.post( + path='/notifications/sms/receive/firetext', + data=data, + headers=[('Content-Type', 'application/x-www-form-urlencoded')]) + + assert response.status_code == 200 + result = json.loads(response.get_data(as_text=True)) + + persisted = InboundSms.query.first() + + assert result['status'] == 'ok' + assert persisted.notify_number == '07111111111' + assert persisted.user_number == '447999999999' + assert persisted.service == service + assert persisted.content == 'this is a message' + assert persisted.provider == 'firetext' + assert persisted.provider_date == datetime(2017, 1, 1, 12, 0, 0, 0) + + +def test_receive_notification_from_firetext_persists_message_with_normalized_phone(notify_db_session, client, mocker): + mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr') + + create_service(service_name='b', sms_sender='07111111111') + + data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00" + + response = client.post( + path='/notifications/sms/receive/firetext', + data=data, + headers=[('Content-Type', 'application/x-www-form-urlencoded')]) + + assert response.status_code == 200 + result = json.loads(response.get_data(as_text=True)) + + persisted = InboundSms.query.first() + + assert result['status'] == 'ok' + assert persisted.user_number == '447999999999' + + +def test_returns_ok_to_firetext_if_mismatched_sms_sender(notify_db_session, client, mocker): + mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr') + + create_service(service_name='b', sms_sender='07111111199') + + data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00" + + response = client.post( + path='/notifications/sms/receive/firetext', + data=data, + headers=[('Content-Type', 'application/x-www-form-urlencoded')]) + + assert response.status_code == 200 + result = json.loads(response.get_data(as_text=True)) + + assert not InboundSms.query.all() + assert result['status'] == 'ok' + mock.assert_has_calls([call('inbound.firetext.failed')]) + + +@pytest.mark.parametrize( + 'number, expected', + [ + ('447123123123', '07123123123'), + ('447123123144', '07123123144'), + ('07123123123', '07123123123'), + ('447444444444', '07444444444') + ] +) +def test_strip_leading_country_code(number, expected): + assert strip_leading_forty_four(number) == expected diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 7b5fda823..2eb7a1f48 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -27,6 +27,7 @@ from app.models import ( ) from tests.app.db import create_user +from tests.conftest import set_config_values def test_get_service_list(client, service_factory): @@ -64,9 +65,9 @@ def test_get_service_list_with_only_active_flag(client, service_factory): def test_get_service_list_with_user_id_and_only_active_flag( - client, - sample_user, - service_factory + client, + sample_user, + service_factory ): other_user = create_user(email='foo@bar.gov.uk') @@ -147,7 +148,32 @@ def test_get_service_by_id(client, sample_service): assert json_resp['data']['sms_sender'] == current_app.config['FROM_NUMBER'] +def test_get_service_by_id_returns_free_sms_limit(client, sample_service): + + auth_header = create_authorization_header() + resp = client.get( + '/service/{}'.format(sample_service.id), + headers=[auth_header] + ) + assert resp.status_code == 200 + json_resp = json.loads(resp.get_data(as_text=True)) + assert json_resp['data']['free_sms_fragment_limit'] == 250000 + + +def test_get_detailed_service_by_id_returns_free_sms_limit(client, sample_service): + + auth_header = create_authorization_header() + resp = client.get( + '/service/{}?detailed=True'.format(sample_service.id), + headers=[auth_header] + ) + assert resp.status_code == 200 + json_resp = json.loads(resp.get_data(as_text=True)) + assert json_resp['data']['free_sms_fragment_limit'] == 250000 + + def test_get_service_list_has_default_permissions(client, service_factory): + service_factory.get('one') service_factory.get('one') service_factory.get('two') service_factory.get('three') @@ -519,7 +545,9 @@ def test_update_service_flags_will_remove_service_permissions(client, notify_db, assert resp.status_code == 200 assert result['data']['can_send_international_sms'] is False - assert set(result['data']['permissions']) == set([SMS_TYPE, EMAIL_TYPE]) + + permissions = ServicePermission.query.filter_by(service_id=service.id).all() + assert set([p.permission for p in permissions]) == set([SMS_TYPE, EMAIL_TYPE]) def test_update_permissions_will_override_permission_flags(client, service_with_no_permissions): @@ -583,14 +611,10 @@ def test_add_service_permission_will_add_permission(client, service_with_no_perm headers=[('Content-Type', 'application/json'), auth_header] ) - resp = client.get( - '/service/{}'.format(service_with_no_permissions.id), - headers=[auth_header] - ) - result = json.loads(resp.get_data(as_text=True)) + permissions = ServicePermission.query.filter_by(service_id=service_with_no_permissions.id).all() assert resp.status_code == 200 - assert result['data']['permissions'] == [permission_to_add] + assert [p.permission for p in permissions] == [permission_to_add] def test_update_permissions_with_an_invalid_permission_will_raise_error(client, sample_service): @@ -852,7 +876,6 @@ def test_add_existing_user_to_another_service_with_all_permissions(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # check which users part of service user_already_in_service = sample_service.users[0] auth_header = create_authorization_header() @@ -927,7 +950,6 @@ def test_add_existing_user_to_another_service_with_send_permissions(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # they must exist in db first user_to_add = User( name='Invited User', @@ -971,7 +993,6 @@ def test_add_existing_user_to_another_service_with_manage_permissions(notify_api sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # they must exist in db first user_to_add = User( name='Invited User', @@ -1015,7 +1036,6 @@ def test_add_existing_user_to_another_service_with_manage_api_keys(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # they must exist in db first user_to_add = User( name='Invited User', @@ -1056,7 +1076,6 @@ def test_add_existing_user_to_non_existing_service_returns404(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - user_to_add = User( name='Invited User', email_address='invited@digital.cabinet-office.gov.uk', @@ -1087,7 +1106,6 @@ def test_add_existing_user_to_non_existing_service_returns404(notify_api, def test_add_existing_user_of_service_to_service_returns400(notify_api, notify_db, notify_db_session, sample_service): with notify_api.test_request_context(): with notify_api.test_client() as client: - existing_user_id = sample_service.users[0].id data = {'permissions': ['send_messages', 'manage_service', 'manage_api_keys']} @@ -1110,7 +1128,6 @@ def test_add_existing_user_of_service_to_service_returns400(notify_api, notify_d def test_add_unknown_user_to_service_returns404(notify_api, notify_db, notify_db_session, sample_service): with notify_api.test_request_context(): with notify_api.test_client() as client: - incorrect_id = 9876 data = {'permissions': ['send_messages', 'manage_service', 'manage_api_keys']} @@ -1131,7 +1148,7 @@ def test_add_unknown_user_to_service_returns404(notify_api, notify_db, notify_db def test_remove_user_from_service( - notify_db, notify_db_session, client, sample_user_service_permission + notify_db, notify_db_session, client, sample_user_service_permission ): second_user = create_user(email="new@digital.cabinet-office.gov.uk") # Simulates successfully adding a user to the service @@ -1151,7 +1168,7 @@ def test_remove_user_from_service( def test_remove_non_existant_user_from_service( - client, sample_user_service_permission + client, sample_user_service_permission ): second_user = create_user(email="new@digital.cabinet-office.gov.uk") endpoint = url_for( @@ -1187,13 +1204,11 @@ def test_cannot_remove_only_user_from_service(notify_api, # This test is just here verify get_service_and_api_key_history that is a temp solution # until proper ui is sorted out on admin app def test_get_service_and_api_key_history(notify_api, notify_db, notify_db_session, sample_service): - from tests.app.conftest import sample_api_key as create_sample_api_key api_key = create_sample_api_key(notify_db, notify_db_session, service=sample_service) with notify_api.test_request_context(): with notify_api.test_client() as client: - auth_header = create_authorization_header() response = client.get( path='/service/{}/history'.format(sample_service.id), @@ -1259,6 +1274,48 @@ def test_get_all_notifications_for_service_in_order(notify_api, notify_db, notif assert response.status_code == 200 +def test_get_notification_for_service_without_uuid(client, notify_db, notify_db_session): + service_1 = create_service(notify_db, notify_db_session, service_name="1", email_from='1') + response = client.get( + path='/service/{}/notifications/{}'.format(service_1.id, 'foo'), + headers=[create_authorization_header()] + ) + assert response.status_code == 404 + + +def test_get_notification_for_service(client, notify_db, notify_db_session): + + service_1 = create_service(notify_db, notify_db_session, service_name="1", email_from='1') + service_2 = create_service(notify_db, notify_db_session, service_name="2", email_from='2') + + service_1_notifications = [ + create_sample_notification(notify_db, notify_db_session, service=service_1), + create_sample_notification(notify_db, notify_db_session, service=service_1), + create_sample_notification(notify_db, notify_db_session, service=service_1), + ] + + service_2_notifications = [ + create_sample_notification(notify_db, notify_db_session, service=service_2) + ] + + for notification in service_1_notifications: + response = client.get( + path='/service/{}/notifications/{}'.format(service_1.id, notification.id), + headers=[create_authorization_header()] + ) + resp = json.loads(response.get_data(as_text=True)) + assert str(resp['id']) == str(notification.id) + assert response.status_code == 200 + + service_2_response = client.get( + path='/service/{}/notifications/{}'.format(service_2.id, notification.id), + headers=[create_authorization_header()] + ) + assert service_2_response.status_code == 404 + service_2_response = json.loads(service_2_response.get_data(as_text=True)) + assert service_2_response == {'message': 'No result found', 'result': 'error'} + + @pytest.mark.parametrize( 'include_from_test_key, expected_count_of_notifications', [ @@ -1267,12 +1324,12 @@ def test_get_all_notifications_for_service_in_order(notify_api, notify_db, notif ] ) def test_get_all_notifications_for_service_including_ones_made_by_jobs( - client, - notify_db, - notify_db_session, - sample_service, - include_from_test_key, - expected_count_of_notifications + client, + notify_db, + notify_db_session, + sample_service, + include_from_test_key, + expected_count_of_notifications ): with_job = sample_notification_with_job(notify_db, notify_db_session, service=sample_service) without_job = create_sample_notification(notify_db, notify_db_session, service=sample_service) @@ -1297,10 +1354,10 @@ def test_get_all_notifications_for_service_including_ones_made_by_jobs( def test_get_only_api_created_notifications_for_service( - client, - notify_db, - notify_db_session, - sample_service + client, + notify_db, + notify_db_session, + sample_service ): with_job = sample_notification_with_job(notify_db, notify_db_session, service=sample_service) without_job = create_sample_notification(notify_db, notify_db_session, service=sample_service) @@ -1317,61 +1374,57 @@ def test_get_only_api_created_notifications_for_service( assert response.status_code == 200 -def test_set_sms_sender_for_service(notify_api, sample_service): - with notify_api.test_request_context(): - with notify_api.test_client() as client: - auth_header = create_authorization_header() - resp = client.get( - '/service/{}'.format(sample_service.id), - headers=[auth_header] - ) - json_resp = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 200 - assert json_resp['data']['name'] == sample_service.name +def test_set_sms_sender_for_service(client, sample_service): + data = { + 'sms_sender': 'elevenchars', + } - data = { - 'sms_sender': 'elevenchars', - } + auth_header = create_authorization_header() - auth_header = create_authorization_header() - - resp = client.post( - '/service/{}'.format(sample_service.id), - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), auth_header] - ) - result = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 200 - assert result['data']['sms_sender'] == 'elevenchars' + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 200 + assert result['data']['sms_sender'] == 'elevenchars' -def test_set_sms_sender_for_service_rejects_invalid_characters(notify_api, sample_service): - with notify_api.test_request_context(): - with notify_api.test_client() as client: - auth_header = create_authorization_header() - resp = client.get( - '/service/{}'.format(sample_service.id), - headers=[auth_header] - ) - json_resp = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 200 - assert json_resp['data']['name'] == sample_service.name +def test_set_sms_sender_for_service_rejects_invalid_characters(client, sample_service): + data = { + 'sms_sender': 'invalid####', + } - data = { - 'sms_sender': 'invalid####', - } + auth_header = create_authorization_header() - auth_header = create_authorization_header() + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 400 + assert result['result'] == 'error' + assert result['message'] == {'sms_sender': ['Only alphanumeric characters allowed']} - resp = client.post( - '/service/{}'.format(sample_service.id), - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), auth_header] - ) - result = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 400 - assert result['result'] == 'error' - assert result['message'] == {'sms_sender': ['Only alphanumeric characters allowed']} + +def test_set_sms_sender_for_service_rejects_null(client, sample_service): + data = { + 'sms_sender': None, + } + + auth_header = create_authorization_header() + + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 400 + assert result['result'] == 'error' + assert result['message'] == {'sms_sender': ['Field may not be null.']} @pytest.mark.parametrize('today_only,stats', [ @@ -1641,11 +1694,11 @@ def test_get_notification_billable_unit_count_missing_year(client, sample_servic ('?year=abcd', 400, {'message': 'Year must be a number', 'result': 'error'}), ]) def test_get_service_provider_aggregate_statistics( - client, - sample_service, - query_string, - expected_status, - expected_json, + client, + sample_service, + query_string, + expected_status, + expected_json, ): response = client.get( '/service/{}/fragment/aggregate_statistics{}'.format(sample_service.id, query_string), @@ -1686,11 +1739,11 @@ def test_get_template_stats_by_month_returns_correct_data(notify_db, notify_db_s ('?year=abcd', 400, {'message': 'Year must be a number', 'result': 'error'}), ]) def test_get_template_stats_by_month_returns_error_for_incorrect_year( - client, - sample_service, - query_string, - expected_status, - expected_json + client, + sample_service, + query_string, + expected_status, + expected_json ): response = client.get( '/service/{}/notifications/templates/monthly{}'.format(sample_service.id, query_string), @@ -1931,20 +1984,90 @@ def test_update_service_does_not_call_send_notification_when_restricted_not_chan assert not send_notification_mock.called -def test_update_service_works_when_sms_sender_is_null(sample_service, client, mocker): - sample_service.sms_sender = None - data = {'name': 'new name'} +def test_get_yearly_billing_usage_count_returns_400_if_missing_year(client, sample_service): + response = client.get( + '/service/{}/yearly-sms-billable-units'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 400 + assert json.loads(response.get_data(as_text=True)) == { + 'message': 'No valid year provided', 'result': 'error' + } - resp = client.post( - 'service/{}'.format(sample_service.id), - data=json.dumps(data), - headers=[create_authorization_header()], - content_type='application/json' + +def test_get_yearly_billing_usage_count_returns_400_if_invalid_year(client, sample_service, mocker): + redis_get_mock = mocker.patch('app.service.rest.redis_store.get_all_from_hash', return_value=None) + redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire') + + response = client.get( + '/service/{}/yearly-sms-billable-units?year=HAHAHAHAH'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 400 + assert json.loads(response.get_data(as_text=True)) == { + 'message': 'No valid year provided', 'result': 'error' + } + redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id))) + redis_set_mock.assert_not_called() + + +def test_get_yearly_billing_usage_count_returns_200_if_year_provided(client, sample_service, mocker): + redis_get_mock = mocker.patch('app.service.rest.redis_store.get_all_from_hash', return_value=None) + redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire') + + start = datetime.utcnow() + end = datetime.utcnow() + timedelta(minutes=10) + mock_query = mocker.patch( + 'app.service.rest.get_total_billable_units_for_sent_sms_notifications_in_date_range', return_value=(100, 200.0) + ) + mock_year = mocker.patch('app.service.rest.get_financial_year', return_value=(start, end)) + response = client.get( + '/service/{}/yearly-sms-billable-units?year=2016'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 200 + assert json.loads(response.get_data(as_text=True)) == { + 'billable_sms_units': 100, + 'total_cost': 200.0 + } + mock_query.assert_called_once_with(start, end, sample_service.id) + mock_year.assert_called_once_with(2016) + redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id))) + redis_set_mock.assert_called_once_with( + "{}-sms_billable_units".format(str(sample_service.id)), + {'billable_units': 100, 'total_cost': 200.0}, + expire_in_seconds=60 ) - assert resp.status_code == 200 - # make sure it wasn't changed to not-null under the hood - assert sample_service.sms_sender is None + +def test_get_yearly_billing_usage_count_returns_from_cache_if_present(client, sample_service, mocker): + redis_get_mock = mocker.patch( + 'app.service.rest.redis_store.get_all_from_hash', + return_value={b'total_cost': 100.0, b'billable_units': 50} + ) + redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire') + mock_query = mocker.patch( + 'app.service.rest.get_total_billable_units_for_sent_sms_notifications_in_date_range', return_value=(50, 100.0) + ) + + start = datetime.utcnow() + end = datetime.utcnow() + timedelta(minutes=10) + mock_year = mocker.patch('app.service.rest.get_financial_year', return_value=(start, end)) + + response = client.get( + '/service/{}/yearly-sms-billable-units?year=2016'.format(sample_service.id), + headers=[create_authorization_header()] + ) + response.get_data(as_text=True) + assert response.status_code == 200 + assert json.loads(response.get_data(as_text=True)) == { + 'billable_sms_units': 50, + 'total_cost': 100.0 + } + redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id))) + mock_year.assert_not_called() + mock_query.assert_not_called() + redis_set_mock.assert_not_called() def test_search_for_notification_by_to_field_filters_by_status(client, notify_db, notify_db_session): diff --git a/tests/app/user/test_rest.py b/tests/app/user/test_rest.py index ddbb3eaae..73f60f05b 100644 --- a/tests/app/user/test_rest.py +++ b/tests/app/user/test_rest.py @@ -374,7 +374,7 @@ def test_send_user_reset_password_should_send_reset_password_link(client, assert resp.status_code == 204 notification = Notification.query.first() - mocked.assert_called_once_with([str(notification.id)], queue="notify") + mocked.assert_called_once_with([str(notification.id)], queue="notify-internal-tasks") def test_send_user_reset_password_should_return_400_when_email_is_missing(client, mocker): @@ -436,7 +436,7 @@ def test_send_already_registered_email(client, sample_user, already_registered_t assert resp.status_code == 204 notification = Notification.query.first() - mocked.assert_called_once_with(([str(notification.id)]), queue="notify") + mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks") def test_send_already_registered_email_returns_400_when_data_is_missing(client, sample_user): @@ -464,7 +464,7 @@ def test_send_user_confirm_new_email_returns_204(client, sample_user, change_ema notification = Notification.query.first() mocked.assert_called_once_with( ([str(notification.id)]), - queue="notify") + queue="notify-internal-tasks") def test_send_user_confirm_new_email_returns_400_when_email_missing(client, sample_user, mocker): diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py index 08f510bbd..84a88f6a0 100644 --- a/tests/app/user/test_rest_verify.py +++ b/tests/app/user/test_rest_verify.py @@ -218,7 +218,7 @@ def test_send_user_sms_code(client, app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( ([str(notification.id)]), - queue="notify" + queue="notify-internal-tasks" ) @@ -246,7 +246,7 @@ def test_send_user_code_for_sms_with_optional_to_field(client, assert notification.to == to_number app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( ([str(notification.id)]), - queue="notify" + queue="notify-internal-tasks" ) @@ -294,7 +294,7 @@ def test_send_user_email_verification(client, headers=[('Content-Type', 'application/json'), auth_header]) assert resp.status_code == 204 notification = Notification.query.first() - mocked.assert_called_once_with(([str(notification.id)]), queue="notify") + mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks") def test_send_email_verification_returns_404_for_bad_input_data(client, notify_db_session, mocker): diff --git a/tests/app/v2/notifications/test_post_notifications.py b/tests/app/v2/notifications/test_post_notifications.py index afce12947..61f2c4517 100644 --- a/tests/app/v2/notifications/test_post_notifications.py +++ b/tests/app/v2/notifications/test_post_notifications.py @@ -3,13 +3,14 @@ import uuid import pytest from freezegun import freeze_time -from app.models import Notification, ScheduledNotification +from app.models import Notification, ScheduledNotification, SCHEDULE_NOTIFICATIONS, EMAIL_TYPE, SMS_TYPE from flask import json, current_app from app.models import Notification from app.v2.errors import RateLimitError from tests import create_authorization_header from tests.app.conftest import sample_template as create_sample_template, sample_service +from tests.app.db import create_service, create_template @pytest.mark.parametrize("reference", [None, "reference_from_client"]) @@ -231,7 +232,7 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori notification_id = json.loads(response.data)['id'] assert response.status_code == 201 - mocked.assert_called_once_with([notification_id], queue='priority') + mocked.assert_called_once_with([notification_id], queue='priority-tasks') @pytest.mark.parametrize( @@ -350,26 +351,28 @@ def test_post_sms_should_persist_supplied_sms_number(client, sample_template_wit assert mocked.called -@pytest.mark.skip("Once the service can be invited to schedule notifications we can add this test.") @pytest.mark.parametrize("notification_type, key_send_to, send_to", [("sms", "phone_number", "07700 900 855"), ("email", "email_address", "sample@email.com")]) @freeze_time("2017-05-14 14:00:00") -def test_post_notification_with_scheduled_for(client, sample_template, sample_email_template, +def test_post_notification_with_scheduled_for(client, notify_db, notify_db_session, notification_type, key_send_to, send_to): + service = create_service(service_name=str(uuid.uuid4()), + service_permissions=[EMAIL_TYPE, SMS_TYPE, SCHEDULE_NOTIFICATIONS]) + template = create_template(service=service, template_type=notification_type) data = { key_send_to: send_to, - 'template_id': str(sample_email_template.id) if notification_type == 'email' else str(sample_template.id), + 'template_id': str(template.id) if notification_type == 'email' else str(template.id), 'scheduled_for': '2017-05-14 14:15' } - auth_header = create_authorization_header(service_id=sample_template.service_id) + auth_header = create_authorization_header(service_id=service.id) response = client.post('/v2/notifications/{}'.format(notification_type), data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) assert response.status_code == 201 resp_json = json.loads(response.get_data(as_text=True)) - scheduled_notification = ScheduledNotification.query.all() + scheduled_notification = ScheduledNotification.query.filter_by(notification_id=resp_json["id"]).all() assert len(scheduled_notification) == 1 assert resp_json["id"] == str(scheduled_notification[0].notification_id) assert resp_json["scheduled_for"] == '2017-05-14 14:15' @@ -379,8 +382,8 @@ def test_post_notification_with_scheduled_for(client, sample_template, sample_em [("sms", "phone_number", "07700 900 855"), ("email", "email_address", "sample@email.com")]) @freeze_time("2017-05-14 14:00:00") -def test_post_notification_with_scheduled_for_raises_bad_request(client, sample_template, sample_email_template, - notification_type, key_send_to, send_to): +def test_post_notification_raises_bad_request_if_service_not_invited_to_schedule( + client, sample_template, sample_email_template, notification_type, key_send_to, send_to): data = { key_send_to: send_to, 'template_id': str(sample_email_template.id) if notification_type == 'email' else str(sample_template.id), @@ -394,4 +397,4 @@ def test_post_notification_with_scheduled_for_raises_bad_request(client, sample_ assert response.status_code == 400 error_json = json.loads(response.get_data(as_text=True)) assert error_json['errors'] == [ - {"error": "BadRequestError", "message": 'Your service must be invited to schedule notifications via the API.'}] + {"error": "BadRequestError", "message": 'Cannot schedule notifications (this feature is invite-only)'}]