diff --git a/.gitignore b/.gitignore index 12f77b15c..ffd54699f 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,5 @@ environment.sh celerybeat-schedule -wheelhouse/ - # CloudFoundry .cf diff --git a/Makefile b/Makefile index 4142722ff..d8c650d5f 100644 --- a/Makefile +++ b/Makefile @@ -80,8 +80,7 @@ generate-version-file: ## Generates the app version file .PHONY: build build: dependencies generate-version-file ## Build project - rm -rf wheelhouse - . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel wheel --wheel-dir=wheelhouse -r requirements.txt + . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements.txt .PHONY: cf-build cf-build: dependencies generate-version-file ## Build project for PAAS @@ -260,7 +259,7 @@ clean-docker-containers: ## Clean up any remaining docker containers .PHONY: clean clean: - rm -rf node_modules cache target venv .coverage build tests/.cache wheelhouse + rm -rf node_modules cache target venv .coverage build tests/.cache .PHONY: cf-login cf-login: ## Log in to Cloud Foundry diff --git a/app/authentication/auth.py b/app/authentication/auth.py index 8242795b2..d10a98bb0 100644 --- a/app/authentication/auth.py +++ b/app/authentication/auth.py @@ -52,13 +52,18 @@ def restrict_ip_sms(): ip_list = ip_route.split(',') if len(ip_list) >= 3: ip = ip_list[len(ip_list) - 3] - current_app.logger.info("Inbound sms ip route list {}".format(ip_route)) + current_app.logger.info("Inbound sms ip route list {}" + .format(ip_route)) + + # Temporary custom header for route security - to experiment if the header passes through + if request.headers.get("X-Custom-forwarder"): + current_app.logger.info("X-Custom-forwarder {}".format(request.headers.get("X-Custom-forwarder"))) if ip in current_app.config.get('SMS_INBOUND_WHITELIST'): current_app.logger.info("Inbound sms ip addresses {} passed ".format(ip)) return else: - current_app.logger.info("Inbound sms ip addresses {} blocked ".format(ip)) + current_app.logger.info("Inbound sms ip addresses blocked {}".format(ip)) return # raise AuthError('Unknown source IP address from the SMS provider', 403) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e86a7c113..17533ac08 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -9,9 +9,17 @@ from sqlalchemy.exc import SQLAlchemyError from app.aws import s3 from app import notify_celery from app import performance_platform_client +from app.dao.date_util import get_month_start_end_date from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago -from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by +from app.dao.jobs_dao import ( + dao_set_scheduled_jobs_to_pending, + dao_get_jobs_older_than_limited_by +) +from app.dao.monthly_billing_dao import ( + get_service_ids_that_need_sms_billing_populated, + create_or_update_monthly_billing_sms +) from app.dao.notifications_dao import ( dao_timeout_notifications, is_delivery_slow_for_provider, @@ -281,3 +289,14 @@ def delete_dvla_response_files_older_than_seven_days(): except SQLAlchemyError as e: current_app.logger.exception("Failed to delete dvla response files") raise + + +@notify_celery.task(name="populate_monthly_billing") +@statsd(namespace="tasks") +def populate_monthly_billing(): + # for every service with billable units this month update billing totals for yesterday + # this will overwrite the existing amount. + yesterday = datetime.utcnow() - timedelta(days=1) + start_date, end_date = get_month_start_end_date(yesterday) + services = get_service_ids_that_need_sms_billing_populated(start_date=start_date, end_date=end_date) + [create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=yesterday) for s in services] diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 293f0cd2c..c890a8333 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -18,7 +18,7 @@ def set_config_env_vars(vcap_services): vcap_application = json.loads(os.environ['VCAP_APPLICATION']) os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name'] - os.environ['LOGGING_STDOUT_JSON'] = '1' + os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log' # Notify common config for s in vcap_services['user-provided']: diff --git a/app/commands.py b/app/commands.py index 6e4bc53d1..a2e692ebd 100644 --- a/app/commands.py +++ b/app/commands.py @@ -5,7 +5,8 @@ from flask.ext.script import Command, Manager, Option from app import db -from app.models import (PROVIDERS, Service, User, NotificationHistory) +from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms +from app.models import (PROVIDERS, User) from app.dao.services_dao import ( delete_service_and_all_associated_db_objects, dao_fetch_all_services_by_user @@ -146,3 +147,26 @@ class CustomDbScript(Command): print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) db.session.commit() result = db.session.execute(subq_hist).fetchall() + + +class PopulateMonthlyBilling(Command): + option_list = ( + Option('-s', '-service-id', dest='service_id', + help="Service id to populate monthly billing for"), + Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017") + ) + + def run(self, service_id, year): + start, end = 1, 13 + if year == '2016': + start = 4 + + print('Starting populating monthly billing for {}'.format(year)) + for i in range(start, end): + self.populate(service_id, year, i) + + def populate(self, service_id, year, month): + create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) + results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) + print("Finished populating data for {} for service id {}".format(month, service_id)) + print(results.monthly_totals) diff --git a/app/config.py b/app/config.py index 99b77cc8d..940c08622 100644 --- a/app/config.py +++ b/app/config.py @@ -22,7 +22,6 @@ class QueueNames(object): PERIODIC = 'periodic-tasks' PRIORITY = 'priority-tasks' DATABASE = 'database-tasks' - SEND_COMBINED = 'send-tasks' SEND_SMS = 'send-sms-tasks' SEND_EMAIL = 'send-email-tasks' RESEARCH_MODE = 'research-mode-tasks' @@ -38,7 +37,6 @@ class QueueNames(object): QueueNames.PRIORITY, QueueNames.PERIODIC, QueueNames.DATABASE, - QueueNames.SEND_COMBINED, QueueNames.SEND_SMS, QueueNames.SEND_EMAIL, QueueNames.RESEARCH_MODE, @@ -97,7 +95,7 @@ class Config(object): # Logging DEBUG = False - LOGGING_STDOUT_JSON = os.getenv('LOGGING_STDOUT_JSON') == '1' + NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH') ########################### # Default config values ### @@ -108,14 +106,12 @@ class Config(object): AWS_REGION = 'eu-west-1' INVITATION_EXPIRATION_DAYS = 2 NOTIFY_APP_NAME = 'api' - NOTIFY_LOG_PATH = '/var/log/notify/application.log' SQLALCHEMY_COMMIT_ON_TEARDOWN = False SQLALCHEMY_RECORD_QUERIES = True SQLALCHEMY_TRACK_MODIFICATIONS = True PAGE_SIZE = 50 API_PAGE_SIZE = 250 SMS_CHAR_COUNT_LIMIT = 495 - BRANDING_PATH = '/images/email-template/crests/' TEST_MESSAGE_FILENAME = 'Test message' ONE_OFF_MESSAGE_FILENAME = 'Report' MAX_VERIFY_CODE_COUNT = 10 @@ -224,6 +220,11 @@ class Config(object): 'task': 'timeout-job-statistics', 'schedule': crontab(minute=0, hour=5), 'options': {'queue': QueueNames.PERIODIC} + }, + 'populate_monthly_billing': { + 'task': 'populate_monthly_billing', + 'schedule': crontab(minute=10, hour=5), + 'options': {'queue': QueueNames.PERIODIC} } } CELERY_QUEUES = [] @@ -275,6 +276,7 @@ class Config(object): ###################### class Development(Config): + NOTIFY_LOG_PATH = 'application.log' SQLALCHEMY_ECHO = False NOTIFY_EMAIL_DOMAIN = 'notify.tools' CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 2471865bd..8019fdecc 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -2,6 +2,8 @@ from datetime import datetime, timedelta import pytz +from app.utils import convert_bst_to_utc + def get_financial_year(year): return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1) @@ -16,3 +18,16 @@ def get_april_fools(year): """ return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( tzinfo=None) + + +def get_month_start_end_date(month_year): + """ + This function return the start and date of the month_year as UTC, + :param month_year: the datetime to calculate the start and end date for that month + :return: start_date, end_date, month + """ + import calendar + _, num_days = calendar.monthrange(month_year.year, month_year.month) + first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0) + last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999) + return convert_bst_to_utc(first_day), convert_bst_to_utc(last_day) diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index ec7bbe2a6..e6a80832e 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -1,3 +1,4 @@ +import uuid from datetime import datetime, timedelta from flask import current_app @@ -10,6 +11,7 @@ from app.models import ( JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, LETTER_TYPE ) +from app.variables import LETTER_TEST_API_FILENAME from app.statsd_decorators import statsd @@ -108,6 +110,8 @@ def dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id): def dao_create_job(job): + if not job.id: + job.id = uuid.uuid4() job_stats = JobStatistics( job_id=job.id, updated_at=datetime.utcnow() @@ -139,9 +143,18 @@ def dao_get_jobs_older_than_limited_by(job_types, older_than=7, limit_days=2): def dao_get_all_letter_jobs(): - return db.session.query(Job).join(Job.template).filter( - Template.template_type == LETTER_TYPE - ).order_by(desc(Job.created_at)).all() + return db.session.query( + Job + ).join( + Job.template + ).filter( + Template.template_type == LETTER_TYPE, + # test letter jobs (or from research mode services) are created with a different filename, + # exclude them so we don't see them on the send to CSV + Job.original_file_name != LETTER_TEST_API_FILENAME + ).order_by( + desc(Job.created_at) + ).all() @statsd(namespace="dao") diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py new file mode 100644 index 000000000..4ddacddfa --- /dev/null +++ b/app/dao/monthly_billing_dao.py @@ -0,0 +1,71 @@ +from datetime import datetime + +from app import db +from app.dao.dao_utils import transactional +from app.dao.date_util import get_month_start_end_date +from app.dao.notification_usage_dao import get_billing_data_for_month +from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory +from app.statsd_decorators import statsd + + +def get_service_ids_that_need_sms_billing_populated(start_date, end_date): + return db.session.query( + NotificationHistory.service_id + ).filter( + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + NotificationHistory.notification_type == SMS_TYPE, + NotificationHistory.billable_units != 0 + ).distinct().all() + + +@transactional +def create_or_update_monthly_billing_sms(service_id, billing_month): + start_date, end_date = get_month_start_end_date(billing_month) + monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date) + # update monthly + monthly_totals = _monthly_billing_data_to_json(monthly) + row = get_monthly_billing_entry(service_id, start_date, SMS_TYPE) + + if row: + row.monthly_totals = monthly_totals + row.updated_at = datetime.utcnow() + else: + row = MonthlyBilling( + service_id=service_id, + notification_type=SMS_TYPE, + monthly_totals=monthly_totals, + start_date=start_date, + end_date=end_date + ) + + db.session.add(row) + + +def get_monthly_billing_entry(service_id, start_date, notification_type): + entry = MonthlyBilling.query.filter_by( + service_id=service_id, + start_date=start_date, + notification_type=notification_type + ).first() + + return entry + + +@statsd(namespace="dao") +def get_monthly_billing_sms(service_id, billing_month): + start_date, end_date = get_month_start_end_date(billing_month) + monthly = MonthlyBilling.query.filter_by(service_id=service_id, + start_date=start_date, + notification_type=SMS_TYPE).first() + return monthly + + +def _monthly_billing_data_to_json(monthly): + # total cost must take into account the free allowance. + # might be a good idea to capture free allowance in this table + return [{"billing_units": x.billing_units, + "rate_multiplier": x.rate_multiplier, + "international": x.international, + "rate": x.rate, + "total_cost": (x.billing_units * x.rate_multiplier) * x.rate} for x in monthly] diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 995462509..cf90fe499 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -6,7 +6,7 @@ from sqlalchemy import func, case, cast from sqlalchemy import literal_column from app import db -from app.dao.date_util import get_financial_year +from app.dao.date_util import get_financial_year, get_month_start_end_date from app.models import (NotificationHistory, Rate, NOTIFICATION_STATUS_TYPES_BILLABLE, @@ -20,25 +20,63 @@ from app.utils import get_london_month_from_utc_column @statsd(namespace="dao") def get_yearly_billing_data(service_id, year): start_date, end_date = get_financial_year(year) - rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) + + if not rates: + return [] def get_valid_from(valid_from): return start_date if valid_from < start_date else valid_from result = [] for r, n in zip(rates, rates[1:]): - result.append(sms_yearly_billing_data_query(r.rate, service_id, get_valid_from(r.valid_from), n.valid_from)) + result.append( + sms_yearly_billing_data_query( + r.rate, + service_id, + get_valid_from(r.valid_from), + n.valid_from + ) + ) result.append( - sms_yearly_billing_data_query(rates[-1].rate, service_id, get_valid_from(rates[-1].valid_from), end_date)) + sms_yearly_billing_data_query( + rates[-1].rate, + service_id, + get_valid_from(rates[-1].valid_from), + end_date + ) + ) + result.append(email_yearly_billing_data_query(service_id, start_date, end_date)) return sum(result, []) +@statsd(namespace="dao") +def get_billing_data_for_month(service_id, start_date, end_date): + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) + + if not rates: + return [] + + result = [] + # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought + for r, n in zip(rates, rates[1:]): + result.extend(sms_billing_data_per_month_query(r.rate, service_id, max(r.valid_from, start_date), + min(n.valid_from, end_date))) + result.extend( + sms_billing_data_per_month_query(rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), end_date)) + + return result + + @statsd(namespace="dao") def get_monthly_billing_data(service_id, year): start_date, end_date = get_financial_year(year) - rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) + + if not rates: + return [] result = [] for r, n in zip(rates, rates[1:]): @@ -103,7 +141,7 @@ def sms_yearly_billing_data_query(rate, service_id, start_date, end_date): return result -def get_rates_for_year(start_date, end_date, notification_type): +def get_rates_for_daterange(start_date, end_date, notification_type): rates = Rate.query.filter(Rate.notification_type == notification_type).order_by(Rate.valid_from).all() results = [] for current_rate, current_rate_expiry_date in zip(rates, rates[1:]): @@ -115,8 +153,10 @@ def get_rates_for_year(start_date, end_date, notification_type): results.append(rates[-1]) if not results: - if start_date >= rates[-1].valid_from: - results.append(rates[-1]) + for x in reversed(rates): + if start_date >= x.valid_from: + results.append(x) + break return results @@ -128,12 +168,12 @@ def is_between(date, start_date, end_date): def sms_billing_data_per_month_query(rate, service_id, start_date, end_date): month = get_london_month_from_utc_column(NotificationHistory.created_at) result = db.session.query( - month, - func.sum(NotificationHistory.billable_units), - rate_multiplier(), + month.label('month'), + func.sum(NotificationHistory.billable_units).label('billing_units'), + rate_multiplier().label('rate_multiplier'), NotificationHistory.international, NotificationHistory.notification_type, - cast(rate, Float()) + cast(rate, Float()).label('rate') ).filter( *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) ).group_by( @@ -193,7 +233,7 @@ def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date def discover_rate_bounds_for_billing_query(start_date, end_date): bounds = [] - rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) def current_valid_from(index): return rates[index].valid_from diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 432224653..c3333fc5a 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -149,11 +149,11 @@ def dao_get_template_usage(service_id, limit_days=None): @statsd(namespace="dao") def dao_get_last_template_usage(template_id): - return NotificationHistory.query.filter( - NotificationHistory.template_id == template_id, - NotificationHistory.key_type != KEY_TYPE_TEST + return Notification.query.filter( + Notification.template_id == template_id, + Notification.key_type != KEY_TYPE_TEST ).order_by( - desc(NotificationHistory.created_at) + desc(Notification.created_at) ).first() @@ -338,7 +338,8 @@ def get_notifications_for_service( filters.append(Notification.created_at < older_than_created_at) if not include_jobs or (key_type and key_type != KEY_TYPE_NORMAL): - filters.append(Notification.job_id.is_(None)) + # we can't say "job_id == None" here, because letters sent via the API still have a job_id :( + filters.append(Notification.api_key_id != None) # noqa if key_type is not None: filters.append(Notification.key_type == key_type) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 310bd8f32..20099c009 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -207,14 +207,14 @@ def delete_service_and_all_associated_db_objects(service): _delete_commit(ProviderStatistics.query.filter_by(service=service)) _delete_commit(InvitedUser.query.filter_by(service=service)) _delete_commit(Permission.query.filter_by(service=service)) - _delete_commit(ApiKey.query.filter_by(service=service)) - _delete_commit(ApiKey.get_history_model().query.filter_by(service_id=service.id)) _delete_commit(NotificationHistory.query.filter_by(service=service)) _delete_commit(Notification.query.filter_by(service=service)) _delete_commit(Job.query.filter_by(service=service)) _delete_commit(Template.query.filter_by(service=service)) _delete_commit(TemplateHistory.query.filter_by(service_id=service.id)) _delete_commit(ServicePermission.query.filter_by(service_id=service.id)) + _delete_commit(ApiKey.query.filter_by(service=service)) + _delete_commit(ApiKey.get_history_model().query.filter_by(service_id=service.id)) verify_codes = VerifyCode.query.join(User).filter(User.id.in_([x.id for x in service.users])) list(map(db.session.delete, verify_codes)) diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index df4c664f9..fcf4348d8 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -18,7 +18,7 @@ from app.dao.templates_dao import dao_get_template_by_id from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE, \ NOTIFICATION_SENT, NOTIFICATION_SENDING -from app.celery.statistics_tasks import record_initial_job_statistics, create_initial_notification_statistic_tasks +from app.celery.statistics_tasks import create_initial_notification_statistic_tasks def send_sms_to_provider(notification): @@ -145,34 +145,20 @@ def provider_to_use(notification_type, notification_id, international=False): return clients.get_client_by_name_and_type(active_providers_in_order[0].identifier, notification_type) -def get_logo_url(base_url, branding_path, logo_file): - """ - Get the complete URL for a given logo. - - We have to convert the base_url into a static url. Our hosted environments all have their own cloudfront instances, - found at the static subdomain (eg https://static.notifications.service.gov.uk). - - If running locally (dev environment), don't try and use cloudfront - just stick to the actual underlying source - ({URL}/static/{PATH}) - """ +def get_logo_url(base_url, logo_file): base_url = parse.urlparse(base_url) netloc = base_url.netloc - # covers both preview and staging - if base_url.netloc.startswith('localhost') or 'notify.works' in base_url.netloc: - path = '/static' + branding_path + logo_file - else: - if base_url.netloc.startswith('www'): - # strip "www." - netloc = base_url.netloc[4:] - - netloc = 'static.' + netloc - path = branding_path + logo_file + if base_url.netloc.startswith('localhost'): + netloc = 'notify.tools' + elif base_url.netloc.startswith('www'): + # strip "www." + netloc = base_url.netloc[4:] logo_url = parse.ParseResult( scheme=base_url.scheme, - netloc=netloc, - path=path, + netloc='static-logos.' + netloc, + path=logo_file, params=base_url.params, query=base_url.query, fragment=base_url.fragment @@ -185,7 +171,6 @@ def get_html_email_options(service): if service.organisation: logo_url = get_logo_url( current_app.config['ADMIN_BASE_URL'], - current_app.config['BRANDING_PATH'], service.organisation.logo ) diff --git a/app/models.py b/app/models.py index 0e3079c80..4ab486276 100644 --- a/app/models.py +++ b/app/models.py @@ -4,7 +4,6 @@ import datetime from flask import url_for, current_app from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.dialects.postgresql import ( UUID, JSON @@ -465,7 +464,7 @@ class Template(db.Model): "created_by": self.created_by.email_address, "version": self.version, "body": self.content, - "subject": self.subject if self.template_type == EMAIL_TYPE else None + "subject": self.subject if self.template_type != SMS_TYPE else None } return serialized @@ -507,18 +506,7 @@ class TemplateHistory(db.Model): default=NORMAL) def serialize(self): - serialized = { - "id": self.id, - "type": self.template_type, - "created_at": self.created_at.strftime(DATETIME_FORMAT), - "updated_at": self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None, - "created_by": self.created_by.email_address, - "version": self.version, - "body": self.content, - "subject": self.subject if self.template_type == EMAIL_TYPE else None - } - - return serialized + return Template.serialize(self) MMG_PROVIDER = "mmg" @@ -620,7 +608,7 @@ class JobStatus(db.Model): class Job(db.Model): __tablename__ = 'jobs' - id = db.Column(UUID(as_uuid=True), primary_key=True) + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) original_file_name = db.Column(db.String, nullable=False) service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False) service = db.relationship('Service', backref=db.backref('jobs', lazy='dynamic')) @@ -655,7 +643,7 @@ class Job(db.Model): unique=False, nullable=True) created_by = db.relationship('User') - created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False) + created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=True) scheduled_for = db.Column( db.DateTime, index=True, @@ -892,7 +880,7 @@ class Notification(db.Model): @property def subject(self): from app.utils import get_template_instance - if self.notification_type == EMAIL_TYPE: + if self.notification_type != SMS_TYPE: template_object = get_template_instance(self.template.__dict__, self.personalisation) return template_object.subject @@ -972,10 +960,24 @@ class Notification(db.Model): "created_at": self.created_at.strftime(DATETIME_FORMAT), "sent_at": self.sent_at.strftime(DATETIME_FORMAT) if self.sent_at else None, "completed_at": self.completed_at(), - "scheduled_for": convert_bst_to_utc(self.scheduled_notification.scheduled_for - ).strftime(DATETIME_FORMAT) if self.scheduled_notification else None + "scheduled_for": ( + convert_bst_to_utc( + self.scheduled_notification.scheduled_for + ).strftime(DATETIME_FORMAT) + if self.scheduled_notification + else None + ) } + if self.notification_type == LETTER_TYPE: + serialized['line_1'] = self.personalisation['address_line_1'] + serialized['line_2'] = self.personalisation.get('address_line_2') + serialized['line_3'] = self.personalisation.get('address_line_3') + serialized['line_4'] = self.personalisation.get('address_line_4') + serialized['line_5'] = self.personalisation.get('address_line_5') + serialized['line_6'] = self.personalisation.get('address_line_6') + serialized['postcode'] = self.personalisation['postcode'] + return serialized @@ -1246,3 +1248,29 @@ class LetterRateDetail(db.Model): letter_rate = db.relationship('LetterRate', backref='letter_rates') page_total = db.Column(db.Integer, nullable=False) rate = db.Column(db.Numeric(), nullable=False) + + +class MonthlyBilling(db.Model): + __tablename__ = 'monthly_billing' + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) + service = db.relationship('Service', backref='monthly_billing') + start_date = db.Column(db.DateTime, nullable=False) + end_date = db.Column(db.DateTime, nullable=False) + notification_type = db.Column(notification_types, nullable=False) + monthly_totals = db.Column(JSON, nullable=False) + updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + + __table_args__ = ( + UniqueConstraint('service_id', 'start_date', 'notification_type', name='uix_monthly_billing'), + ) + + def serialized(self): + return { + "start_date": self.start_date, + "end_date": self.end_date, + "service_id": str(self.service_id), + "notification_type": self.notification_type, + "monthly_totals": self.monthly_totals + } diff --git a/app/notifications/process_letter_notifications.py b/app/notifications/process_letter_notifications.py new file mode 100644 index 000000000..332f6ed32 --- /dev/null +++ b/app/notifications/process_letter_notifications.py @@ -0,0 +1,45 @@ +from app import create_random_identifier +from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND, Job +from app.dao.jobs_dao import dao_create_job +from app.notifications.process_notifications import persist_notification +from app.v2.errors import InvalidRequest +from app.variables import LETTER_API_FILENAME + + +def create_letter_api_job(template): + service = template.service + if not service.active: + raise InvalidRequest('Service {} is inactive'.format(service.id), 403) + if template.archived: + raise InvalidRequest('Template {} is deleted'.format(template.id), 400) + + job = Job( + original_file_name=LETTER_API_FILENAME, + service=service, + template=template, + template_version=template.version, + notification_count=1, + job_status=JOB_STATUS_READY_TO_SEND, + created_by=None + ) + dao_create_job(job) + return job + + +def create_letter_notification(letter_data, job, api_key): + notification = persist_notification( + template_id=job.template.id, + template_version=job.template.version, + # we only accept addresses_with_underscores from the API (from CSV we also accept dashes, spaces etc) + recipient=letter_data['personalisation']['address_line_1'], + service=job.service, + personalisation=letter_data['personalisation'], + notification_type=LETTER_TYPE, + api_key_id=api_key.id, + key_type=api_key.key_type, + job_id=job.id, + job_row_number=0, + reference=create_random_identifier(), + client_reference=letter_data.get('reference') + ) + return notification diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index f2a424881..e7e8779f2 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -36,6 +36,7 @@ def check_placeholders(template_object): def persist_notification( + *, template_id, template_version, recipient, @@ -54,7 +55,7 @@ def persist_notification( created_by_id=None ): notification_created_at = created_at or datetime.utcnow() - if not notification_id and simulated: + if not notification_id: notification_id = uuid.uuid4() notification = Notification( id=notification_id, diff --git a/app/notifications/validators.py b/app/notifications/validators.py index 3aa86c963..83b620506 100644 --- a/app/notifications/validators.py +++ b/app/notifications/validators.py @@ -9,7 +9,7 @@ from notifications_utils.clients.redis import rate_limit_cache_key, daily_limit_ from app.dao import services_dao, templates_dao from app.models import ( - INTERNATIONAL_SMS_TYPE, SMS_TYPE, + INTERNATIONAL_SMS_TYPE, SMS_TYPE, EMAIL_TYPE, KEY_TYPE_TEST, KEY_TYPE_TEAM, SCHEDULE_NOTIFICATIONS ) from app.service.utils import service_allowed_to_send_to @@ -104,7 +104,7 @@ def validate_and_format_recipient(send_to, key_type, service, notification_type) number=send_to, international=international_phone_info.international ) - else: + elif notification_type == EMAIL_TYPE: return validate_and_format_email_address(email_address=send_to) diff --git a/app/schema_validation/definitions.py b/app/schema_validation/definitions.py index f0fdc9356..f0dbb0a58 100644 --- a/app/schema_validation/definitions.py +++ b/app/schema_validation/definitions.py @@ -14,12 +14,14 @@ uuid = { personalisation = { "type": "object", - "validationMessage": "should contain key value pairs", "code": "1001", # yet to be implemented "link": "link to our error documentation not yet implemented" } +letter_personalisation = dict(personalisation, required=["address_line_1", "address_line_2", "postcode"]) + + https_url = { "type": "string", "format": "uri", diff --git a/app/service/rest.py b/app/service/rest.py index ae57f5390..743fa5d25 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -68,7 +68,6 @@ from app.schemas import ( user_schema, permission_schema, notification_with_template_schema, - notification_with_personalisation_schema, notifications_filter_schema, detailed_service_schema ) @@ -597,3 +596,26 @@ def handle_sql_errror(e): def create_one_off_notification(service_id): resp = send_one_off_notification(service_id, request.get_json()) return jsonify(resp), 201 + + +@service_blueprint.route('/unique', methods=["GET"]) +def is_service_name_unique(): + name, email_from = check_request_args(request) + + name_exists = Service.query.filter_by(name=name).first() + email_from_exists = Service.query.filter_by(email_from=email_from).first() + result = not (name_exists or email_from_exists) + return jsonify(result=result), 200 + + +def check_request_args(request): + name = request.args.get('name', None) + email_from = request.args.get('email_from', None) + errors = [] + if not name: + errors.append({'name': ["Can't be empty"]}) + if not email_from: + errors.append({'email_from': ["Can't be empty"]}) + if errors: + raise InvalidRequest(errors, status_code=400) + return name, email_from diff --git a/app/template_statistics/rest.py b/app/template_statistics/rest.py index 16cd17672..78ae711e4 100644 --- a/app/template_statistics/rest.py +++ b/app/template_statistics/rest.py @@ -7,8 +7,12 @@ from flask import ( from app import redis_store from app.dao.notifications_dao import ( dao_get_template_usage, - dao_get_last_template_usage) -from app.dao.templates_dao import dao_get_templates_for_cache + dao_get_last_template_usage +) +from app.dao.templates_dao import ( + dao_get_templates_for_cache, + dao_get_template_by_id_and_service_id +) from app.schemas import notification_with_template_schema from app.utils import cache_key_for_service_template_counter @@ -52,12 +56,17 @@ def get_template_statistics_for_service_by_day(service_id): @template_statistics.route('/') def get_template_statistics_for_template_id(service_id, template_id): - notification = dao_get_last_template_usage(template_id) - if not notification: + template = dao_get_template_by_id_and_service_id(template_id, service_id) + if not template: message = 'No template found for id {}'.format(template_id) errors = {'template_id': [message]} raise InvalidRequest(errors, status_code=404) - data = notification_with_template_schema.dump(notification).data + + data = None + notification = dao_get_last_template_usage(template_id) + if notification: + data = notification_with_template_schema.dump(notification).data + return jsonify(data=data) diff --git a/app/utils.py b/app/utils.py index ea30a6df2..3cb69294e 100644 --- a/app/utils.py +++ b/app/utils.py @@ -30,7 +30,7 @@ def url_with_token(data, url, config): def get_template_instance(template, values): from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE return { - SMS_TYPE: SMSMessageTemplate, EMAIL_TYPE: PlainTextEmailTemplate, LETTER_TYPE: LetterPreviewTemplate + SMS_TYPE: SMSMessageTemplate, EMAIL_TYPE: PlainTextEmailTemplate, LETTER_TYPE: PlainTextEmailTemplate }[template['template_type']](template, values) diff --git a/app/v2/errors.py b/app/v2/errors.py index d38477474..0fa6bddac 100644 --- a/app/v2/errors.py +++ b/app/v2/errors.py @@ -29,10 +29,10 @@ class RateLimitError(InvalidRequest): class BadRequestError(InvalidRequest): - status_code = 400 message = "An error occurred" - def __init__(self, fields=[], message=None): + def __init__(self, fields=[], message=None, status_code=400): + self.status_code = status_code self.fields = fields self.message = message if message else self.message diff --git a/app/v2/notifications/create_response.py b/app/v2/notifications/create_response.py new file mode 100644 index 000000000..7eecfb1b9 --- /dev/null +++ b/app/v2/notifications/create_response.py @@ -0,0 +1,45 @@ + +def create_post_sms_response_from_notification(notification, content, from_number, url_root, scheduled_for): + noti = __create_notification_response(notification, url_root, scheduled_for) + noti['content'] = { + 'from_number': from_number, + 'body': content + } + return noti + + +def create_post_email_response_from_notification(notification, content, subject, email_from, url_root, scheduled_for): + noti = __create_notification_response(notification, url_root, scheduled_for) + noti['content'] = { + "from_email": email_from, + "body": content, + "subject": subject + } + return noti + + +def create_post_letter_response_from_notification(notification, content, subject, url_root, scheduled_for): + noti = __create_notification_response(notification, url_root, scheduled_for) + noti['content'] = { + "body": content, + "subject": subject + } + return noti + + +def __create_notification_response(notification, url_root, scheduled_for): + return { + "id": notification.id, + "reference": notification.client_reference, + "uri": "{}v2/notifications/{}".format(url_root, str(notification.id)), + 'template': { + "id": notification.template_id, + "version": notification.template_version, + "uri": "{}services/{}/templates/{}".format( + url_root, + str(notification.service_id), + str(notification.template_id) + ) + }, + "scheduled_for": scheduled_for if scheduled_for else None + } diff --git a/app/v2/notifications/notification_schemas.py b/app/v2/notifications/notification_schemas.py index 69372c4e1..fefbf5634 100644 --- a/app/v2/notifications/notification_schemas.py +++ b/app/v2/notifications/notification_schemas.py @@ -1,5 +1,5 @@ from app.models import NOTIFICATION_STATUS_TYPES, TEMPLATE_TYPES -from app.schema_validation.definitions import (uuid, personalisation) +from app.schema_validation.definitions import (uuid, personalisation, letter_personalisation) template = { @@ -192,40 +192,44 @@ post_email_response = { } -def create_post_sms_response_from_notification(notification, body, from_number, url_root, service_id, scheduled_for): - return {"id": notification.id, - "reference": notification.client_reference, - "content": {'body': body, - 'from_number': from_number}, - "uri": "{}v2/notifications/{}".format(url_root, str(notification.id)), - "template": __create_template_from_notification(notification=notification, - url_root=url_root, - service_id=service_id), - "scheduled_for": scheduled_for if scheduled_for else None - } +post_letter_request = { + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "POST letter notification schema", + "type": "object", + "title": "POST v2/notifications/letter", + "properties": { + "reference": {"type": "string"}, + "template_id": uuid, + "personalisation": letter_personalisation + }, + "required": ["template_id", "personalisation"] +} +letter_content = { + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Letter content for POST letter notification", + "type": "object", + "title": "notification letter content", + "properties": { + "body": {"type": "string"}, + "subject": {"type": "string"} + }, + "required": ["body", "subject"] +} -def create_post_email_response_from_notification(notification, content, subject, email_from, url_root, service_id, - scheduled_for): - return { - "id": notification.id, - "reference": notification.client_reference, - "content": { - "from_email": email_from, - "body": content, - "subject": subject - }, - "uri": "{}v2/notifications/{}".format(url_root, str(notification.id)), - "template": __create_template_from_notification(notification=notification, - url_root=url_root, - service_id=service_id), - "scheduled_for": scheduled_for if scheduled_for else None - } - - -def __create_template_from_notification(notification, url_root, service_id): - return { - "id": notification.template_id, - "version": notification.template_version, - "uri": "{}services/{}/templates/{}".format(url_root, str(service_id), str(notification.template_id)) - } +post_letter_response = { + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "POST sms notification response schema", + "type": "object", + "title": "response v2/notifications/letter", + "properties": { + "id": uuid, + "reference": {"type": ["string", "null"]}, + "content": letter_content, + "uri": {"type": "string", "format": "uri"}, + "template": template, + # letters cannot be scheduled + "scheduled_for": {"type": "null"} + }, + "required": ["id", "content", "uri", "template"] +} diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index 274005386..d18bb3b4d 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -1,38 +1,54 @@ -from flask import request, jsonify, current_app +import functools + +from flask import request, jsonify, current_app, abort from app import api_user, authenticated_service from app.config import QueueNames -from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY, SCHEDULE_NOTIFICATIONS +from app.dao.jobs_dao import dao_update_job +from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, PRIORITY, KEY_TYPE_TEST, KEY_TYPE_TEAM +from app.celery.tasks import build_dvla_file, update_job_to_sent_to_dvla from app.notifications.process_notifications import ( persist_notification, send_notification_to_queue, simulated_recipient, persist_scheduled_notification) +from app.notifications.process_letter_notifications import ( + create_letter_api_job, + create_letter_notification +) from app.notifications.validators import ( validate_and_format_recipient, check_rate_limiting, - service_has_permission, check_service_can_schedule_notification, check_service_has_permission, validate_template ) from app.schema_validation import validate -from app.utils import get_public_notify_type_text +from app.v2.errors import BadRequestError from app.v2.notifications import v2_notification_blueprint from app.v2.notifications.notification_schemas import ( post_sms_request, - create_post_sms_response_from_notification, post_email_request, - create_post_email_response_from_notification) -from app.v2.errors import BadRequestError + post_letter_request +) +from app.v2.notifications.create_response import ( + create_post_sms_response_from_notification, + create_post_email_response_from_notification, + create_post_letter_response_from_notification +) +from app.variables import LETTER_TEST_API_FILENAME @v2_notification_blueprint.route('/', methods=['POST']) def post_notification(notification_type): if notification_type == EMAIL_TYPE: form = validate(request.get_json(), post_email_request) - else: + elif notification_type == SMS_TYPE: form = validate(request.get_json(), post_sms_request) + elif notification_type == LETTER_TYPE: + form = validate(request.get_json(), post_letter_request) + else: + abort(404) check_service_has_permission(notification_type, authenticated_service.permissions) @@ -41,12 +57,6 @@ def post_notification(notification_type): check_rate_limiting(authenticated_service, api_user) - form_send_to = form['phone_number'] if notification_type == SMS_TYPE else form['email_address'] - send_to = validate_and_format_recipient(send_to=form_send_to, - key_type=api_user.key_type, - service=authenticated_service, - notification_type=notification_type) - template, template_with_content = validate_template( form['template_id'], form.get('personalisation', {}), @@ -54,20 +64,73 @@ def post_notification(notification_type): notification_type, ) + if notification_type == LETTER_TYPE: + notification = process_letter_notification( + letter_data=form, + api_key=api_user, + template=template, + ) + else: + notification = process_sms_or_email_notification( + form=form, + notification_type=notification_type, + api_key=api_user, + template=template, + service=authenticated_service + ) + + if notification_type == SMS_TYPE: + sms_sender = authenticated_service.sms_sender or current_app.config.get('FROM_NUMBER') + create_resp_partial = functools.partial( + create_post_sms_response_from_notification, + from_number=sms_sender + ) + elif notification_type == EMAIL_TYPE: + create_resp_partial = functools.partial( + create_post_email_response_from_notification, + subject=template_with_content.subject, + email_from=authenticated_service.email_from + ) + elif notification_type == LETTER_TYPE: + create_resp_partial = functools.partial( + create_post_letter_response_from_notification, + subject=template_with_content.subject, + ) + + resp = create_resp_partial( + notification=notification, + content=str(template_with_content), + url_root=request.url_root, + scheduled_for=scheduled_for + ) + return jsonify(resp), 201 + + +def process_sms_or_email_notification(*, form, notification_type, api_key, template, service): + form_send_to = form['email_address'] if notification_type == EMAIL_TYPE else form['phone_number'] + + send_to = validate_and_format_recipient(send_to=form_send_to, + key_type=api_key.key_type, + service=service, + notification_type=notification_type) + # Do not persist or send notification to the queue if it is a simulated recipient simulated = simulated_recipient(send_to, notification_type) - notification = persist_notification(template_id=template.id, - template_version=template.version, - recipient=form_send_to, - service=authenticated_service, - personalisation=form.get('personalisation', None), - notification_type=notification_type, - api_key_id=api_user.id, - key_type=api_user.key_type, - client_reference=form.get('reference', None), - simulated=simulated) + notification = persist_notification( + template_id=template.id, + template_version=template.version, + recipient=form_send_to, + service=service, + personalisation=form.get('personalisation', None), + notification_type=notification_type, + api_key_id=api_key.id, + key_type=api_key.key_type, + client_reference=form.get('reference', None), + simulated=simulated + ) + scheduled_for = form.get("scheduled_for", None) if scheduled_for: persist_scheduled_notification(notification.id, form["scheduled_for"]) else: @@ -75,26 +138,34 @@ def post_notification(notification_type): queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None send_notification_to_queue( notification=notification, - research_mode=authenticated_service.research_mode, + research_mode=service.research_mode, queue=queue_name ) else: current_app.logger.info("POST simulated notification for id: {}".format(notification.id)) - if notification_type == SMS_TYPE: - sms_sender = authenticated_service.sms_sender or current_app.config.get('FROM_NUMBER') - resp = create_post_sms_response_from_notification(notification=notification, - body=str(template_with_content), - from_number=sms_sender, - url_root=request.url_root, - service_id=authenticated_service.id, - scheduled_for=scheduled_for) + return notification + + +def process_letter_notification(*, letter_data, api_key, template): + if api_key.key_type == KEY_TYPE_TEAM: + raise BadRequestError(message='Cannot send letters with a team api key', status_code=403) + + job = create_letter_api_job(template) + notification = create_letter_notification(letter_data, job, api_key) + + if api_key.service.research_mode or api_key.key_type == KEY_TYPE_TEST: + + # distinguish real API jobs from test jobs by giving the test jobs a different filename + job.original_file_name = LETTER_TEST_API_FILENAME + dao_update_job(job) + + update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE) else: - resp = create_post_email_response_from_notification(notification=notification, - content=str(template_with_content), - subject=template_with_content.subject, - email_from=authenticated_service.email_from, - url_root=request.url_root, - service_id=authenticated_service.id, - scheduled_for=scheduled_for) - return jsonify(resp), 201 + build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS) + + current_app.logger.info("send job {} for api notification {} to build-dvla-file in the process-job queue".format( + job.id, + notification.id + )) + return notification diff --git a/app/v2/template/get_template.py b/app/v2/template/get_template.py index 4054e161a..8440b231d 100644 --- a/app/v2/template/get_template.py +++ b/app/v2/template/get_template.py @@ -18,5 +18,4 @@ def get_template_by_id(template_id, version=None): template = templates_dao.dao_get_template_by_id_and_service_id( template_id, authenticated_service.id, data.get('version')) - return jsonify(template.serialize()), 200 diff --git a/app/variables.py b/app/variables.py new file mode 100644 index 000000000..513e30b96 --- /dev/null +++ b/app/variables.py @@ -0,0 +1,3 @@ +# all jobs for letters created via the api must have this filename +LETTER_API_FILENAME = 'letter submitted via api' +LETTER_TEST_API_FILENAME = 'test letter submitted via api' diff --git a/application.py b/application.py index b2a59a6fd..5ff7dd1e4 100644 --- a/application.py +++ b/application.py @@ -16,6 +16,7 @@ manager.add_command('db', MigrateCommand) manager.add_command('create_provider_rate', commands.CreateProviderRateCommand) manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) manager.add_command('custom_db_script', commands.CustomDbScript) +manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling) @manager.command diff --git a/aws_run_celery.py b/aws_run_celery.py deleted file mode 100644 index 36a1f480a..000000000 --- a/aws_run_celery.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -from app import notify_celery, create_app -from credstash import getAllSecrets -import os - -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) - -application = create_app("delivery") -application.app_context().push() diff --git a/db.py b/db.py index 3a8da01d4..bc558cbd2 100644 --- a/db.py +++ b/db.py @@ -1,12 +1,8 @@ from flask.ext.script import Manager, Server from flask_migrate import Migrate, MigrateCommand -from app import create_app, db -from credstash import getAllSecrets -import os -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) +from app import create_app, db + application = create_app() diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 597df9f3f..78053a729 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -16,39 +16,39 @@ memory: 1G applications: - name: notify-delivery-celery-beat - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery beat --loglevel=INFO + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO instances: 1 memory: 128M env: NOTIFY_APP_NAME: delivery-celery-beat - name: notify-delivery-worker-database - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks env: NOTIFY_APP_NAME: delivery-worker-database - name: notify-delivery-worker-research - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks env: NOTIFY_APP_NAME: delivery-worker-research - name: notify-delivery-worker-sender - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-tasks,send-sms-tasks,send-email-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms-tasks,send-email-tasks env: NOTIFY_APP_NAME: delivery-worker-sender - name: notify-delivery-worker-periodic - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks instances: 1 env: NOTIFY_APP_NAME: delivery-worker-periodic - name: notify-delivery-worker-priority - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks env: NOTIFY_APP_NAME: delivery-worker-priority - name: notify-delivery-worker - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,notify-internal-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,notify-internal-tasks env: NOTIFY_APP_NAME: delivery-worker diff --git a/migrations/versions/0110_monthly_billing.py b/migrations/versions/0110_monthly_billing.py new file mode 100644 index 000000000..19fa1dbdd --- /dev/null +++ b/migrations/versions/0110_monthly_billing.py @@ -0,0 +1,38 @@ +"""empty message + +Revision ID: 0110_monthly_billing +Revises: 0109_rem_old_noti_status +Create Date: 2017-07-13 14:35:03.183659 + +""" + +# revision identifiers, used by Alembic. +revision = '0110_monthly_billing' +down_revision = '0109_rem_old_noti_status' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +def upgrade(): + + op.create_table('monthly_billing', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('month', sa.String(), nullable=False), + sa.Column('year', sa.Float(), nullable=False), + sa.Column('notification_type', + postgresql.ENUM('email', 'sms', 'letter', name='notification_type', create_type=False), + nullable=False), + sa.Column('monthly_totals', postgresql.JSON(), nullable=False), + sa.Column('updated_at', sa.DateTime, nullable=False), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_monthly_billing_service_id'), 'monthly_billing', ['service_id'], unique=False) + op.create_index(op.f('uix_monthly_billing'), 'monthly_billing', ['service_id', 'month', 'year', 'notification_type'], unique=True) + + +def downgrade(): + op.drop_table('monthly_billing') diff --git a/migrations/versions/0111_drop_old_service_flags.py b/migrations/versions/0111_drop_old_service_flags.py new file mode 100644 index 000000000..fd9bc5487 --- /dev/null +++ b/migrations/versions/0111_drop_old_service_flags.py @@ -0,0 +1,28 @@ +"""empty message + +Revision ID: 0111_drop_old_service_flags +Revises: 0110_monthly_billing +Create Date: 2017-07-12 13:35:45.636618 + +""" + +# revision identifiers, used by Alembic. +revision = '0111_drop_old_service_flags' +down_revision = '0110_monthly_billing' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + op.drop_column('services', 'can_send_letters') + op.drop_column('services', 'can_send_international_sms') + op.drop_column('services_history', 'can_send_letters') + op.drop_column('services_history', 'can_send_international_sms') + + +def downgrade(): + op.add_column('services_history', sa.Column('can_send_international_sms', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False)) + op.add_column('services_history', sa.Column('can_send_letters', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False)) + op.add_column('services', sa.Column('can_send_international_sms', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False)) + op.add_column('services', sa.Column('can_send_letters', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False)) diff --git a/migrations/versions/0112_add_start_end_dates.py b/migrations/versions/0112_add_start_end_dates.py new file mode 100644 index 000000000..cbb96b64c --- /dev/null +++ b/migrations/versions/0112_add_start_end_dates.py @@ -0,0 +1,43 @@ +"""empty message + +Revision ID: 0112_add_start_end_dates +Revises: 0111_drop_old_service_flags +Create Date: 2017-07-12 13:35:45.636618 + +""" +from datetime import datetime +from alembic import op +import sqlalchemy as sa +from app.dao.date_util import get_month_start_end_date + +down_revision = '0111_drop_old_service_flags' +revision = '0112_add_start_end_dates' + + +def upgrade(): + op.drop_index('uix_monthly_billing', 'monthly_billing') + op.alter_column('monthly_billing', 'month', nullable=True) + op.alter_column('monthly_billing', 'year', nullable=True) + op.add_column('monthly_billing', sa.Column('start_date', sa.DateTime)) + op.add_column('monthly_billing', sa.Column('end_date', sa.DateTime)) + conn = op.get_bind() + results = conn.execute("Select id, month, year from monthly_billing") + res = results.fetchall() + for x in res: + start_date, end_date = get_month_start_end_date( + datetime(int(x.year), datetime.strptime(x.month, '%B').month, 1)) + conn.execute("update monthly_billing set start_date = '{}', end_date = '{}' where id = '{}'".format(start_date, + end_date, + x.id)) + op.alter_column('monthly_billing', 'start_date', nullable=False) + op.alter_column('monthly_billing', 'end_date', nullable=False) + op.create_index(op.f('uix_monthly_billing'), 'monthly_billing', ['service_id', 'start_date', 'notification_type'], + unique=True) + + +def downgrade(): + op.drop_column('monthly_billing', 'start_date') + op.drop_column('monthly_billing', 'end_date') + + op.create_index(op.f('uix_monthly_billing'), 'monthly_billing', + ['service_id', 'month', 'year', 'notification_type'], unique=True) diff --git a/migrations/versions/0113_job_created_by_nullable.py b/migrations/versions/0113_job_created_by_nullable.py new file mode 100644 index 000000000..c6a391523 --- /dev/null +++ b/migrations/versions/0113_job_created_by_nullable.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 0113_job_created_by_nullable +Revises: 0112_add_start_end_dates +Create Date: 2017-07-27 11:12:34.938086 + +""" + +# revision identifiers, used by Alembic. +revision = '0113_job_created_by_nullable' +down_revision = '0112_add_start_end_dates' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + op.alter_column('jobs', 'created_by_id', nullable=True) + + +def downgrade(): + # This will error if there are any jobs with no created_by - we'll have to decide how to handle those as and when + # we downgrade + op.alter_column('jobs', 'created_by_id', nullable=False) diff --git a/requirements.txt b/requirements.txt index 684bc8ff0..79da8b7ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,21 +6,20 @@ Flask-SQLAlchemy==2.0 psycopg2==2.6.2 SQLAlchemy==1.0.15 SQLAlchemy-Utils==0.32.9 -PyJWT==1.4.2 +PyJWT==1.5.2 marshmallow==2.4.2 marshmallow-sqlalchemy==0.8.0 flask-marshmallow==0.6.2 Flask-Bcrypt==0.6.2 -credstash==1.8.0 boto3==1.4.4 -celery==3.1.25 -monotonic==1.2 +monotonic==1.3 statsd==3.2.1 -jsonschema==2.5.1 +jsonschema==2.6.0 gunicorn==19.6.0 docopt==0.6.2 six==1.10.0 -iso8601==0.1.11 +iso8601==0.1.12 +celery==3.1.25 # pyup: <4 # pin to minor version 3.1.x notifications-python-client==4.3.1 @@ -29,6 +28,6 @@ notifications-python-client==4.3.1 awscli>=1.11,<1.12 awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@17.5.7#egg=notifications-utils==17.5.7 +git+https://github.com/alphagov/notifications-utils.git@18.0.0#egg=notifications-utils==18.0.0 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 62e3e4049..501387acd 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,11 +1,11 @@ -r requirements.txt pycodestyle==2.3.1 -pytest==3.0.1 -pytest-mock==1.2 -pytest-cov==2.3.1 +pytest==3.2.0 +pytest-mock==1.6.2 +pytest-cov==2.5.1 coveralls==1.1 -moto==0.4.25 -flex==5.8.0 -freezegun==0.3.7 -requests-mock==1.0.0 +moto==1.0.1 +flex==6.11.0 +freezegun==0.3.9 +requests-mock==1.3.0 strict-rfc3339==0.7 diff --git a/run_celery.py b/run_celery.py index 066735e63..013499615 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# notify_celery is referenced from manifest_delivery_base.yml, and cannot be removed from app import notify_celery, create_app application = create_app('delivery') diff --git a/scripts/aws_install_dependencies.sh b/scripts/aws_install_dependencies.sh index 03daadb21..b82881d45 100755 --- a/scripts/aws_install_dependencies.sh +++ b/scripts/aws_install_dependencies.sh @@ -5,4 +5,4 @@ set -eo pipefail echo "Install dependencies" cd /home/notify-app/notifications-api; -pip3 install --find-links=wheelhouse -r /home/notify-app/notifications-api/requirements.txt +pip3 install -r /home/notify-app/notifications-api/requirements.txt diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index 2ac52c388..2112e4789 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -52,11 +52,9 @@ function on_exit { kill 0 } -function start_appplication { - exec "$@" 2>&1 | while read line; do echo $line; echo $line >> /home/vcap/logs/app.log.`date +%Y-%m-%d`; done & - LOGGER_PID=$! +function start_application { + exec "$@" & APP_PID=`jobs -p` - echo "Logger process pid: ${LOGGER_PID}" echo "Application process pid: ${APP_PID}" } @@ -69,7 +67,6 @@ function start_aws_logs_agent { function run { while true; do kill -0 ${APP_PID} 2&>/dev/null || break - kill -0 ${LOGGER_PID} 2&>/dev/null || break kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent sleep 1 done @@ -84,7 +81,7 @@ trap "on_exit" EXIT configure_aws_logs # The application has to start first! -start_appplication "$@" +start_application "$@" start_aws_logs_agent diff --git a/server_commands.py b/server_commands.py index 85bf13137..af071db47 100644 --- a/server_commands.py +++ b/server_commands.py @@ -1,7 +1,6 @@ from flask.ext.script import Manager, Server from flask_migrate import Migrate, MigrateCommand from app import (create_app, db, commands) -from credstash import getAllSecrets import os default_env_file = '/home/ubuntu/environment' @@ -11,10 +10,6 @@ if os.path.isfile(default_env_file): with open(default_env_file, 'r') as environment_file: environment = environment_file.readline().strip() -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) - from app.config import configs os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment] diff --git a/tests/app/authentication/test_authentication.py b/tests/app/authentication/test_authentication.py index e423ab9a2..f0420e755 100644 --- a/tests/app/authentication/test_authentication.py +++ b/tests/app/authentication/test_authentication.py @@ -351,7 +351,7 @@ def test_reject_invalid_ips(restrict_ip_sms_app): assert exc_info.value.short_message == 'Unknown source IP address from the SMS provider' -@pytest.mark.xfail(reason='Currently not blocking invalid IPs', strict=True) +@pytest.mark.xfail(reason='Currently not blocking invalid senders', strict=True) def test_illegitimate_ips(restrict_ip_sms_app): with pytest.raises(AuthError) as exc_info: restrict_ip_sms_app.get( @@ -361,4 +361,4 @@ def test_illegitimate_ips(restrict_ip_sms_app): ] ) - assert exc_info.value.short_message == 'Unknown source IP address from the SMS provider' + assert exc_info.value.short_message == 'Unknown IP route not from known SMS provider' diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 0282f8074..8ce518dfb 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -25,8 +25,8 @@ from app.celery.scheduled_tasks import ( send_scheduled_notifications, switch_current_sms_provider_on_slow_delivery, timeout_job_statistics, - timeout_notifications -) + timeout_notifications, + populate_monthly_billing) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.dao.jobs_dao import dao_get_job_by_id from app.dao.notifications_dao import dao_get_scheduled_notifications @@ -36,10 +36,10 @@ from app.dao.provider_details_dao import ( ) from app.models import ( Service, Template, - SMS_TYPE, LETTER_TYPE -) + SMS_TYPE, LETTER_TYPE, + MonthlyBilling) from app.utils import get_london_midnight_in_utc -from tests.app.db import create_notification, create_service, create_template, create_job +from tests.app.db import create_notification, create_service, create_template, create_job, create_rate from tests.app.conftest import ( sample_job as create_sample_job, sample_notification_history as create_notification_history, @@ -98,6 +98,8 @@ def test_should_have_decorated_tasks_functions(): 'remove_transformed_dvla_files' assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \ 'delete_dvla_response_files_older_than_seven_days' + assert populate_monthly_billing.__wrapped__.__name__ == \ + 'populate_monthly_billing' @pytest.fixture(scope='function') @@ -607,3 +609,30 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files( delete_dvla_response_files_older_than_seven_days() remove_s3_mock.assert_not_called() + + +@freeze_time("2017-07-12 02:00:00") +def test_populate_monthly_billing(sample_template): + yesterday = datetime(2017, 7, 11, 13, 30) + create_rate(datetime(2016, 1, 1), 0.0123, 'sms') + create_notification(template=sample_template, status='delivered', created_at=yesterday) + create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1)) + create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1)) + # not included in billing + create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30)) + + assert len(MonthlyBilling.query.all()) == 0 + populate_monthly_billing() + + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 1 + assert monthly_billing[0].service_id == sample_template.service_id + assert monthly_billing[0].start_date == datetime(2017, 6, 30, 23) + assert monthly_billing[0].end_date == datetime(2017, 7, 31, 22, 59, 59, 99999) + assert monthly_billing[0].notification_type == 'sms' + assert len(monthly_billing[0].monthly_totals) == 1 + assert sorted(monthly_billing[0].monthly_totals[0]) == sorted({'international': False, + 'rate_multiplier': 1, + 'billing_units': 3, + 'rate': 0.0123, + 'total_cost': 0.0369}) diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 4fb05d422..30a9d4210 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -39,7 +39,7 @@ from app.dao.invited_user_dao import save_invited_user from app.dao.provider_rates_dao import create_provider_rates from app.clients.sms.firetext import FiretextClient from tests import create_authorization_header -from tests.app.db import create_user, create_template, create_notification +from tests.app.db import create_user, create_template, create_notification, create_api_key @pytest.yield_fixture @@ -255,8 +255,8 @@ def sample_template_without_email_permission(notify_db, notify_db_session): @pytest.fixture -def sample_letter_template(sample_service): - return create_template(sample_service, template_type=LETTER_TYPE) +def sample_letter_template(sample_service_full_permissions): + return create_template(sample_service_full_permissions, template_type=LETTER_TYPE) @pytest.fixture(scope='function') @@ -397,17 +397,18 @@ def sample_email_job(notify_db, @pytest.fixture -def sample_letter_job(sample_service, sample_letter_template): +def sample_letter_job(sample_letter_template): + service = sample_letter_template.service data = { 'id': uuid.uuid4(), - 'service_id': sample_service.id, - 'service': sample_service, + 'service_id': service.id, + 'service': service, 'template_id': sample_letter_template.id, 'template_version': sample_letter_template.version, 'original_file_name': 'some.csv', 'notification_count': 1, 'created_at': datetime.utcnow(), - 'created_by': sample_service.created_by, + 'created_by': service.created_by, } job = Job(**data) dao_create_job(job) @@ -429,7 +430,7 @@ def sample_notification_with_job( sent_at=None, billable_units=1, personalisation=None, - api_key_id=None, + api_key=None, key_type=KEY_TYPE_NORMAL ): if job is None: @@ -448,7 +449,7 @@ def sample_notification_with_job( sent_at=sent_at, billable_units=billable_units, personalisation=personalisation, - api_key_id=api_key_id, + api_key=api_key, key_type=key_type ) @@ -468,7 +469,7 @@ def sample_notification( sent_at=None, billable_units=1, personalisation=None, - api_key_id=None, + api_key=None, key_type=KEY_TYPE_NORMAL, sent_by=None, client_reference=None, @@ -483,6 +484,12 @@ def sample_notification( if template is None: template = sample_template(notify_db, notify_db_session, service=service) + if job is None and api_key is None: + # we didn't specify in test - lets create it + api_key = ApiKey.query.filter(ApiKey.service == template.service, ApiKey.key_type == key_type).first() + if not api_key: + api_key = create_api_key(template.service, key_type=key_type) + notification_id = uuid.uuid4() if to_field: @@ -507,8 +514,9 @@ def sample_notification( 'billable_units': billable_units, 'personalisation': personalisation, 'notification_type': template.template_type, - 'api_key_id': api_key_id, - 'key_type': key_type, + 'api_key': api_key, + 'api_key_id': api_key and api_key.id, + 'key_type': api_key.key_type if api_key else key_type, 'sent_by': sent_by, 'updated_at': created_at if status in NOTIFICATION_STATUS_TYPES_COMPLETED else None, 'client_reference': client_reference, @@ -549,11 +557,12 @@ def sample_letter_notification(sample_letter_template): @pytest.fixture(scope='function') def sample_notification_with_api_key(notify_db, notify_db_session): notification = sample_notification(notify_db, notify_db_session) - notification.api_key_id = sample_api_key( + notification.api_key = sample_api_key( notify_db, notify_db_session, name='Test key' - ).id + ) + notification.api_key_id = notification.api_key.id return notification @@ -1026,7 +1035,7 @@ def admin_request(client): headers=[('Content-Type', 'application/json'), create_authorization_header()] ) json_resp = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == _expected_status + assert resp.status_code == _expected_status, json_resp return json_resp @staticmethod @@ -1036,7 +1045,7 @@ def admin_request(client): headers=[create_authorization_header()] ) json_resp = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == _expected_status + assert resp.status_code == _expected_status, json_resp return json_resp return AdminRequest diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py index 760923c5a..1533b4834 100644 --- a/tests/app/dao/test_date_utils.py +++ b/tests/app/dao/test_date_utils.py @@ -1,4 +1,8 @@ -from app.dao.date_util import get_financial_year, get_april_fools +from datetime import datetime + +import pytest + +from app.dao.date_util import get_financial_year, get_april_fools, get_month_start_end_date def test_get_financial_year(): @@ -11,3 +15,17 @@ def test_get_april_fools(): april_fools = get_april_fools(2016) assert str(april_fools) == '2016-03-31 23:00:00' assert april_fools.tzinfo is None + + +@pytest.mark.parametrize("month, year, expected_start, expected_end", + [ + (7, 2017, datetime(2017, 6, 30, 23, 00, 00), datetime(2017, 7, 31, 22, 59, 59, 99999)), + (2, 2016, datetime(2016, 2, 1, 00, 00, 00), datetime(2016, 2, 29, 23, 59, 59, 99999)), + (2, 2017, datetime(2017, 2, 1, 00, 00, 00), datetime(2017, 2, 28, 23, 59, 59, 99999)), + (9, 2018, datetime(2018, 8, 31, 23, 00, 00), datetime(2018, 9, 30, 22, 59, 59, 99999)), + (12, 2019, datetime(2019, 12, 1, 00, 00, 00), datetime(2019, 12, 31, 23, 59, 59, 99999))]) +def test_get_month_start_end_date(month, year, expected_start, expected_end): + month_year = datetime(year, month, 10, 13, 30, 00) + result = get_month_start_end_date(month_year) + assert result[0] == expected_start + assert result[1] == expected_end diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py new file mode 100644 index 000000000..bb40b7f9f --- /dev/null +++ b/tests/app/dao/test_monthly_billing.py @@ -0,0 +1,184 @@ +from datetime import datetime, timedelta +from freezegun import freeze_time +from freezegun.api import FakeDatetime + +from app import db +from app.dao.monthly_billing_dao import ( + create_or_update_monthly_billing_sms, + get_monthly_billing_entry, + get_monthly_billing_sms, + get_service_ids_that_need_sms_billing_populated +) +from app.models import MonthlyBilling, SMS_TYPE +from tests.app.db import create_notification, create_rate, create_service, create_template + + +def create_sample_monthly_billing_entry( + service_id, + monthly_totals, + start_date, + end_date, + notification_type=SMS_TYPE +): + entry = MonthlyBilling( + service_id=service_id, + notification_type=notification_type, + monthly_totals=monthly_totals, + start_date=start_date, + end_date=end_date + ) + db.session.add(entry) + db.session.commit() + + return entry + + +def test_add_monthly_billing(sample_template): + jan = datetime(2017, 1, 1) + feb = datetime(2017, 2, 15) + create_rate(start_date=jan, value=0.0158, notification_type=SMS_TYPE) + create_rate(start_date=datetime(2017, 3, 31, 23, 00, 00), value=0.123, notification_type=SMS_TYPE) + create_notification(template=sample_template, created_at=jan, billable_units=1, status='delivered') + create_notification(template=sample_template, created_at=feb, billable_units=2, status='delivered') + + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=jan) + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=feb) + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 2 + assert monthly_billing[0].start_date == datetime(2017, 1, 1) + assert monthly_billing[1].start_date == datetime(2017, 2, 1) + + january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=jan) + expected_jan = {"billing_units": 1, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 1 * 0.0158} + assert_monthly_billing(january, sample_template.service_id, 1, expected_jan, + start_date=datetime(2017, 1, 1), end_date=datetime(2017, 1, 31)) + + february = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=feb) + expected_feb = {"billing_units": 2, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 2 * 0.0158} + assert_monthly_billing(february, sample_template.service_id, 1, expected_feb, + start_date=datetime(2017, 2, 1), end_date=datetime(2017, 2, 28)) + + +def test_add_monthly_billing_multiple_rates_in_a_month(sample_template): + rate_1 = datetime(2016, 12, 1) + rate_2 = datetime(2017, 1, 15) + create_rate(start_date=rate_1, value=0.0158, notification_type=SMS_TYPE) + create_rate(start_date=rate_2, value=0.0124, notification_type=SMS_TYPE) + + create_notification(template=sample_template, created_at=datetime(2017, 1, 1), billable_units=1, status='delivered') + create_notification(template=sample_template, created_at=datetime(2017, 1, 14, 23, 59), billable_units=1, + status='delivered') + + create_notification(template=sample_template, created_at=datetime(2017, 1, 15), billable_units=2, + status='delivered') + create_notification(template=sample_template, created_at=datetime(2017, 1, 17, 13, 30, 57), billable_units=4, + status='delivered') + + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=rate_2) + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 1 + assert monthly_billing[0].start_date == datetime(2017, 1, 1) + + january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=rate_2) + first_row = {"billing_units": 2, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 3 * 0.0158} + assert_monthly_billing(january, sample_template.service_id, 2, first_row, + start_date=datetime(2017, 1, 1), end_date=datetime(2017, 1, 1)) + second_row = {"billing_units": 6, + "rate_multiplier": 1, + "international": False, + "rate": 0.0124, + "total_cost": 1 * 0.0124} + assert sorted(january.monthly_totals[1]) == sorted(second_row) + + +def test_update_monthly_billing_overwrites_old_totals(sample_template): + july = datetime(2017, 7, 1) + create_rate(july, 0.123, SMS_TYPE) + create_notification(template=sample_template, created_at=datetime(2017, 7, 2), billable_units=1, status='delivered') + with freeze_time('2017-07-20 02:30:00'): + create_or_update_monthly_billing_sms(sample_template.service_id, july) + first_update = get_monthly_billing_sms(sample_template.service_id, july) + expected = {"billing_units": 1, + "rate_multiplier": 1, + "international": False, + "rate": 0.123, + "total_cost": 1 * 0.123} + assert_monthly_billing(first_update, sample_template.service_id, 1, expected, + start_date=datetime(2017, 6, 30, 23), end_date=datetime(2017, 7, 31, 23, 59, 59, 99999)) + first_updated_at = first_update.updated_at + with freeze_time('2017-07-20 03:30:00'): + create_notification(template=sample_template, created_at=datetime(2017, 7, 5), billable_units=2, + status='delivered') + + create_or_update_monthly_billing_sms(sample_template.service_id, july) + second_update = get_monthly_billing_sms(sample_template.service_id, july) + expected_update = {"billing_units": 3, + "rate_multiplier": 1, + "international": False, + "rate": 0.123, + "total_cost": 3 * 0.123} + assert_monthly_billing(second_update, sample_template.service_id, 1, expected_update, + start_date=datetime(2017, 6, 30, 23), end_date=datetime(2017, 7, 31, 23, 59, 59, 99999)) + assert second_update.updated_at == FakeDatetime(2017, 7, 20, 3, 30) + assert first_updated_at != second_update.updated_at + + +def assert_monthly_billing(monthly_billing, service_id, expected_len, first_row, start_date, end_date): + assert monthly_billing.service_id == service_id + assert len(monthly_billing.monthly_totals) == expected_len + assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row) + + +def test_get_service_id(notify_db_session): + service_1 = create_service(service_name="Service One") + template_1 = create_template(service=service_1) + service_2 = create_service(service_name="Service Two") + template_2 = create_template(service=service_2) + create_notification(template=template_1, created_at=datetime(2017, 6, 30, 13, 30), status='delivered') + create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered') + create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30)) + create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30)) + services = get_service_ids_that_need_sms_billing_populated(start_date=datetime(2017, 7, 1), + end_date=datetime(2017, 7, 16)) + expected_services = [service_1.id, service_2.id] + assert sorted([x.service_id for x in services]) == sorted(expected_services) + + +def test_get_monthly_billing_entry_filters_by_service(notify_db, notify_db_session): + service_1 = create_service(service_name="Service One") + service_2 = create_service(service_name="Service Two") + now = datetime.utcnow() + + create_sample_monthly_billing_entry( + service_id=service_1.id, + monthly_totals=[], + start_date=now, + end_date=now + timedelta(days=30) + ) + + create_sample_monthly_billing_entry( + service_id=service_2.id, + monthly_totals=[], + start_date=now, + end_date=now + timedelta(days=30) + ) + + entry = get_monthly_billing_entry(service_2.id, now, SMS_TYPE) + + assert entry.start_date == now + assert entry.service_id == service_2.id diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index 6e50c8718..58680f39d 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -46,7 +46,7 @@ from app.dao.notifications_dao import ( dao_created_scheduled_notification, dao_get_scheduled_notifications, set_scheduled_notification_to_processed) from app.dao.services_dao import dao_update_service -from tests.app.db import create_notification +from tests.app.db import create_notification, create_api_key from tests.app.conftest import ( sample_notification, sample_template, @@ -117,14 +117,14 @@ def test_template_usage_should_ignore_test_keys( notify_db_session, created_at=two_minutes_ago, template=sms, - api_key_id=sample_team_api_key.id, + api_key=sample_team_api_key, key_type=KEY_TYPE_TEAM) sample_notification( notify_db, notify_db_session, created_at=one_minute_ago, template=sms, - api_key_id=sample_test_api_key.id, + api_key=sample_test_api_key, key_type=KEY_TYPE_TEST) results = dao_get_last_template_usage(sms.id) @@ -169,11 +169,11 @@ def test_template_history_should_ignore_test_keys( sms = sample_template(notify_db, notify_db_session) sample_notification( - notify_db, notify_db_session, template=sms, api_key_id=sample_api_key.id, key_type=KEY_TYPE_NORMAL) + notify_db, notify_db_session, template=sms, api_key=sample_api_key, key_type=KEY_TYPE_NORMAL) sample_notification( - notify_db, notify_db_session, template=sms, api_key_id=sample_team_api_key.id, key_type=KEY_TYPE_TEAM) + notify_db, notify_db_session, template=sms, api_key=sample_team_api_key, key_type=KEY_TYPE_TEAM) sample_notification( - notify_db, notify_db_session, template=sms, api_key_id=sample_test_api_key.id, key_type=KEY_TYPE_TEST) + notify_db, notify_db_session, template=sms, api_key=sample_test_api_key, key_type=KEY_TYPE_TEST) sample_notification( notify_db, notify_db_session, template=sms) @@ -836,13 +836,16 @@ def test_get_notification_by_id(notify_db, notify_db_session, sample_template): assert notification_from_db.scheduled_notification.scheduled_for == datetime(2017, 5, 5, 14, 15) -def test_get_notifications_by_reference(notify_db, notify_db_session, sample_service): +def test_get_notifications_by_reference(sample_template): client_reference = 'some-client-ref' assert len(Notification.query.all()) == 0 - sample_notification(notify_db, notify_db_session, client_reference=client_reference) - sample_notification(notify_db, notify_db_session, client_reference=client_reference) - sample_notification(notify_db, notify_db_session, client_reference='other-ref') - all_notifications = get_notifications_for_service(sample_service.id, client_reference=client_reference).items + create_notification(sample_template, client_reference=client_reference) + create_notification(sample_template, client_reference=client_reference) + create_notification(sample_template, client_reference='other-ref') + all_notifications = get_notifications_for_service( + sample_template.service_id, + client_reference=client_reference + ).items assert len(all_notifications) == 2 @@ -1066,22 +1069,22 @@ def test_should_not_delete_notification_history(notify_db, notify_db_session, sa @freeze_time("2016-01-10") -def test_should_limit_notifications_return_by_day_limit_plus_one(notify_db, notify_db_session, sample_service): +def test_should_limit_notifications_return_by_day_limit_plus_one(sample_template): assert len(Notification.query.all()) == 0 # create one notification a day between 1st and 9th for i in range(1, 11): past_date = '2016-01-{0:02d}'.format(i) with freeze_time(past_date): - sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow(), status="failed") + create_notification(sample_template, created_at=datetime.utcnow(), status="failed") all_notifications = Notification.query.all() assert len(all_notifications) == 10 - all_notifications = get_notifications_for_service(sample_service.id, limit_days=10).items + all_notifications = get_notifications_for_service(sample_template.service_id, limit_days=10).items assert len(all_notifications) == 10 - all_notifications = get_notifications_for_service(sample_service.id, limit_days=1).items + all_notifications = get_notifications_for_service(sample_template.service_id, limit_days=1).items assert len(all_notifications) == 2 @@ -1302,42 +1305,20 @@ def test_dao_timeout_notifications_doesnt_affect_letters(sample_letter_template) assert updated == 0 -def test_should_return_notifications_excluding_jobs_by_default(notify_db, notify_db_session, sample_service): - assert len(Notification.query.all()) == 0 +def test_should_return_notifications_excluding_jobs_by_default(sample_template, sample_job, sample_api_key): + with_job = create_notification(sample_template, job=sample_job) + without_job = create_notification(sample_template, api_key=sample_api_key) - job = sample_job(notify_db, notify_db_session) - with_job = sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), status="delivered", job=job - ) - without_job = sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), status="delivered" - ) + include_jobs = get_notifications_for_service(sample_template.service_id, include_jobs=True).items + assert len(include_jobs) == 2 - all_notifications = Notification.query.all() - assert len(all_notifications) == 2 + exclude_jobs_by_default = get_notifications_for_service(sample_template.service_id).items + assert len(exclude_jobs_by_default) == 1 + assert exclude_jobs_by_default[0].id == without_job.id - all_notifications = get_notifications_for_service(sample_service.id).items - assert len(all_notifications) == 1 - assert all_notifications[0].id == without_job.id - - -def test_should_return_notifications_including_jobs(notify_db, notify_db_session, sample_service): - assert len(Notification.query.all()) == 0 - - job = sample_job(notify_db, notify_db_session) - with_job = sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), status="delivered", job=job - ) - - all_notifications = Notification.query.all() - assert len(all_notifications) == 1 - - all_notifications = get_notifications_for_service(sample_service.id).items - assert len(all_notifications) == 0 - - all_notifications = get_notifications_for_service(sample_service.id, limit_days=1, include_jobs=True).items - assert len(all_notifications) == 1 - assert all_notifications[0].id == with_job.id + exclude_jobs_manually = get_notifications_for_service(sample_template.service_id, include_jobs=False).items + assert len(exclude_jobs_manually) == 1 + assert exclude_jobs_manually[0].id == without_job.id def test_get_notifications_created_by_api_or_csv_are_returned_correctly_excluding_test_key_notifications( @@ -1353,15 +1334,15 @@ def test_get_notifications_created_by_api_or_csv_are_returned_correctly_excludin notify_db, notify_db_session, created_at=datetime.utcnow(), job=sample_job ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_api_key, key_type=sample_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_team_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_team_api_key, key_type=sample_team_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_test_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_test_api_key, key_type=sample_test_api_key.key_type ) @@ -1394,15 +1375,15 @@ def test_get_notifications_with_a_live_api_key_type( notify_db, notify_db_session, created_at=datetime.utcnow(), job=sample_job ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_api_key, key_type=sample_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_team_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_team_api_key, key_type=sample_team_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_test_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_test_api_key, key_type=sample_test_api_key.key_type ) @@ -1432,15 +1413,15 @@ def test_get_notifications_with_a_test_api_key_type( notify_db, notify_db_session, created_at=datetime.utcnow(), job=sample_job ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_api_key, key_type=sample_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_team_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_team_api_key, key_type=sample_team_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_test_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_test_api_key, key_type=sample_test_api_key.key_type ) @@ -1467,15 +1448,15 @@ def test_get_notifications_with_a_team_api_key_type( notify_db, notify_db_session, created_at=datetime.utcnow(), job=sample_job ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_api_key, key_type=sample_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_team_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_team_api_key, key_type=sample_team_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_test_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_test_api_key, key_type=sample_test_api_key.key_type ) @@ -1503,15 +1484,15 @@ def test_should_exclude_test_key_notifications_by_default( ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_api_key, key_type=sample_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_team_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_team_api_key, key_type=sample_team_api_key.key_type ) sample_notification( - notify_db, notify_db_session, created_at=datetime.utcnow(), api_key_id=sample_test_api_key.id, + notify_db, notify_db_session, created_at=datetime.utcnow(), api_key=sample_test_api_key, key_type=sample_test_api_key.key_type ) @@ -1784,13 +1765,15 @@ def test_dao_update_notifications_sent_to_dvla(notify_db, notify_db_session, sam assert history.updated_at -def test_dao_update_notifications_sent_to_dvla_does_update_history_if_test_key( - notify_db, notify_db_session, sample_letter_template, sample_api_key): - job = sample_job(notify_db=notify_db, notify_db_session=notify_db_session, template=sample_letter_template) +def test_dao_update_notifications_sent_to_dvla_does_update_history_if_test_key(sample_letter_job): + api_key = create_api_key(sample_letter_job.service, key_type=KEY_TYPE_TEST) notification = create_notification( - template=sample_letter_template, job=job, api_key_id=sample_api_key.id, key_type='test') + sample_letter_job.template, + job=sample_letter_job, + api_key=api_key + ) - updated_count = dao_update_notifications_sent_to_dvla(job_id=job.id, provider='some provider') + updated_count = dao_update_notifications_sent_to_dvla(job_id=sample_letter_job.id, provider='some provider') assert updated_count == 1 updated_notification = Notification.query.get(notification.id) @@ -1798,7 +1781,7 @@ def test_dao_update_notifications_sent_to_dvla_does_update_history_if_test_key( assert updated_notification.sent_by == 'some provider' assert updated_notification.sent_at assert updated_notification.updated_at - assert not NotificationHistory.query.get(notification.id) + assert NotificationHistory.query.count() == 0 def test_dao_get_notifications_by_to_field(sample_template): diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py index 83bfa264c..394c71540 100644 --- a/tests/app/dao/test_notification_usage_dao.py +++ b/tests/app/dao/test_notification_usage_dao.py @@ -1,13 +1,15 @@ +import pytest import uuid from datetime import datetime, timedelta +from freezegun import freeze_time -import pytest from flask import current_app from app.dao.date_util import get_financial_year from app.dao.notification_usage_dao import ( - get_rates_for_year, + get_rates_for_daterange, get_yearly_billing_data, + get_billing_data_for_month, get_monthly_billing_data, get_total_billable_units_for_sent_sms_notifications_in_date_range, discover_rate_bounds_for_billing_query @@ -16,30 +18,35 @@ from app.models import ( Rate, NOTIFICATION_DELIVERED, NOTIFICATION_STATUS_TYPES_BILLABLE, - NOTIFICATION_STATUS_TYPES_NON_BILLABLE) -from tests.app.conftest import sample_notification, sample_email_template, sample_letter_template, sample_service -from tests.app.db import create_notification -from freezegun import freeze_time - + NOTIFICATION_STATUS_TYPES_NON_BILLABLE, + SMS_TYPE, +) +from tests.app.conftest import ( + sample_notification, + sample_email_template, + sample_letter_template, + sample_service +) +from tests.app.db import create_notification, create_rate from tests.conftest import set_config -def test_get_rates_for_year(notify_db, notify_db_session): +def test_get_rates_for_daterange(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) set_up_rate(notify_db, datetime(2017, 3, 31, 23), 0.0158) start_date, end_date = get_financial_year(2017) - rates = get_rates_for_year(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, 'sms') assert len(rates) == 1 assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-03-31 23:00:00" assert rates[0].rate == 0.0158 -def test_get_rates_for_year_multiple_result_per_year(notify_db, notify_db_session): +def test_get_rates_for_daterange_multiple_result_per_year(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) set_up_rate(notify_db, datetime(2017, 4, 1), 0.0158) start_date, end_date = get_financial_year(2016) - rates = get_rates_for_year(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, 'sms') assert len(rates) == 2 assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00" assert rates[0].rate == 0.015 @@ -47,12 +54,12 @@ def test_get_rates_for_year_multiple_result_per_year(notify_db, notify_db_sessio assert rates[1].rate == 0.016 -def test_get_rates_for_year_returns_correct_rates(notify_db, notify_db_session): +def test_get_rates_for_daterange_returns_correct_rates(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2017) - rates_2017 = get_rates_for_year(start_date, end_date, 'sms') + rates_2017 = get_rates_for_daterange(start_date, end_date, 'sms') assert len(rates_2017) == 2 assert datetime.strftime(rates_2017[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-09-01 00:00:00" assert rates_2017[0].rate == 0.016 @@ -60,43 +67,56 @@ def test_get_rates_for_year_returns_correct_rates(notify_db, notify_db_session): assert rates_2017[1].rate == 0.0175 -def test_get_rates_for_year_in_the_future(notify_db, notify_db_session): +def test_get_rates_for_daterange_in_the_future(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2018) - rates = get_rates_for_year(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, 'sms') assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" assert rates[0].rate == 0.0175 -def test_get_rates_for_year_returns_empty_list_if_year_is_before_earliest_rate(notify_db, notify_db_session): +def test_get_rates_for_daterange_returns_empty_list_if_year_is_before_earliest_rate(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2015) - rates = get_rates_for_year(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, 'sms') assert rates == [] -def test_get_rates_for_year_early_rate(notify_db, notify_db_session): +def test_get_rates_for_daterange_early_rate(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2015, 6, 1), 0.014) set_up_rate(notify_db, datetime(2016, 6, 1), 0.015) set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2016) - rates = get_rates_for_year(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, 'sms') assert len(rates) == 3 -def test_get_rates_for_year_edge_case(notify_db, notify_db_session): +def test_get_rates_for_daterange_edge_case(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 3, 31, 23, 00), 0.015) set_up_rate(notify_db, datetime(2017, 3, 31, 23, 00), 0.0175) start_date, end_date = get_financial_year(2016) - rates = get_rates_for_year(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, 'sms') assert len(rates) == 1 assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-03-31 23:00:00" assert rates[0].rate == 0.015 +def test_get_rates_for_daterange_where_daterange_is_one_month_that_falls_between_rate_valid_from( + notify_db, notify_db_session +): + set_up_rate(notify_db, datetime(2017, 1, 1), 0.175) + set_up_rate(notify_db, datetime(2017, 3, 31), 0.123) + start_date = datetime(2017, 2, 1, 00, 00, 00) + end_date = datetime(2017, 2, 28, 23, 59, 59, 99999) + rates = get_rates_for_daterange(start_date, end_date, 'sms') + assert len(rates) == 1 + assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-01-01 00:00:00" + assert rates[0].rate == 0.175 + + def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) set_up_rate(notify_db, datetime(2016, 6, 1), 0.0158) @@ -254,8 +274,7 @@ def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_sessi assert results[3] == ('June', 4, 1, False, 'sms', 0.0175) -def test_get_monthly_billing_data_with_no_notifications_for_year(notify_db, notify_db_session, sample_template, - sample_email_template): +def test_get_monthly_billing_data_with_no_notifications_for_daterange(notify_db, notify_db_session, sample_template): set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) results = get_monthly_billing_data(sample_template.service_id, 2016) assert len(results) == 0 @@ -710,3 +729,46 @@ def test_deducts_free_tier_from_bill_across_rate_boundaries( )[1] == expected_cost finally: current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] = start_value + + +def test_get_yearly_billing_data_for_start_date_before_rate_returns_empty( + sample_template +): + create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE) + + results = get_yearly_billing_data( + service_id=sample_template.service_id, + year=2015 + ) + + assert not results + + +@freeze_time("2016-05-01") +def test_get_billing_data_for_month_where_start_date_before_rate_returns_empty( + sample_template +): + create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE) + + results = get_monthly_billing_data( + service_id=sample_template.service_id, + year=2015 + ) + + assert not results + + +@freeze_time("2016-05-01") +def test_get_monthly_billing_data_where_start_date_before_rate_returns_empty( + sample_template +): + now = datetime.utcnow() + create_rate(now, 0.014, SMS_TYPE) + + results = get_billing_data_for_month( + service_id=sample_template.service_id, + start_date=now - timedelta(days=2), + end_date=now - timedelta(days=1) + ) + + assert not results diff --git a/tests/app/dao/test_provider_rates_dao.py b/tests/app/dao/test_provider_rates_dao.py index 417612781..c78290a90 100644 --- a/tests/app/dao/test_provider_rates_dao.py +++ b/tests/app/dao/test_provider_rates_dao.py @@ -1,3 +1,4 @@ +import uuid from datetime import datetime from decimal import Decimal from app.dao.provider_rates_dao import create_provider_rates diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 38ef273cd..ee4f4f912 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -552,11 +552,11 @@ def test_fetch_stats_counts_should_ignore_team_key( sample_team_api_key ): # two created email, one failed email, and one created sms - create_notification(notify_db, notify_db_session, api_key_id=sample_api_key.id, key_type=sample_api_key.key_type) + create_notification(notify_db, notify_db_session, api_key=sample_api_key, key_type=sample_api_key.key_type) create_notification( - notify_db, notify_db_session, api_key_id=sample_test_api_key.id, key_type=sample_test_api_key.key_type) + notify_db, notify_db_session, api_key=sample_test_api_key, key_type=sample_test_api_key.key_type) create_notification( - notify_db, notify_db_session, api_key_id=sample_team_api_key.id, key_type=sample_team_api_key.key_type) + notify_db, notify_db_session, api_key=sample_team_api_key, key_type=sample_team_api_key.key_type) create_notification( notify_db, notify_db_session) @@ -757,24 +757,17 @@ def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys(samp ("8", "4", "2")]) # a date range that starts more than 7 days ago def test_fetch_stats_by_date_range_for_all_services_returns_test_notifications(notify_db, notify_db_session, - sample_api_key, start_delta, end_delta, expected): - result_one = create_notification(notify_db, notify_db_session, created_at=datetime.now(), - api_key_id=sample_api_key.id, key_type='test') - create_notification(notify_db, notify_db_session, created_at=datetime.now() - timedelta(days=2), - api_key_id=sample_api_key.id, key_type='test') - create_notification(notify_db, notify_db_session, created_at=datetime.now() - timedelta(days=3), - api_key_id=sample_api_key.id, key_type='test') - create_notification(notify_db, notify_db_session, created_at=datetime.now() - timedelta(days=4), - api_key_id=sample_api_key.id, key_type='normal') - create_notification(notify_db, notify_db_session, created_at=datetime.now() - timedelta(days=4), - api_key_id=sample_api_key.id, key_type='test') - create_notification(notify_db, notify_db_session, created_at=datetime.now() - timedelta(days=8), - api_key_id=sample_api_key.id, key_type='test') - create_notification(notify_db, notify_db_session, created_at=datetime.now() - timedelta(days=8), - api_key_id=sample_api_key.id, key_type='normal') + create_noti = functools.partial(create_notification, notify_db, notify_db_session) + result_one = create_noti(created_at=datetime.now(), key_type='test') + create_noti(created_at=datetime.now() - timedelta(days=2), key_type='test') + create_noti(created_at=datetime.now() - timedelta(days=3), key_type='test') + create_noti(created_at=datetime.now() - timedelta(days=4), key_type='normal') + create_noti(created_at=datetime.now() - timedelta(days=4), key_type='test') + create_noti(created_at=datetime.now() - timedelta(days=8), key_type='test') + create_noti(created_at=datetime.now() - timedelta(days=8), key_type='normal') start_date = (datetime.utcnow() - timedelta(days=int(start_delta))).date() end_date = (datetime.utcnow() - timedelta(days=int(end_delta))).date() diff --git a/tests/app/db.py b/tests/app/db.py index ea9e4cbdd..71bf29bcb 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -1,16 +1,18 @@ from datetime import datetime import uuid - +from app import db from app.dao.jobs_dao import dao_create_job from app.dao.service_inbound_api_dao import save_service_inbound_api from app.models import ( + ApiKey, Service, User, Template, Notification, ScheduledNotification, ServicePermission, + Rate, Job, InboundSms, Organisation, @@ -49,7 +51,9 @@ def create_service( service_id=None, restricted=False, service_permissions=[EMAIL_TYPE, SMS_TYPE], - sms_sender='testing' + sms_sender='testing', + research_mode=False, + active=True, ): service = Service( name=service_name, @@ -57,9 +61,13 @@ def create_service( restricted=restricted, email_from=service_name.lower().replace(' ', '.'), created_by=user or create_user(), - sms_sender=sms_sender + sms_sender=sms_sender, ) dao_create_service(service, service.created_by, service_id, service_permissions=service_permissions) + + service.active = active + service.research_mode = research_mode + return service @@ -96,7 +104,7 @@ def create_notification( updated_at=None, billable_units=1, personalisation=None, - api_key_id=None, + api_key=None, key_type=KEY_TYPE_NORMAL, sent_by=None, client_reference=None, @@ -113,14 +121,20 @@ def create_notification( sent_at = sent_at or datetime.utcnow() updated_at = updated_at or datetime.utcnow() + if job is None and api_key is None: + # we didn't specify in test - lets create it + api_key = ApiKey.query.filter(ApiKey.service == template.service, ApiKey.key_type == key_type).first() + if not api_key: + api_key = create_api_key(template.service, key_type=key_type) + data = { 'id': uuid.uuid4(), 'to': to_field, - 'job_id': job.id if job else None, + 'job_id': job and job.id, 'job': job, 'service_id': template.service.id, 'service': template.service, - 'template_id': template.id if template else None, + 'template_id': template and template.id, 'template': template, 'template_version': template.version, 'status': status, @@ -130,8 +144,9 @@ def create_notification( 'billable_units': billable_units, 'personalisation': personalisation, 'notification_type': template.template_type, - 'api_key_id': api_key_id, - 'key_type': key_type, + 'api_key': api_key, + 'api_key_id': api_key and api_key.id, + 'key_type': api_key.key_type if api_key else key_type, 'sent_by': sent_by, 'updated_at': updated_at, 'client_reference': client_reference, @@ -239,3 +254,25 @@ def create_organisation(colour='blue', logo='test_x2.png', name='test_org_1'): dao_create_organisation(organisation) return organisation + + +def create_rate(start_date, value, notification_type): + rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=notification_type) + db.session.add(rate) + db.session.commit() + return rate + + +def create_api_key(service, key_type=KEY_TYPE_NORMAL): + id_ = uuid.uuid4() + api_key = ApiKey( + service=service, + name='{} api key {}'.format(key_type, id_), + created_by=service.created_by, + key_type=key_type, + id=id_, + secret=uuid.uuid4() + ) + db.session.add(api_key) + db.session.commit() + return api_key diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index e1a4ec4a2..d14fee6d8 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -435,29 +435,22 @@ def test_get_html_email_renderer_prepends_logo_path(notify_api): renderer = send_to_providers.get_html_email_options(service) - assert renderer['brand_logo'] == 'http://localhost:6012/static/images/email-template/crests/justice-league.png' + assert renderer['brand_logo'] == 'http://static-logos.notify.tools/justice-league.png' @pytest.mark.parametrize('base_url, expected_url', [ # don't change localhost to prevent errors when testing locally - ('http://localhost:6012', 'http://localhost:6012/static/sub-path/filename.png'), - # on other environments, replace www with staging - ('https://www.notifications.service.gov.uk', 'https://static.notifications.service.gov.uk/sub-path/filename.png'), - - # staging and preview do not have cloudfront running, so should act as localhost - pytest.mark.xfail(('https://www.notify.works', 'https://static.notify.works/sub-path/filename.png')), - pytest.mark.xfail(('https://www.staging-notify.works', 'https://static.notify.works/sub-path/filename.png')), - pytest.mark.xfail(('https://notify.works', 'https://static.notify.works/sub-path/filename.png')), - pytest.mark.xfail(('https://staging-notify.works', 'https://static.notify.works/sub-path/filename.png')), - # these tests should be removed when cloudfront works on staging/preview - ('https://www.notify.works', 'https://www.notify.works/static/sub-path/filename.png'), - ('https://www.staging-notify.works', 'https://www.staging-notify.works/static/sub-path/filename.png'), + ('http://localhost:6012', 'http://static-logos.notify.tools/filename.png'), + ('https://www.notifications.service.gov.uk', 'https://static-logos.notifications.service.gov.uk/filename.png'), + ('https://notify.works', 'https://static-logos.notify.works/filename.png'), + ('https://staging-notify.works', 'https://static-logos.staging-notify.works/filename.png'), + ('https://www.notify.works', 'https://static-logos.notify.works/filename.png'), + ('https://www.staging-notify.works', 'https://static-logos.staging-notify.works/filename.png'), ]) def test_get_logo_url_works_for_different_environments(base_url, expected_url): - branding_path = '/sub-path/' logo_file = 'filename.png' - logo_url = send_to_providers.get_logo_url(base_url, branding_path, logo_file) + logo_url = send_to_providers.get_logo_url(base_url, logo_file) assert logo_url == expected_url diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index ca25e0769..7ce45b51e 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -276,7 +276,6 @@ def test_create_job_returns_400_if_missing_data(notify_api, sample_template, moc assert resp_json['result'] == 'error' assert 'Missing data for required field.' in resp_json['message']['original_file_name'] assert 'Missing data for required field.' in resp_json['message']['notification_count'] - assert 'Missing data for required field.' in resp_json['message']['id'] def test_create_job_returns_404_if_template_does_not_exist(notify_api, sample_service, mocker): diff --git a/tests/app/letters/test_send_letter_jobs.py b/tests/app/letters/test_send_letter_jobs.py index 1d32baf12..b0d34b987 100644 --- a/tests/app/letters/test_send_letter_jobs.py +++ b/tests/app/letters/test_send_letter_jobs.py @@ -2,6 +2,8 @@ import uuid from flask import json +from app.variables import LETTER_TEST_API_FILENAME + from tests import create_authorization_header @@ -41,7 +43,7 @@ def test_send_letter_jobs_throws_validation_error(client, mocker): assert not mock_celery.called -def test_send_letter_jobs_throws_validation_error(client, sample_letter_job): +def test_get_letter_jobs_excludes_non_letter_jobs(client, sample_letter_job, sample_job): auth_header = create_authorization_header() response = client.get( path='/letter-jobs', @@ -53,3 +55,11 @@ def test_send_letter_jobs_throws_validation_error(client, sample_letter_job): assert json_resp['data'][0]['id'] == str(sample_letter_job.id) assert json_resp['data'][0]['service_name']['name'] == sample_letter_job.service.name assert json_resp['data'][0]['job_status'] == 'pending' + + +def test_get_letter_jobs_excludes_test_jobs(admin_request, sample_letter_job): + sample_letter_job.original_file_name = LETTER_TEST_API_FILENAME + + json_resp = admin_request.get('letter-job.get_letter_jobs') + + assert len(json_resp['data']) == 0 diff --git a/tests/app/notifications/rest/test_send_notification.py b/tests/app/notifications/rest/test_send_notification.py index 3822395cd..73eab744a 100644 --- a/tests/app/notifications/rest/test_send_notification.py +++ b/tests/app/notifications/rest/test_send_notification.py @@ -572,128 +572,119 @@ def test_should_not_send_sms_if_team_api_key_and_not_a_service_user(notify_api, ] == json_resp['message']['to'] -def test_should_send_email_if_team_api_key_and_a_service_user(notify_api, sample_email_template, fake_uuid, mocker): - with notify_api.test_request_context(), notify_api.test_client() as client: - mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') - mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid) +def test_should_send_email_if_team_api_key_and_a_service_user(client, sample_email_template, fake_uuid, mocker): + mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') + mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid) - data = { - 'to': sample_email_template.service.created_by.email_address, - 'template': sample_email_template.id - } - api_key = ApiKey(service=sample_email_template.service, - name='team_key', - created_by=sample_email_template.created_by, - key_type=KEY_TYPE_TEAM) - save_model_api_key(api_key) - auth_header = create_jwt_token(secret=api_key.secret, client_id=str(api_key.service_id)) + data = { + 'to': sample_email_template.service.created_by.email_address, + 'template': sample_email_template.id + } + auth_header = create_authorization_header(service_id=sample_email_template.service_id, key_type=KEY_TYPE_TEAM) - response = client.post( - path='/notifications/email', - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) + response = client.post( + path='/notifications/email', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) - app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( - [fake_uuid], - queue='send-email-tasks' - ) - assert response.status_code == 201 + app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( + [fake_uuid], + queue='send-email-tasks' + ) + assert response.status_code == 201 @pytest.mark.parametrize('restricted', [True, False]) @pytest.mark.parametrize('limit', [0, 1]) def test_should_send_sms_to_anyone_with_test_key( - notify_api, sample_template, mocker, restricted, limit, fake_uuid + client, sample_template, mocker, restricted, limit, fake_uuid ): - with notify_api.test_request_context(), notify_api.test_client() as client: - mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') - mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid) + mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid) - data = { - 'to': '07811111111', - 'template': sample_template.id - } - sample_template.service.restricted = restricted - sample_template.service.message_limit = limit - api_key = ApiKey( - service=sample_template.service, - name='test_key', - created_by=sample_template.created_by, - key_type=KEY_TYPE_TEST - ) - save_model_api_key(api_key) - auth_header = create_jwt_token(secret=api_key.secret, client_id=str(api_key.service_id)) + data = { + 'to': '07811111111', + 'template': sample_template.id + } + sample_template.service.restricted = restricted + sample_template.service.message_limit = limit + api_key = ApiKey( + service=sample_template.service, + name='test_key', + created_by=sample_template.created_by, + key_type=KEY_TYPE_TEST + ) + save_model_api_key(api_key) + auth_header = create_jwt_token(secret=api_key.secret, client_id=str(api_key.service_id)) - response = client.post( - path='/notifications/sms', - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] - ) - app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( - [fake_uuid], queue='research-mode-tasks' - ) - assert response.status_code == 201 + response = client.post( + path='/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] + ) + app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( + [fake_uuid], queue='research-mode-tasks' + ) + assert response.status_code == 201 @pytest.mark.parametrize('restricted', [True, False]) @pytest.mark.parametrize('limit', [0, 1]) def test_should_send_email_to_anyone_with_test_key( - notify_api, sample_email_template, mocker, restricted, limit, fake_uuid + client, sample_email_template, mocker, restricted, limit, fake_uuid ): - with notify_api.test_request_context(), notify_api.test_client() as client: - mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') - mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid) + mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') + mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid) - data = { - 'to': 'anyone123@example.com', - 'template': sample_email_template.id - } - sample_email_template.service.restricted = restricted - sample_email_template.service.message_limit = limit - api_key = ApiKey( - service=sample_email_template.service, - name='test_key', - created_by=sample_email_template.created_by, - key_type=KEY_TYPE_TEST - ) - save_model_api_key(api_key) - auth_header = create_jwt_token(secret=api_key.secret, client_id=str(api_key.service_id)) + data = { + 'to': 'anyone123@example.com', + 'template': sample_email_template.id + } + sample_email_template.service.restricted = restricted + sample_email_template.service.message_limit = limit + api_key = ApiKey( + service=sample_email_template.service, + name='test_key', + created_by=sample_email_template.created_by, + key_type=KEY_TYPE_TEST + ) + save_model_api_key(api_key) + auth_header = create_jwt_token(secret=api_key.secret, client_id=str(api_key.service_id)) - response = client.post( - path='/notifications/email', - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] - ) + response = client.post( + path='/notifications/email', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] + ) - app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( - [fake_uuid], queue='research-mode-tasks' - ) - assert response.status_code == 201 + app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( + [fake_uuid], queue='research-mode-tasks' + ) + assert response.status_code == 201 -def test_should_send_sms_if_team_api_key_and_a_service_user(notify_api, sample_template, fake_uuid, mocker): - with notify_api.test_request_context(), notify_api.test_client() as client: - mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') - mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid) +def test_should_send_sms_if_team_api_key_and_a_service_user(client, sample_template, fake_uuid, mocker): + mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid) - data = { - 'to': sample_template.service.created_by.mobile_number, - 'template': sample_template.id - } - api_key = ApiKey(service=sample_template.service, - name='team_key', - created_by=sample_template.created_by, - key_type=KEY_TYPE_TEAM) - save_model_api_key(api_key) - auth_header = create_jwt_token(secret=api_key.secret, client_id=str(api_key.service_id)) + data = { + 'to': sample_template.service.created_by.mobile_number, + 'template': sample_template.id + } + api_key = ApiKey(service=sample_template.service, + name='team_key', + created_by=sample_template.created_by, + key_type=KEY_TYPE_TEAM) + save_model_api_key(api_key) + auth_header = create_jwt_token(secret=api_key.secret, client_id=str(api_key.service_id)) - response = client.post( - path='/notifications/sms', - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) + response = client.post( + path='/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='send-sms-tasks') - assert response.status_code == 201 + app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='send-sms-tasks') + assert response.status_code == 201 @pytest.mark.parametrize('template_type,queue_name', [ @@ -710,7 +701,8 @@ def test_should_persist_notification( queue_name ): mocked = mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type)) - mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid) + mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid) + template = sample_template if template_type == SMS_TYPE else sample_email_template to = sample_template.service.created_by.mobile_number if template_type == SMS_TYPE \ else sample_email_template.service.created_by.email_address @@ -757,7 +749,8 @@ def test_should_delete_notification_and_return_error_if_sqs_fails( 'app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type), side_effect=Exception("failed to talk to SQS") ) - mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid) + mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid) + template = sample_template if template_type == SMS_TYPE else sample_email_template to = sample_template.service.created_by.mobile_number if template_type == SMS_TYPE \ else sample_email_template.service.created_by.email_address diff --git a/tests/app/notifications/test_process_letter_notifications.py b/tests/app/notifications/test_process_letter_notifications.py new file mode 100644 index 000000000..8d964b706 --- /dev/null +++ b/tests/app/notifications/test_process_letter_notifications.py @@ -0,0 +1,84 @@ +import pytest + +from app.dao.services_dao import dao_archive_service +from app.models import Job +from app.models import JOB_STATUS_READY_TO_SEND +from app.models import LETTER_TYPE +from app.models import Notification +from app.notifications.process_letter_notifications import create_letter_api_job +from app.notifications.process_letter_notifications import create_letter_notification +from app.v2.errors import InvalidRequest +from app.variables import LETTER_API_FILENAME + +from tests.app.db import create_service +from tests.app.db import create_template + + +def test_create_job_rejects_inactive_service(notify_db_session): + service = create_service() + template = create_template(service, template_type=LETTER_TYPE) + dao_archive_service(service.id) + + with pytest.raises(InvalidRequest) as exc_info: + create_letter_api_job(template) + + assert exc_info.value.message == 'Service {} is inactive'.format(service.id) + + +def test_create_job_rejects_archived_template(sample_letter_template): + sample_letter_template.archived = True + + with pytest.raises(InvalidRequest) as exc_info: + create_letter_api_job(sample_letter_template) + + assert exc_info.value.message == 'Template {} is deleted'.format(sample_letter_template.id) + + +def test_create_job_creates_job(sample_letter_template): + job = create_letter_api_job(sample_letter_template) + + assert job == Job.query.one() + assert job.original_file_name == LETTER_API_FILENAME + assert job.service == sample_letter_template.service + assert job.template_id == sample_letter_template.id + assert job.template_version == sample_letter_template.version + assert job.notification_count == 1 + assert job.job_status == JOB_STATUS_READY_TO_SEND + assert job.created_by is None + + +def test_create_letter_notification_creates_notification(sample_letter_job, sample_api_key): + data = { + 'personalisation': { + 'address_line_1': 'The Queen', + 'address_line_2': 'Buckingham Palace', + 'postcode': 'SW1 1AA', + } + } + + notification = create_letter_notification(data, sample_letter_job, sample_api_key) + + assert notification == Notification.query.one() + assert notification.job == sample_letter_job + assert notification.template == sample_letter_job.template + assert notification.api_key == sample_api_key + assert notification.notification_type == LETTER_TYPE + assert notification.key_type == sample_api_key.key_type + assert notification.job_row_number == 0 + assert notification.reference is not None + assert notification.client_reference is None + + +def test_create_letter_notification_sets_reference(sample_letter_job, sample_api_key): + data = { + 'personalisation': { + 'address_line_1': 'The Queen', + 'address_line_2': 'Buckingham Palace', + 'postcode': 'SW1 1AA', + }, + 'reference': 'foo' + } + + notification = create_letter_notification(data, sample_letter_job, sample_api_key) + + assert notification.client_reference == 'foo' diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py index 3e4d1d9c2..fbe9787c6 100644 --- a/tests/app/notifications/test_process_notification.py +++ b/tests/app/notifications/test_process_notification.py @@ -51,10 +51,18 @@ def test_persist_notification_creates_and_save_to_db(sample_template, sample_api assert Notification.query.count() == 0 assert NotificationHistory.query.count() == 0 - notification = persist_notification(sample_template.id, sample_template.version, '+447111111111', - sample_template.service, {}, 'sms', sample_api_key.id, - sample_api_key.key_type, job_id=sample_job.id, - job_row_number=100, reference="ref") + notification = persist_notification( + template_id=sample_template.id, + template_version=sample_template.version, + recipient='+447111111111', + service=sample_template.service, + personalisation={}, + notification_type='sms', + api_key_id=sample_api_key.id, + key_type=sample_api_key.key_type, + job_id=sample_job.id, + job_row_number=100, + reference="ref") assert Notification.query.get(notification.id) is not None assert NotificationHistory.query.get(notification.id) is not None @@ -127,14 +135,14 @@ def test_persist_notification_does_not_increment_cache_if_test_key( assert Notification.query.count() == 0 assert NotificationHistory.query.count() == 0 persist_notification( - sample_template.id, - sample_template.version, - '+447111111111', - sample_template.service, - {}, - 'sms', - api_key.id, - api_key.key_type, + template_id=sample_template.id, + template_version=sample_template.version, + recipient='+447111111111', + service=sample_template.service, + personalisation={}, + notification_type='sms', + api_key_id=api_key.id, + key_type=api_key.key_type, job_id=sample_job.id, job_row_number=100, reference="ref", @@ -193,18 +201,33 @@ def test_persist_notification_increments_cache_if_key_exists(sample_template, sa mock_incr = mocker.patch('app.notifications.process_notifications.redis_store.incr') mock_incr_hash_value = mocker.patch('app.notifications.process_notifications.redis_store.increment_hash_value') - persist_notification(sample_template.id, sample_template.version, '+447111111111', - sample_template.service, {}, 'sms', sample_api_key.id, - sample_api_key.key_type, reference="ref") + persist_notification( + template_id=sample_template.id, + template_version=sample_template.version, + recipient='+447111111111', + service=sample_template.service, + personalisation={}, + notification_type='sms', + api_key_id=sample_api_key.id, + key_type=sample_api_key.key_type, + reference="ref" + ) mock_incr.assert_not_called() mock_incr_hash_value.assert_not_called() mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=1) mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash', return_value={sample_template.id, 1}) - persist_notification(sample_template.id, sample_template.version, '+447111111122', - sample_template.service, {}, 'sms', sample_api_key.id, - sample_api_key.key_type, reference="ref2") + persist_notification( + template_id=sample_template.id, + template_version=sample_template.version, + recipient='+447111111122', + service=sample_template.service, + personalisation={}, + notification_type='sms', + api_key_id=sample_api_key.id, + key_type=sample_api_key.key_type, + reference="ref2") mock_incr.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count", ) mock_incr_hash_value.assert_called_once_with(cache_key_for_service_template_counter(sample_template.service_id), sample_template.id) diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 811fc1290..6a0b53925 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -9,51 +9,53 @@ from app.dao.notifications_dao import dao_update_notification from app.dao.api_key_dao import save_model_api_key from app.dao.templates_dao import dao_update_template from app.models import ApiKey, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST + from tests import create_authorization_header from tests.app.conftest import sample_notification as create_sample_notification +from tests.app.db import create_notification, create_api_key @pytest.mark.parametrize('type', ('email', 'sms')) def test_get_notification_by_id(client, sample_notification, sample_email_notification, type): - if type == 'email': - notification_to_get = sample_email_notification - if type == 'sms': - notification_to_get = sample_notification + if type == 'email': + notification_to_get = sample_email_notification + if type == 'sms': + notification_to_get = sample_notification - auth_header = create_authorization_header(service_id=notification_to_get.service_id) - response = client.get( - '/notifications/{}'.format(notification_to_get.id), - headers=[auth_header]) + auth_header = create_authorization_header(service_id=notification_to_get.service_id) + response = client.get( + '/notifications/{}'.format(notification_to_get.id), + headers=[auth_header]) - assert response.status_code == 200 - notification = json.loads(response.get_data(as_text=True))['data']['notification'] - assert notification['status'] == 'created' - assert notification['template'] == { - 'id': str(notification_to_get.template.id), - 'name': notification_to_get.template.name, - 'template_type': notification_to_get.template.template_type, - 'version': 1 - } - assert notification['to'] == notification_to_get.to - assert notification['service'] == str(notification_to_get.service_id) - assert notification['body'] == notification_to_get.template.content - assert notification.get('subject', None) == notification_to_get.subject + assert response.status_code == 200 + notification = json.loads(response.get_data(as_text=True))['data']['notification'] + assert notification['status'] == 'created' + assert notification['template'] == { + 'id': str(notification_to_get.template.id), + 'name': notification_to_get.template.name, + 'template_type': notification_to_get.template.template_type, + 'version': 1 + } + assert notification['to'] == notification_to_get.to + assert notification['service'] == str(notification_to_get.service_id) + assert notification['body'] == notification_to_get.template.content + assert notification.get('subject', None) == notification_to_get.subject @pytest.mark.parametrize("id", ["1234-badly-formatted-id-7890", "0"]) @pytest.mark.parametrize('type', ('email', 'sms')) def test_get_notification_by_invalid_id(client, sample_notification, sample_email_notification, id, type): - if type == 'email': - notification_to_get = sample_email_notification - if type == 'sms': - notification_to_get = sample_notification - auth_header = create_authorization_header(service_id=notification_to_get.service_id) + if type == 'email': + notification_to_get = sample_email_notification + if type == 'sms': + notification_to_get = sample_notification + auth_header = create_authorization_header(service_id=notification_to_get.service_id) - response = client.get( - '/notifications/{}'.format(id), - headers=[auth_header]) + response = client.get( + '/notifications/{}'.format(id), + headers=[auth_header]) - assert response.status_code == 405 + assert response.status_code == 405 def test_get_notifications_empty_result(client, sample_api_key): @@ -156,7 +158,7 @@ def test_normal_api_key_returns_notifications_created_from_jobs_and_from_api( api_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=sample_api_key.id + api_key=sample_api_key ) api_notification.job = None @@ -200,19 +202,19 @@ def test_get_all_notifications_only_returns_notifications_of_matching_type( normal_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=normal_api_key.id, + api_key=normal_api_key, key_type=KEY_TYPE_NORMAL ) team_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=team_api_key.id, + api_key=team_api_key, key_type=KEY_TYPE_TEAM ) test_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=test_api_key.id, + api_key=test_api_key, key_type=KEY_TYPE_TEST ) @@ -236,54 +238,18 @@ def test_get_all_notifications_only_returns_notifications_of_matching_type( @pytest.mark.parametrize('key_type', [KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST]) def test_no_api_keys_return_job_notifications_by_default( client, - notify_db, - notify_db_session, - sample_service, + sample_template, sample_job, key_type ): - team_api_key = ApiKey(service=sample_service, - name='team_api_key', - created_by=sample_service.created_by, - key_type=KEY_TYPE_TEAM) - save_model_api_key(team_api_key) + team_api_key = create_api_key(sample_template.service, KEY_TYPE_TEAM) + normal_api_key = create_api_key(sample_template.service, KEY_TYPE_NORMAL) + test_api_key = create_api_key(sample_template.service, KEY_TYPE_TEST) - normal_api_key = ApiKey(service=sample_service, - name='normal_api_key', - created_by=sample_service.created_by, - key_type=KEY_TYPE_NORMAL) - save_model_api_key(normal_api_key) - - test_api_key = ApiKey(service=sample_service, - name='test_api_key', - created_by=sample_service.created_by, - key_type=KEY_TYPE_TEST) - save_model_api_key(test_api_key) - - job_notification = create_sample_notification( - notify_db, - notify_db_session, - api_key_id=normal_api_key.id, - job=sample_job - ) - normal_notification = create_sample_notification( - notify_db, - notify_db_session, - api_key_id=normal_api_key.id, - key_type=KEY_TYPE_NORMAL - ) - team_notification = create_sample_notification( - notify_db, - notify_db_session, - api_key_id=team_api_key.id, - key_type=KEY_TYPE_TEAM - ) - test_notification = create_sample_notification( - notify_db, - notify_db_session, - api_key_id=test_api_key.id, - key_type=KEY_TYPE_TEST - ) + job_notification = create_notification(sample_template, job=sample_job) + normal_notification = create_notification(sample_template, api_key=normal_api_key) + team_notification = create_notification(sample_template, api_key=team_api_key) + test_notification = create_notification(sample_template, api_key=test_api_key) notification_objs = { KEY_TYPE_NORMAL: normal_notification, @@ -336,25 +302,24 @@ def test_only_normal_api_keys_can_return_job_notifications( job_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=normal_api_key.id, job=sample_job ) normal_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=normal_api_key.id, + api_key=normal_api_key, key_type=KEY_TYPE_NORMAL ) team_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=team_api_key.id, + api_key=team_api_key, key_type=KEY_TYPE_TEAM ) test_notification = create_sample_notification( notify_db, notify_db_session, - api_key_id=test_api_key.id, + api_key=test_api_key, key_type=KEY_TYPE_TEST ) @@ -505,20 +470,10 @@ def test_filter_by_template_type(client, notify_db, notify_db_session, sample_te def test_filter_by_multiple_template_types(client, - notify_db, - notify_db_session, sample_template, sample_email_template): - notification_1 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_template) - notification_2 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_email_template) + notification_1 = create_notification(sample_template) + notification_2 = create_notification(sample_email_template) auth_header = create_authorization_header(service_id=sample_email_template.service_id) @@ -529,23 +484,12 @@ def test_filter_by_multiple_template_types(client, assert response.status_code == 200 notifications = json.loads(response.get_data(as_text=True)) assert len(notifications['notifications']) == 2 - set(['sms', 'email']) == set( - [x['template']['template_type'] for x in notifications['notifications']]) + assert {'sms', 'email'} == set(x['template']['template_type'] for x in notifications['notifications']) -def test_filter_by_status(client, notify_db, notify_db_session, sample_email_template): - notification_1 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_email_template, - status="delivered") - - notification_2 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_email_template) +def test_filter_by_status(client, sample_email_template): + notification_1 = create_notification(sample_email_template, status="delivered") + notification_2 = create_notification(sample_email_template) auth_header = create_authorization_header(service_id=sample_email_template.service_id) @@ -559,58 +503,27 @@ def test_filter_by_status(client, notify_db, notify_db_session, sample_email_tem assert response.status_code == 200 -def test_filter_by_multiple_statuss(client, - notify_db, - notify_db_session, - sample_email_template): - notification_1 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_email_template, - status="delivered") - - notification_2 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_email_template, - status='sending') +def test_filter_by_multiple_statuses(client, sample_email_template): + notification_1 = create_notification(sample_email_template, status="delivered") + notification_2 = create_notification(sample_email_template, status='sending') auth_header = create_authorization_header(service_id=sample_email_template.service_id) response = client.get( '/notifications?status=delivered&status=sending', - headers=[auth_header]) + headers=[auth_header] + ) assert response.status_code == 200 notifications = json.loads(response.get_data(as_text=True)) assert len(notifications['notifications']) == 2 - set(['delivered', 'sending']) == set( - [x['status'] for x in notifications['notifications']]) + assert {'delivered', 'sending'} == set(x['status'] for x in notifications['notifications']) -def test_filter_by_status_and_template_type(client, - notify_db, - notify_db_session, - sample_template, - sample_email_template): - notification_1 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_template) - notification_2 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_email_template) - notification_3 = create_sample_notification( - notify_db, - notify_db_session, - service=sample_email_template.service, - template=sample_email_template, - status="delivered") +def test_filter_by_status_and_template_type(client, sample_template, sample_email_template): + notification_1 = create_notification(sample_template) + notification_2 = create_notification(sample_email_template) + notification_3 = create_notification(sample_email_template, status="delivered") auth_header = create_authorization_header(service_id=sample_email_template.service_id) @@ -625,15 +538,9 @@ def test_filter_by_status_and_template_type(client, assert notifications['notifications'][0]['status'] == 'delivered' -def test_get_notification_by_id_returns_merged_template_content(notify_db, - notify_db_session, - client, - sample_template_with_placeholders): +def test_get_notification_by_id_returns_merged_template_content(client, sample_template_with_placeholders): - sample_notification = create_sample_notification(notify_db, - notify_db_session, - template=sample_template_with_placeholders, - personalisation={"name": "world"}) + sample_notification = create_notification(sample_template_with_placeholders, personalisation={"name": "world"}) auth_header = create_authorization_header(service_id=sample_notification.service_id) @@ -649,15 +556,13 @@ def test_get_notification_by_id_returns_merged_template_content(notify_db, def test_get_notification_by_id_returns_merged_template_content_for_email( - notify_db, - notify_db_session, client, sample_email_template_with_placeholders ): - sample_notification = create_sample_notification(notify_db, - notify_db_session, - template=sample_email_template_with_placeholders, - personalisation={"name": "world"}) + sample_notification = create_notification( + sample_email_template_with_placeholders, + personalisation={"name": "world"} + ) auth_header = create_authorization_header(service_id=sample_notification.service_id) response = client.get( diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 7b0478916..77a3a9d40 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1,26 +1,23 @@ -from datetime import datetime, timedelta, date -from functools import partial import json import uuid +from datetime import datetime, timedelta, date +from functools import partial from unittest.mock import ANY import pytest from flask import url_for, current_app from freezegun import freeze_time -from app import encryption -from app.dao.users_dao import save_model_user from app.dao.services_dao import dao_remove_user_from_service from app.dao.templates_dao import dao_redact_template +from app.dao.users_dao import save_model_user from app.models import ( User, Organisation, Rate, Service, ServicePermission, Notification, DVLA_ORG_LAND_REGISTRY, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST, EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE, ) - from tests import create_authorization_header -from tests.app.db import create_template, create_service_inbound_api, create_user, create_notification from tests.app.conftest import ( sample_service as create_service, sample_user_service_permission as create_user_service_permission, @@ -28,8 +25,8 @@ from tests.app.conftest import ( sample_notification_history as create_notification_history, sample_notification_with_job ) +from tests.app.db import create_template, create_service_inbound_api, create_notification from tests.app.db import create_user -from tests.conftest import set_config_values def test_get_service_list(client, service_factory): @@ -1362,24 +1359,20 @@ def test_get_all_notifications_for_service_including_ones_made_by_jobs( def test_get_only_api_created_notifications_for_service( - client, - notify_db, - notify_db_session, - sample_service + admin_request, + sample_job, + sample_template ): - with_job = sample_notification_with_job(notify_db, notify_db_session, service=sample_service) - without_job = create_sample_notification(notify_db, notify_db_session, service=sample_service) + with_job = create_notification(sample_template, job=sample_job) + without_job = create_notification(sample_template) - auth_header = create_authorization_header() - - response = client.get( - path='/service/{}/notifications?include_jobs=false'.format(sample_service.id), - headers=[auth_header]) - - resp = json.loads(response.get_data(as_text=True)) + resp = admin_request.get( + 'service.get_all_notifications_for_service', + service_id=sample_template.service_id, + include_jobs=False + ) assert len(resp['notifications']) == 1 assert resp['notifications'][0]['id'] == str(without_job.id) - assert response.status_code == 200 def test_set_sms_sender_for_service(client, sample_service): @@ -2315,3 +2308,32 @@ def test_search_for_notification_by_to_field_returns_personlisation( assert len(notifications) == 1 assert 'personalisation' in notifications[0].keys() assert notifications[0]['personalisation']['name'] == 'Foo' + + +def test_is_service_name_unique_returns_200_if_unique(client): + response = client.get('/service/unique?name=something&email_from=something', + headers=[create_authorization_header()]) + assert response.status_code == 200 + assert json.loads(response.get_data(as_text=True)) == {"result": True} + + +@pytest.mark.parametrize('name, email_from', + [("something unique", "something"), + ("unique", "something.unique"), + ("something unique", "something.unique") + ]) +def test_is_service_name_unique_returns_200_and_false(client, notify_db, notify_db_session, name, email_from): + create_service(notify_db=notify_db, notify_db_session=notify_db_session, + service_name='something unique', email_from='something.unique') + response = client.get('/service/unique?name={}&email_from={}'.format(name, email_from), + headers=[create_authorization_header()]) + assert response.status_code == 200 + assert json.loads(response.get_data(as_text=True)) == {"result": False} + + +def test_is_service_name_unique_returns_400_when_name_does_not_exist(client): + response = client.get('/service/unique', headers=[create_authorization_header()]) + assert response.status_code == 400 + json_resp = json.loads(response.get_data(as_text=True)) + assert json_resp["message"][0]["name"] == ["Can't be empty"] + assert json_resp["message"][1]["email_from"] == ["Can't be empty"] diff --git a/tests/app/template_statistics/test_rest.py b/tests/app/template_statistics/test_rest.py index 4f79541ec..a13993768 100644 --- a/tests/app/template_statistics/test_rest.py +++ b/tests/app/template_statistics/test_rest.py @@ -5,7 +5,12 @@ import pytest from freezegun import freeze_time from tests import create_authorization_header -from tests.app.conftest import (sample_template as create_sample_template, sample_notification, sample_email_template) +from tests.app.conftest import ( + sample_template as create_sample_template, + sample_notification, + sample_notification_history, + sample_email_template +) def test_get_all_template_statistics_with_bad_arg_returns_400(client, sample_service): @@ -211,8 +216,8 @@ def test_get_template_statistics_by_id_returns_last_notification( def test_get_template_statistics_for_template_returns_empty_if_no_statistics( - client, - sample_template, + client, + sample_template, ): auth_header = create_authorization_header() @@ -221,7 +226,44 @@ def test_get_template_statistics_for_template_returns_empty_if_no_statistics( headers=[('Content-Type', 'application/json'), auth_header], ) + assert response.status_code == 200 + json_resp = json.loads(response.get_data(as_text=True)) + assert not json_resp['data'] + + +def test_get_template_statistics_raises_error_for_nonexistent_template( + client, + sample_service, + fake_uuid +): + auth_header = create_authorization_header() + + response = client.get( + '/service/{}/template-statistics/{}'.format(sample_service.id, fake_uuid), + headers=[('Content-Type', 'application/json'), auth_header], + ) + assert response.status_code == 404 json_resp = json.loads(response.get_data(as_text=True)) + assert json_resp['message'] == 'No result found' assert json_resp['result'] == 'error' - assert json_resp['message']['template_id'] == ['No template found for id {}'.format(sample_template.id)] + + +def test_get_template_statistics_by_id_returns_empty_for_old_notification( + notify_db, + notify_db_session, + client, + sample_template +): + sample_notification_history(notify_db, notify_db_session, sample_template) + + auth_header = create_authorization_header() + + response = client.get( + '/service/{}/template-statistics/{}'.format(sample_template.service.id, sample_template.id), + headers=[('Content-Type', 'application/json'), auth_header], + ) + + assert response.status_code == 200 + json_resp = json.loads(response.get_data(as_text=True))['data'] + assert not json_resp diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index 15ff3dd55..a393fc8bb 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -122,8 +122,8 @@ def test_extract_cloudfoundry_config_populates_other_vars(): extract_cloudfoundry_config() assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgres uri' - assert os.environ['LOGGING_STDOUT_JSON'] == '1' assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌' + assert os.environ['NOTIFY_LOG_PATH'] == '/home/vcap/logs/app.log' @pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 63a9bff9a..c1e1cbeee 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -57,24 +57,3 @@ def test_load_config_if_cloudfoundry_not_available(monkeypatch, reload_config): def test_cloudfoundry_config_has_different_defaults(): # these should always be set on Sandbox assert config.Sandbox.REDIS_ENABLED is False - - -def test_logging_stdout_json_defaults_to_off(reload_config): - os.environ.pop('LOGGING_STDOUT_JSON', None) - assert config.Config.LOGGING_STDOUT_JSON is False - - -def test_logging_stdout_json_sets_to_off_if_not_recognised(reload_config): - os.environ['LOGGING_STDOUT_JSON'] = 'foo' - - importlib.reload(config) - - assert config.Config.LOGGING_STDOUT_JSON is False - - -def test_logging_stdout_json_sets_to_on_if_set_to_1(reload_config): - os.environ['LOGGING_STDOUT_JSON'] = '1' - - importlib.reload(config) - - assert config.Config.LOGGING_STDOUT_JSON is True diff --git a/tests/app/test_model.py b/tests/app/test_model.py index d662c6670..66b96ba4e 100644 --- a/tests/app/test_model.py +++ b/tests/app/test_model.py @@ -6,6 +6,7 @@ from app import encryption from app.models import ( ServiceWhitelist, Notification, + SMS_TYPE, MOBILE_TYPE, EMAIL_TYPE, NOTIFICATION_CREATED, @@ -18,6 +19,7 @@ from tests.app.conftest import ( sample_template as create_sample_template, sample_notification_with_job as create_sample_notification_with_job ) +from tests.app.db import create_notification @pytest.mark.parametrize('mobile_number', [ @@ -148,21 +150,71 @@ def test_notification_for_csv_returns_bst_correctly(notify_db, notify_db_session assert serialized['created_at'] == 'Monday 27 March 2017 at 00:01' -def test_notification_personalisation_getter_returns_empty_dict_from_None(sample_notification): - sample_notification._personalisation = None - assert sample_notification.personalisation == {} +def test_notification_personalisation_getter_returns_empty_dict_from_None(): + noti = Notification() + noti._personalisation = None + assert noti.personalisation == {} -def test_notification_personalisation_getter_always_returns_empty_dict(sample_notification): - sample_notification._personalisation = encryption.encrypt({}) - assert sample_notification.personalisation == {} +def test_notification_personalisation_getter_always_returns_empty_dict(): + noti = Notification() + noti._personalisation = encryption.encrypt({}) + assert noti.personalisation == {} @pytest.mark.parametrize('input_value', [ None, {} ]) -def test_notification_personalisation_setter_always_sets_empty_dict(sample_notification, input_value): - sample_notification.personalisation = input_value +def test_notification_personalisation_setter_always_sets_empty_dict(input_value): + noti = Notification() + noti.personalisation = input_value - assert sample_notification._personalisation == encryption.encrypt({}) + assert noti._personalisation == encryption.encrypt({}) + + +def test_notification_subject_is_none_for_sms(): + assert Notification(notification_type=SMS_TYPE).subject is None + + +def test_notification_subject_fills_in_placeholders_for_email(sample_email_template_with_placeholders): + noti = create_notification(sample_email_template_with_placeholders, personalisation={'name': 'hello'}) + assert noti.subject == 'hello' + + +def test_notification_subject_fills_in_placeholders_for_letter(sample_letter_template): + sample_letter_template.subject = '((name))' + noti = create_notification(sample_letter_template, personalisation={'name': 'hello'}) + assert noti.subject == 'hello' + + +def test_letter_notification_serializes_with_address(client, sample_letter_notification): + sample_letter_notification.personalisation = { + 'address_line_1': 'foo', + 'address_line_3': 'bar', + 'address_line_5': None, + 'postcode': 'SW1 1AA' + } + res = sample_letter_notification.serialize() + assert res['line_1'] == 'foo' + assert res['line_2'] is None + assert res['line_3'] == 'bar' + assert res['line_4'] is None + assert res['line_5'] is None + assert res['line_6'] is None + assert res['postcode'] == 'SW1 1AA' + + +def test_sms_notification_serializes_without_subject(client, sample_template): + res = sample_template.serialize() + assert res['subject'] is None + + +def test_email_notification_serializes_with_subject(client, sample_email_template): + res = sample_email_template.serialize() + assert res['subject'] == 'Email Subject' + + +def test_letter_notification_serializes_with_subject(client, sample_letter_template): + res = sample_letter_template.serialize() + assert res['subject'] == 'Template subject' diff --git a/tests/app/v2/notifications/test_notification_schemas.py b/tests/app/v2/notifications/test_notification_schemas.py index 0aafb38cc..83ad8c5f7 100644 --- a/tests/app/v2/notifications/test_notification_schemas.py +++ b/tests/app/v2/notifications/test_notification_schemas.py @@ -127,7 +127,7 @@ def test_post_sms_schema_with_personalisation_that_is_not_a_dict(): error = json.loads(str(e.value)) assert len(error.get('errors')) == 1 assert error['errors'] == [{'error': 'ValidationError', - 'message': "personalisation should contain key value pairs"}] + 'message': "personalisation not_a_dict is not of type object"}] assert error.get('status_code') == 400 assert len(error.keys()) == 2 diff --git a/tests/app/v2/notifications/test_post_letter_notifications.py b/tests/app/v2/notifications/test_post_letter_notifications.py new file mode 100644 index 000000000..429e53eb8 --- /dev/null +++ b/tests/app/v2/notifications/test_post_letter_notifications.py @@ -0,0 +1,234 @@ +import uuid + +from flask import json +from flask import url_for +import pytest + +from app.models import EMAIL_TYPE +from app.models import Job +from app.models import KEY_TYPE_NORMAL +from app.models import KEY_TYPE_TEAM +from app.models import KEY_TYPE_TEST +from app.models import LETTER_TYPE +from app.models import Notification +from app.models import SMS_TYPE +from app.v2.errors import RateLimitError +from app.variables import LETTER_TEST_API_FILENAME +from app.variables import LETTER_API_FILENAME + +from tests import create_authorization_header +from tests.app.db import create_service +from tests.app.db import create_template + + +def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201): + resp = client.post( + url_for('v2_notifications.post_notification', notification_type='letter'), + data=json.dumps(data), + headers=[ + ('Content-Type', 'application/json'), + create_authorization_header(service_id=service_id, key_type=key_type) + ] + ) + json_resp = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == _expected_status, json_resp + return json_resp + + +@pytest.mark.parametrize('reference', [None, 'reference_from_client']) +def test_post_letter_notification_returns_201(client, sample_letter_template, mocker, reference): + mocked = mocker.patch('app.celery.tasks.build_dvla_file.apply_async') + data = { + 'template_id': str(sample_letter_template.id), + 'personalisation': { + 'address_line_1': 'Her Royal Highness Queen Elizabeth II', + 'address_line_2': 'Buckingham Palace', + 'address_line_3': 'London', + 'postcode': 'SW1 1AA', + 'name': 'Lizzie' + } + } + + if reference: + data.update({'reference': reference}) + + resp_json = letter_request(client, data, service_id=sample_letter_template.service_id) + + job = Job.query.one() + assert job.original_file_name == LETTER_API_FILENAME + notification = Notification.query.one() + notification_id = notification.id + assert resp_json['id'] == str(notification_id) + assert resp_json['reference'] == reference + assert resp_json['content']['subject'] == sample_letter_template.subject + assert resp_json['content']['body'] == sample_letter_template.content + assert 'v2/notifications/{}'.format(notification_id) in resp_json['uri'] + assert resp_json['template']['id'] == str(sample_letter_template.id) + assert resp_json['template']['version'] == sample_letter_template.version + assert ( + 'services/{}/templates/{}'.format( + sample_letter_template.service_id, + sample_letter_template.id + ) in resp_json['template']['uri'] + ) + assert not resp_json['scheduled_for'] + + mocked.assert_called_once_with([str(job.id)], queue='job-tasks') + + +def test_post_letter_notification_returns_400_and_missing_template( + client, + sample_service_full_permissions +): + data = { + 'template_id': str(uuid.uuid4()), + 'personalisation': {'address_line_1': '', 'address_line_2': '', 'postcode': ''} + } + + error_json = letter_request(client, data, service_id=sample_service_full_permissions.id, _expected_status=400) + + assert error_json['status_code'] == 400 + assert error_json['errors'] == [{'error': 'BadRequestError', 'message': 'Template not found'}] + + +def test_notification_returns_400_for_missing_template_field( + client, + sample_service_full_permissions +): + data = { + 'personalisation': {'address_line_1': '', 'address_line_2': '', 'postcode': ''} + } + + error_json = letter_request(client, data, service_id=sample_service_full_permissions.id, _expected_status=400) + + assert error_json['status_code'] == 400 + assert error_json['errors'] == [{ + 'error': 'ValidationError', + 'message': 'template_id is a required property' + }] + + +def test_notification_returns_400_if_address_doesnt_have_underscores( + client, + sample_letter_template +): + data = { + 'template_id': str(sample_letter_template.id), + 'personalisation': { + 'address line 1': 'Her Royal Highness Queen Elizabeth II', + 'address-line-2': 'Buckingham Palace', + 'postcode': 'SW1 1AA', + } + } + + error_json = letter_request(client, data, service_id=sample_letter_template.service_id, _expected_status=400) + + assert error_json['status_code'] == 400 + assert len(error_json['errors']) == 2 + assert { + 'error': 'ValidationError', + 'message': 'personalisation address_line_1 is a required property' + } in error_json['errors'] + assert { + 'error': 'ValidationError', + 'message': 'personalisation address_line_2 is a required property' + } in error_json['errors'] + + +def test_returns_a_429_limit_exceeded_if_rate_limit_exceeded( + client, + sample_letter_template, + mocker +): + persist_mock = mocker.patch('app.v2.notifications.post_notifications.persist_notification') + mocker.patch( + 'app.v2.notifications.post_notifications.check_rate_limiting', + side_effect=RateLimitError('LIMIT', 'INTERVAL', 'TYPE') + ) + + data = { + 'template_id': str(sample_letter_template.id), + 'personalisation': {'address_line_1': '', 'address_line_2': '', 'postcode': ''} + } + + error_json = letter_request(client, data, service_id=sample_letter_template.service_id, _expected_status=429) + + assert error_json['status_code'] == 429 + assert error_json['errors'] == [{ + 'error': 'RateLimitError', + 'message': 'Exceeded rate limit for key type TYPE of LIMIT requests per INTERVAL seconds' + }] + + assert not persist_mock.called + + +@pytest.mark.parametrize('service_args', [ + {'service_permissions': [EMAIL_TYPE, SMS_TYPE]}, + {'restricted': True} +]) +def test_post_letter_notification_returns_403_if_not_allowed_to_send_notification( + client, + notify_db_session, + service_args +): + service = create_service(**service_args) + template = create_template(service, template_type=LETTER_TYPE) + + data = { + 'template_id': str(template.id), + 'personalisation': {'address_line_1': '', 'address_line_2': '', 'postcode': ''} + } + + error_json = letter_request(client, data, service_id=service.id, _expected_status=400) + assert error_json['status_code'] == 400 + assert error_json['errors'] == [ + {'error': 'BadRequestError', 'message': 'Cannot send letters'} + ] + + +@pytest.mark.parametrize('research_mode, key_type', [ + (True, KEY_TYPE_NORMAL), + (False, KEY_TYPE_TEST) +]) +def test_post_letter_notification_doesnt_queue_task( + client, + notify_db_session, + mocker, + research_mode, + key_type +): + real_task = mocker.patch('app.celery.tasks.build_dvla_file.apply_async') + fake_task = mocker.patch('app.celery.tasks.update_job_to_sent_to_dvla.apply_async') + + service = create_service(research_mode=research_mode, service_permissions=[LETTER_TYPE]) + template = create_template(service, template_type=LETTER_TYPE) + + data = { + 'template_id': str(template.id), + 'personalisation': {'address_line_1': 'Foo', 'address_line_2': 'Bar', 'postcode': 'Baz'} + } + + letter_request(client, data, service_id=service.id, key_type=key_type) + + job = Job.query.one() + assert job.original_file_name == LETTER_TEST_API_FILENAME + assert not real_task.called + fake_task.assert_called_once_with([str(job.id)], queue='research-mode-tasks') + + +def test_post_letter_notification_doesnt_accept_team_key(client, sample_letter_template): + data = { + 'template_id': str(sample_letter_template.id), + 'personalisation': {'address_line_1': 'Foo', 'address_line_2': 'Bar', 'postcode': 'Baz'} + } + + error_json = letter_request( + client, + data, + sample_letter_template.service_id, + key_type=KEY_TYPE_TEAM, + _expected_status=403 + ) + + assert error_json['status_code'] == 403 + assert error_json['errors'] == [{'error': 'BadRequestError', 'message': 'Cannot send letters with a team api key'}] diff --git a/tests/app/v2/notifications/test_post_notifications.py b/tests/app/v2/notifications/test_post_notifications.py index 7433b9bcf..c108faacf 100644 --- a/tests/app/v2/notifications/test_post_notifications.py +++ b/tests/app/v2/notifications/test_post_notifications.py @@ -58,8 +58,8 @@ def test_post_sms_notification_returns_201(client, sample_template_with_placehol @pytest.mark.parametrize("notification_type, key_send_to, send_to", [("sms", "phone_number", "+447700900855"), ("email", "email_address", "sample@email.com")]) -def test_post_sms_notification_returns_400_and_missing_template(client, sample_service, - notification_type, key_send_to, send_to): +def test_post_notification_returns_400_and_missing_template(client, sample_service, + notification_type, key_send_to, send_to): data = { key_send_to: send_to, 'template_id': str(uuid.uuid4()) @@ -80,10 +80,12 @@ def test_post_sms_notification_returns_400_and_missing_template(client, sample_s "message": 'Template not found'}] -@pytest.mark.parametrize("notification_type, key_send_to, send_to", - [("sms", "phone_number", "+447700900855"), - ("email", "email_address", "sample@email.com")]) -def test_post_notification_returns_403_and_well_formed_auth_error(client, sample_template, +@pytest.mark.parametrize("notification_type, key_send_to, send_to", [ + ("sms", "phone_number", "+447700900855"), + ("email", "email_address", "sample@email.com"), + ("letter", "personalisation", {"address_line_1": "The queen", "postcode": "SW1 1AA"}) +]) +def test_post_notification_returns_401_and_well_formed_auth_error(client, sample_template, notification_type, key_send_to, send_to): data = { key_send_to: send_to, @@ -434,3 +436,15 @@ def test_post_notification_raises_bad_request_if_service_not_invited_to_schedule error_json = json.loads(response.get_data(as_text=True)) assert error_json['errors'] == [ {"error": "BadRequestError", "message": 'Cannot schedule notifications (this feature is invite-only)'}] + + +def test_post_notification_raises_bad_request_if_not_valid_notification_type(client, sample_service): + auth_header = create_authorization_header(service_id=sample_service.id) + response = client.post( + '/v2/notifications/foo', + data='{}', + headers=[('Content-Type', 'application/json'), auth_header] + ) + assert response.status_code == 404 + error_json = json.loads(response.get_data(as_text=True)) + assert 'The requested URL was not found on the server.' in error_json['message'] diff --git a/tests/app/v2/template/test_get_template.py b/tests/app/v2/template/test_get_template.py index 0f028a967..5de2b2814 100644 --- a/tests/app/v2/template/test_get_template.py +++ b/tests/app/v2/template/test_get_template.py @@ -3,16 +3,20 @@ import pytest from flask import json from app import DATETIME_FORMAT -from app.models import EMAIL_TYPE, TEMPLATE_TYPES +from app.models import TEMPLATE_TYPES, EMAIL_TYPE, SMS_TYPE, LETTER_TYPE from tests import create_authorization_header from tests.app.db import create_template valid_version_params = [None, 1] -@pytest.mark.parametrize("tmp_type", TEMPLATE_TYPES) +@pytest.mark.parametrize("tmp_type, expected_subject", [ + (SMS_TYPE, None), + (EMAIL_TYPE, 'Template subject'), + (LETTER_TYPE, 'Template subject') +]) @pytest.mark.parametrize("version", valid_version_params) -def test_get_template_by_id_returns_200(client, sample_service, tmp_type, version): +def test_get_template_by_id_returns_200(client, sample_service, tmp_type, expected_subject, version): template = create_template(sample_service, template_type=tmp_type) auth_header = create_authorization_header(service_id=sample_service.id) @@ -34,7 +38,7 @@ def test_get_template_by_id_returns_200(client, sample_service, tmp_type, versio 'version': template.version, 'created_by': template.created_by.email_address, 'body': template.content, - "subject": template.subject if tmp_type == EMAIL_TYPE else None + "subject": expected_subject } assert json_response == expected_response diff --git a/tests/app/v2/template/test_template_schemas.py b/tests/app/v2/template/test_template_schemas.py index 4517ad7e8..1e5a91e11 100644 --- a/tests/app/v2/template/test_template_schemas.py +++ b/tests/app/v2/template/test_template_schemas.py @@ -51,11 +51,18 @@ valid_json_post_args = { } invalid_json_post_args = [ - ({"id": "invalid_uuid", "personalisation": {"key": "value"}}, ["id is not a valid UUID"]), - ({"id": str(uuid.uuid4()), "personalisation": "invalid_personalisation"}, - ["personalisation should contain key value pairs"]), - ({"personalisation": "invalid_personalisation"}, - ["id is a required property", "personalisation should contain key value pairs"]) + ( + {"id": "invalid_uuid", "personalisation": {"key": "value"}}, + ["id is not a valid UUID"] + ), + ( + {"id": str(uuid.uuid4()), "personalisation": ['a', 'b']}, + ["personalisation [a, b] is not of type object"] + ), + ( + {"personalisation": "invalid_personalisation"}, + ["id is a required property", "personalisation invalid_personalisation is not of type object"] + ) ] valid_json_post_response = { diff --git a/wsgi.py b/wsgi.py index 2df2c3976..9fbeb28ac 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,13 +1,6 @@ -import os - from app import create_app -from credstash import getAllSecrets -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) - application = create_app() if __name__ == "__main__":