diff --git a/Makefile b/Makefile index 5954b4705..3612540bc 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,8 @@ freeze-requirements: rm -rf venv-freeze virtualenv -p python3 venv-freeze $$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt - echo '# This file is autogenerated. Do not edit it manually.' > requirements.txt + echo '# pyup: ignore file' > requirements.txt + echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt cat requirements-app.txt >> requirements.txt echo '' >> requirements.txt $$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt diff --git a/README.md b/README.md index 810718d97..dd0c7fab2 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,13 @@ To run the API you will need appropriate AWS credentials. You should receive the Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly. -### Virtualenv +### Virtualenv ``` mkvirtualenv -p /usr/local/bin/python3 notifications-api ``` -### `environment.sh` +### `environment.sh` Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account). diff --git a/app/billing/rest.py b/app/billing/rest.py index e18f08282..418bb39d5 100644 --- a/app/billing/rest.py +++ b/app/billing/rest.py @@ -1,4 +1,3 @@ -import json from datetime import datetime from flask import Blueprint, jsonify, request @@ -15,12 +14,8 @@ from app.dao.annual_billing_dao import ( dao_update_annual_billing_for_future_years ) from app.dao.date_util import get_current_financial_year_start_year -from app.dao.date_util import get_months_for_financial_year from app.dao.fact_billing_dao import fetch_monthly_billing_for_year, fetch_billing_totals_for_year -from app.dao.monthly_billing_dao import ( - get_billing_data_for_financial_year, - get_monthly_billing_by_notification_type -) + from app.errors import InvalidRequest from app.errors import register_errors from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE @@ -38,6 +33,7 @@ register_errors(billing_blueprint) @billing_blueprint.route('/ft-monthly-usage') +@billing_blueprint.route('/monthly-usage') def get_yearly_usage_by_monthly_from_ft_billing(service_id): try: year = int(request.args.get('year')) @@ -49,6 +45,7 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id): @billing_blueprint.route('/ft-yearly-usage-summary') +@billing_blueprint.route('/yearly-usage-summary') def get_yearly_billing_usage_summary_from_ft_billing(service_id): try: year = int(request.args.get('year')) @@ -60,41 +57,6 @@ def get_yearly_billing_usage_summary_from_ft_billing(service_id): return jsonify(data) -@billing_blueprint.route('/monthly-usage') -def get_yearly_usage_by_month(service_id): - try: - year = int(request.args.get('year')) - results = [] - for month in get_months_for_financial_year(year): - letter_billing_for_month = get_monthly_billing_by_notification_type(service_id, month, LETTER_TYPE) - if letter_billing_for_month: - results.extend(_transform_billing_for_month_letters(letter_billing_for_month)) - billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE) - if billing_for_month: - results.append(_transform_billing_for_month_sms(billing_for_month)) - return jsonify(results) - - except TypeError: - return jsonify(result='error', message='No valid year provided'), 400 - - -@billing_blueprint.route('/yearly-usage-summary') -def get_yearly_billing_usage_summary(service_id): - try: - year = int(request.args.get('year')) - billing_data = get_billing_data_for_financial_year(service_id, year) - notification_types = [EMAIL_TYPE, LETTER_TYPE, SMS_TYPE] - response = [ - _get_total_billable_units_and_rate_for_notification_type(billing_data, notification_type) - for notification_type in notification_types - ] - - return json.dumps(response) - - except TypeError: - return jsonify(result='error', message='No valid year provided'), 400 - - def _get_total_billable_units_and_rate_for_notification_type(billing_data, noti_type): total_sent = 0 rate = 0 diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 2b90eb623..eb2741be1 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -21,7 +21,7 @@ def create_nightly_billing(day_start=None): else: # When calling the task its a string in the format of "YYYY-MM-DD" day_start = datetime.strptime(day_start, "%Y-%m-%d") - for i in range(0, 3): + for i in range(0, 10): process_day = day_start - timedelta(days=i) transit_data = fetch_billing_data_for_day(process_day=process_day) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index dd15735a2..0b0b6ad85 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -15,11 +15,10 @@ from app import performance_platform_client, zendesk_client from app.aws import s3 from app.celery.service_callback_tasks import ( send_delivery_status_to_service, - create_encrypted_callback_data, + create_delivery_status_callback_data, ) from app.celery.tasks import process_job from app.config import QueueNames, TaskNames -from app.dao.date_util import get_month_start_and_end_date_in_utc from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago @@ -29,10 +28,6 @@ from app.dao.jobs_dao import ( dao_get_jobs_older_than_limited_by ) from app.dao.jobs_dao import dao_update_job -from app.dao.monthly_billing_dao import ( - get_service_ids_that_need_billing_populated, - create_or_update_monthly_billing -) from app.dao.notifications_dao import ( dao_timeout_notifications, is_delivery_slow_for_provider, @@ -67,9 +62,6 @@ from app.models import ( ) from app.notifications.process_notifications import send_notification_to_queue from app.performance_platform import total_sent_notifications, processing_time -from app.utils import ( - convert_utc_to_bst -) from app.v2.errors import JobIncompleteError @@ -206,7 +198,7 @@ def timeout_notifications(): # queue callback task only if the service_callback_api exists service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) if service_callback_api: - encrypted_notification = create_encrypted_callback_data(notification, service_callback_api) + encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api) send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification], queue=QueueNames.CALLBACKS) @@ -380,18 +372,6 @@ def raise_alert_if_letter_notifications_still_sending(): current_app.logger.info(message) -@notify_celery.task(name="populate_monthly_billing") -@statsd(namespace="tasks") -def populate_monthly_billing(): - # for every service with billable units this month update billing totals for yesterday - # this will overwrite the existing amount. - yesterday = datetime.utcnow() - timedelta(days=1) - yesterday_in_bst = convert_utc_to_bst(yesterday) - start_date, end_date = get_month_start_and_end_date_in_utc(yesterday_in_bst) - services = get_service_ids_that_need_billing_populated(start_date=start_date, end_date=end_date) - [create_or_update_monthly_billing(service_id=s.service_id, billing_month=end_date) for s in services] - - @notify_celery.task(name="run-letter-jobs") @statsd(namespace="tasks") def run_letter_jobs(): diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index ebf71d1f7..258ba9f9b 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -17,44 +17,78 @@ from app.config import QueueNames @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300) @statsd(namespace="tasks") -def send_delivery_status_to_service(self, notification_id, - encrypted_status_update - ): +def send_delivery_status_to_service( + self, notification_id, encrypted_status_update +): + status_update = encryption.decrypt(encrypted_status_update) + + data = { + "id": str(notification_id), + "reference": status_update['notification_client_reference'], + "to": status_update['notification_to'], + "status": status_update['notification_status'], + "created_at": status_update['notification_created_at'], + "completed_at": status_update['notification_updated_at'], + "sent_at": status_update['notification_sent_at'], + "notification_type": status_update['notification_type'] + } + _send_data_to_service_callback_api( + self, + data, + status_update['service_callback_api_url'], + status_update['service_callback_api_bearer_token'], + 'send_delivery_status_to_service' + ) + + +@notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300) +@statsd(namespace="tasks") +def send_complaint_to_service(self, complaint_data): + complaint = encryption.decrypt(complaint_data) + + data = { + "notification_id": complaint['notification_id'], + "complaint_id": complaint['complaint_id'], + "reference": complaint['reference'], + "to": complaint['to'], + "complaint_date": complaint['complaint_date'] + } + + _send_data_to_service_callback_api( + self, + data, + complaint['service_callback_api_url'], + complaint['service_callback_api_bearer_token'], + 'send_complaint_to_service' + ) + + +def _send_data_to_service_callback_api(self, data, service_callback_url, token, function_name): + notification_id = (data["notification_id"] if "notification_id" in data else data["id"]) try: - status_update = encryption.decrypt(encrypted_status_update) - - data = { - "id": str(notification_id), - "reference": status_update['notification_client_reference'], - "to": status_update['notification_to'], - "status": status_update['notification_status'], - "created_at": status_update['notification_created_at'], - "completed_at": status_update['notification_updated_at'], - "sent_at": status_update['notification_sent_at'], - "notification_type": status_update['notification_type'] - } - response = request( method="POST", - url=status_update['service_callback_api_url'], + url=service_callback_url, data=json.dumps(data), headers={ 'Content-Type': 'application/json', - 'Authorization': 'Bearer {}'.format(status_update['service_callback_api_bearer_token']) + 'Authorization': 'Bearer {}'.format(token) }, timeout=60 ) - current_app.logger.info('send_delivery_status_to_service sending {} to {}, response {}'.format( + current_app.logger.info('{} sending {} to {}, response {}'.format( + function_name, notification_id, - status_update['service_callback_api_url'], + service_callback_url, response.status_code )) response.raise_for_status() except RequestException as e: current_app.logger.warning( - "send_delivery_status_to_service request failed for notification_id: {} and url: {}. exc: {}".format( + "{} request failed for notification_id: {} and url: {}. exc: {}".format( + function_name, notification_id, - status_update['service_callback_api_url'], + service_callback_url, e ) ) @@ -63,12 +97,12 @@ def send_delivery_status_to_service(self, notification_id, self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: current_app.logger.exception( - """Retry: send_delivery_status_to_service has retried the max num of times - for notification: {}""".format(notification_id) + """Retry: {} has retried the max num of times + for notification: {}""".format(function_name, notification_id) ) -def create_encrypted_callback_data(notification, service_callback_api): +def create_delivery_status_callback_data(notification, service_callback_api): from app import DATETIME_FORMAT, encryption data = { "notification_id": str(notification.id), @@ -84,3 +118,17 @@ def create_encrypted_callback_data(notification, service_callback_api): "service_callback_api_bearer_token": service_callback_api.bearer_token, } return encryption.encrypt(data) + + +def create_complaint_callback_data(complaint, notification, service_callback_api, recipient): + from app import DATETIME_FORMAT, encryption + data = { + "complaint_id": str(complaint.id), + "notification_id": str(notification.id), + "reference": notification.client_reference, + "to": recipient, + "complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT), + "service_callback_api_url": service_callback_api.url, + "service_callback_api_bearer_token": service_callback_api.bearer_token, + } + return encryption.encrypt(data) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index dd6d88495..3c2089942 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -491,8 +491,10 @@ def check_billable_units(notification_update): if int(notification_update.page_count) != notification.billable_units: msg = 'Notification with id {} had {} billable_units but a page count of {}'.format( notification.id, notification.billable_units, notification_update.page_count) - - current_app.logger.error(msg) + try: + raise DVLAException(msg) + except DVLAException: + current_app.logger.exception(msg) @notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300) diff --git a/app/commands.py b/app/commands.py index 13fb5c4ed..cd61bdbc7 100644 --- a/app/commands.py +++ b/app/commands.py @@ -13,18 +13,17 @@ from sqlalchemy import func from notifications_utils.statsd_decorators import statsd from app import db, DATETIME_FORMAT, encryption, redis_store -from app.billing.rest import get_yearly_usage_by_month, get_yearly_usage_by_monthly_from_ft_billing from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform from app.celery.service_callback_tasks import send_delivery_status_to_service from app.celery.letters_pdf_tasks import create_letters_pdf from app.config import QueueNames -from app.dao.date_util import get_financial_year -from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing -from app.dao.monthly_billing_dao import ( - create_or_update_monthly_billing, - get_monthly_billing_by_notification_type, - get_service_ids_that_need_billing_populated +from app.dao.fact_billing_dao import ( + delete_billing_data_for_service_for_day, + fetch_billing_data_for_day, + get_service_ids_that_need_billing_populated, + update_fact_billing, ) + from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service from app.dao.services_dao import ( @@ -32,9 +31,9 @@ from app.dao.services_dao import ( dao_fetch_all_services_by_user, dao_fetch_service_by_id ) -from app.dao.users_dao import (delete_model_user, delete_user_verify_codes) -from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification -from app.performance_platform.processing_time import (send_processing_time_for_start_and_end) +from app.dao.users_dao import delete_model_user, delete_user_verify_codes +from app.models import PROVIDERS, User, Notification +from app.performance_platform.processing_time import send_processing_time_for_start_and_end from app.utils import ( cache_key_for_service_template_usage_per_day, get_london_midnight_in_utc, @@ -187,51 +186,6 @@ def fix_notification_statuses_not_in_sync(): result = db.session.execute(subq_hist).fetchall() -@notify_command() -@click.option('-y', '--year', required=True, help="e.g. 2017", type=int) -@click.option('-s', '--service_id', required=False, help="Enter the service id", type=click.UUID) -@click.option('-m', '--month', required=False, help="e.g. 1 for January", type=int) -def populate_monthly_billing(year, service_id=None, month=None): - """ - Populate monthly billing table for all services for a given year. - If service_id and month provided then only rebuild monthly billing for that month. - """ - def populate(service_id, year, month): - create_or_update_monthly_billing(service_id, datetime(year, month, 1)) - sms_res = get_monthly_billing_by_notification_type( - service_id, datetime(year, month, 1), SMS_TYPE - ) - email_res = get_monthly_billing_by_notification_type( - service_id, datetime(year, month, 1), EMAIL_TYPE - ) - letter_res = get_monthly_billing_by_notification_type( - service_id, datetime(year, month, 1), 'letter' - ) - - print("Finished populating data for {}-{} for service id {}".format(month, year, str(service_id))) - print('SMS: {}'.format(sms_res.monthly_totals)) - print('Email: {}'.format(email_res.monthly_totals)) - print('Letter: {}'.format(letter_res.monthly_totals)) - - if service_id and month: - populate(service_id, year, month) - else: - service_ids = get_service_ids_that_need_billing_populated( - start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16) - ) - start, end = 1, 13 - - if year == 2016: - start = 4 - - for service_id in service_ids: - print('Starting to populate data for service {}'.format(str(service_id))) - print('Starting populating monthly billing for {}'.format(year)) - for i in range(start, end): - print('Population for {}-{}'.format(i, year)) - populate(service_id, year, i) - - @notify_command() @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) @@ -526,55 +480,34 @@ def rebuild_ft_billing_for_day(service_id, day): Rebuild the data in ft_billing for the given service_id and date """ def rebuild_ft_data(process_day, service): + deleted_rows = delete_billing_data_for_service_for_day(process_day, service) + current_app.logger.info('deleted {} existing billing rows for {} on {}'.format( + deleted_rows, + service, + process_day + )) transit_data = fetch_billing_data_for_day(process_day=process_day, service_id=service) + # transit_data = every row that should exist for data in transit_data: + # upsert existing rows update_fact_billing(data, process_day) + current_app.logger.info('added/updated {} billing rows for {} on {}'.format( + len(transit_data), + service, + process_day + )) + if service_id: # confirm the service exists dao_fetch_service_by_id(service_id) rebuild_ft_data(day, service_id) else: - services = get_service_ids_that_need_billing_populated(day, day) - for service_id in services: - rebuild_ft_data(day, service_id) - - -@notify_command(name='compare-ft-billing-to-monthly-billing') -@click.option('-y', '--year', required=True) -@click.option('-s', '--service_id', required=False, type=click.UUID) -def compare_ft_billing_to_monthly_billing(year, service_id=None): - """ - This command checks the results of monthly_billing to ft_billing for the given year. - If service id is not included all services are compared for the given year. - """ - def compare_monthly_billing_to_ft_billing(ft_billing_resp, monthly_billing_resp): - # Remove the rows with 0 billing_units and rate, ft_billing doesn't populate those rows. - mo_json = json.loads(monthly_billing_resp.get_data(as_text=True)) - rm_zero_rows = [x for x in mo_json if x['billing_units'] != 0 and x['rate'] != 0] - try: - assert rm_zero_rows == json.loads(ft_billing_resp.get_data(as_text=True)) - except AssertionError: - print("Comparison failed for service: {} and year: {}".format(service_id, year)) - - if not service_id: - start_date, end_date = get_financial_year(year=int(year)) - services = get_service_ids_that_need_billing_populated(start_date, end_date) - for service_id in services: - with current_app.test_request_context( - path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)): - monthly_billing_response = get_yearly_usage_by_month(service_id) - with current_app.test_request_context( - path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)): - ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id) - compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response) - else: - with current_app.test_request_context( - path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)): - monthly_billing_response = get_yearly_usage_by_month(service_id) - with current_app.test_request_context( - path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)): - ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id) - compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response) + services = get_service_ids_that_need_billing_populated( + get_london_midnight_in_utc(day), + get_london_midnight_in_utc(day + timedelta(days=1)) + ) + for row in services: + rebuild_ft_data(day, row.service_id) @notify_command(name='migrate-data-to-ft-notification-status') diff --git a/app/config.py b/app/config.py index 0a490588f..f4486121e 100644 --- a/app/config.py +++ b/app/config.py @@ -238,11 +238,6 @@ class Config(object): 'schedule': crontab(hour=4, minute=40), 'options': {'queue': QueueNames.PERIODIC} }, - 'populate_monthly_billing': { - 'task': 'populate_monthly_billing', - 'schedule': crontab(hour=5, minute=10), - 'options': {'queue': QueueNames.PERIODIC} - }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=16, minute=30), diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index db5758d69..f07e6c85d 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -125,6 +125,18 @@ def fetch_monthly_billing_for_year(service_id, year): return yearly_data +def delete_billing_data_for_service_for_day(process_day, service_id): + """ + Delete all ft_billing data for a given service on a given bst_date + + Returns how many rows were deleted + """ + return FactBilling.query.filter( + FactBilling.bst_date == process_day, + FactBilling.service_id == service_id + ).delete() + + def fetch_billing_data_for_day(process_day, service_id=None): start_date = convert_bst_to_utc(datetime.combine(process_day, time.min)) end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) @@ -188,6 +200,17 @@ def get_rates_for_billing(): return non_letter_rates, letter_rates +def get_service_ids_that_need_billing_populated(start_date, end_date): + return db.session.query( + NotificationHistory.service_id + ).filter( + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]), + NotificationHistory.billable_units != 0 + ).distinct().all() + + def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, letter_page_count=None): if notification_type == LETTER_TYPE: return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and letter_page_count == r[2]) diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py deleted file mode 100644 index 502df8fa5..000000000 --- a/app/dao/monthly_billing_dao.py +++ /dev/null @@ -1,128 +0,0 @@ -from datetime import datetime - -from notifications_utils.statsd_decorators import statsd - -from app import db -from app.dao.dao_utils import transactional -from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year -from app.dao.notification_usage_dao import get_billing_data_for_month -from app.models import ( - SMS_TYPE, - EMAIL_TYPE, - LETTER_TYPE, - MonthlyBilling, - NotificationHistory -) -from app.utils import convert_utc_to_bst - - -def get_service_ids_that_need_billing_populated(start_date, end_date): - return db.session.query( - NotificationHistory.service_id - ).filter( - NotificationHistory.created_at >= start_date, - NotificationHistory.created_at <= end_date, - NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]), - NotificationHistory.billable_units != 0 - ).distinct().all() - - -@statsd(namespace="dao") -def create_or_update_monthly_billing(service_id, billing_month): - start_date, end_date = get_month_start_and_end_date_in_utc(billing_month) - _update_monthly_billing(service_id, start_date, end_date, SMS_TYPE) - _update_monthly_billing(service_id, start_date, end_date, EMAIL_TYPE) - _update_monthly_billing(service_id, start_date, end_date, LETTER_TYPE) - - -def _monthly_billing_data_to_json(billing_data): - results = [] - if billing_data: - # total cost must take into account the free allowance. - # might be a good idea to capture free allowance in this table - results = [{ - "billing_units": x.billing_units, - "rate_multiplier": x.rate_multiplier, - "international": x.international, - "rate": x.rate, - "total_cost": (x.billing_units * x.rate_multiplier) * x.rate - } for x in billing_data] - return results - - -@statsd(namespace="dao") -@transactional -def _update_monthly_billing(service_id, start_date, end_date, notification_type): - billing_data = get_billing_data_for_month( - service_id=service_id, - start_date=start_date, - end_date=end_date, - notification_type=notification_type - ) - monthly_totals = _monthly_billing_data_to_json(billing_data) - row = get_monthly_billing_entry(service_id, start_date, notification_type) - if row: - row.monthly_totals = monthly_totals - row.updated_at = datetime.utcnow() - else: - row = MonthlyBilling( - service_id=service_id, - notification_type=notification_type, - monthly_totals=monthly_totals, - start_date=start_date, - end_date=end_date - ) - - db.session.add(row) - - -def get_monthly_billing_entry(service_id, start_date, notification_type): - entry = MonthlyBilling.query.filter_by( - service_id=service_id, - start_date=start_date, - notification_type=notification_type - ).first() - - return entry - - -@statsd(namespace="dao") -def get_yearly_billing_data_for_date_range( - service_id, start_date, end_date, notification_types -): - results = db.session.query( - MonthlyBilling.notification_type, - MonthlyBilling.monthly_totals, - MonthlyBilling.start_date, - ).filter( - MonthlyBilling.service_id == service_id, - MonthlyBilling.start_date >= start_date, - MonthlyBilling.end_date <= end_date, - MonthlyBilling.notification_type.in_(notification_types) - ).order_by( - MonthlyBilling.start_date, - MonthlyBilling.notification_type, - ).all() - - return results - - -@statsd(namespace="dao") -def get_monthly_billing_by_notification_type(service_id, billing_month, notification_type): - billing_month_in_bst = convert_utc_to_bst(billing_month) - start_date, _ = get_month_start_and_end_date_in_utc(billing_month_in_bst) - return get_monthly_billing_entry(service_id, start_date, notification_type) - - -@statsd(namespace="dao") -def get_billing_data_for_financial_year(service_id, year, notification_types=[SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]): - now = convert_utc_to_bst(datetime.utcnow()) - start_date, end_date = get_financial_year(year) - if start_date <= now <= end_date: - # Update totals to the latest so we include data for today - create_or_update_monthly_billing(service_id=service_id, billing_month=now) - - results = get_yearly_billing_data_for_date_range( - service_id, start_date, end_date, notification_types - ) - return results diff --git a/app/dao/service_callback_api_dao.py b/app/dao/service_callback_api_dao.py index 26a0478a3..ea23d6306 100644 --- a/app/dao/service_callback_api_dao.py +++ b/app/dao/service_callback_api_dao.py @@ -4,7 +4,7 @@ from app import db, create_uuid from app.dao.dao_utils import transactional, version_class from app.models import ServiceCallbackApi -from app.models import DELIVERY_STATUS_CALLBACK_TYPE +from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE @transactional @@ -39,6 +39,13 @@ def get_service_delivery_status_callback_api_for_service(service_id): ).first() +def get_service_complaint_callback_api_for_service(service_id): + return ServiceCallbackApi.query.filter_by( + service_id=service_id, + callback_type=COMPLAINT_CALLBACK_TYPE + ).first() + + @transactional def delete_service_callback_api(service_callback_api): db.session.delete(service_callback_api) diff --git a/app/models.py b/app/models.py index 216c3bc16..f5bf62087 100644 --- a/app/models.py +++ b/app/models.py @@ -597,7 +597,7 @@ class ServiceInboundApi(db.Model, Versioned): class ServiceCallbackApi(db.Model, Versioned): __tablename__ = 'service_callback_api' id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False, unique=True) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) service = db.relationship('Service', backref='service_callback_api') url = db.Column(db.String(), nullable=False) callback_type = db.Column(db.String(), db.ForeignKey('service_callback_type.name'), nullable=True) @@ -607,6 +607,10 @@ class ServiceCallbackApi(db.Model, Versioned): updated_by = db.relationship('User') updated_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False) + __table_args__ = ( + UniqueConstraint('service_id', 'callback_type', name='uix_service_callback_type'), + ) + @property def bearer_token(self): if self._bearer_token: @@ -1639,35 +1643,6 @@ class LetterRate(db.Model): post_class = db.Column(db.String, nullable=False) -class MonthlyBilling(db.Model): - __tablename__ = 'monthly_billing' - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) - service = db.relationship('Service', backref='monthly_billing') - start_date = db.Column(db.DateTime, nullable=False) - end_date = db.Column(db.DateTime, nullable=False) - notification_type = db.Column(notification_types, nullable=False) - monthly_totals = db.Column(JSON, nullable=False) - updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - - __table_args__ = ( - UniqueConstraint('service_id', 'start_date', 'notification_type', name='uix_monthly_billing'), - ) - - def serialized(self): - return { - "start_date": self.start_date, - "end_date": self.end_date, - "service_id": str(self.service_id), - "notification_type": self.notification_type, - "monthly_totals": self.monthly_totals - } - - def __repr__(self): - return str(self.serialized()) - - class ServiceEmailReplyTo(db.Model): __tablename__ = "service_email_reply_to" diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index acd0d605e..e90cfeced 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -12,12 +12,16 @@ from app.dao import ( ) from app.dao.complaint_dao import save_complaint from app.dao.notifications_dao import dao_get_notification_history_by_reference -from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service +from app.dao.service_callback_api_dao import ( + get_service_delivery_status_callback_api_for_service, get_service_complaint_callback_api_for_service +) from app.models import Complaint from app.notifications.process_client_response import validate_callback_data from app.celery.service_callback_tasks import ( send_delivery_status_to_service, - create_encrypted_callback_data, + send_complaint_to_service, + create_delivery_status_callback_data, + create_complaint_callback_data ) from app.config import QueueNames @@ -38,7 +42,8 @@ def process_ses_response(ses_request): if notification_type == 'Bounce': notification_type = determine_notification_bounce_type(notification_type, ses_message) elif notification_type == 'Complaint': - handle_complaint(ses_message) + complaint, notification, recipient = handle_complaint(ses_message) + _check_and_queue_complaint_callback_task(complaint, notification, recipient) return try: @@ -105,7 +110,7 @@ def determine_notification_bounce_type(notification_type, ses_message): def handle_complaint(ses_message): - remove_emails_from_complaint(ses_message) + recipient_email = remove_emails_from_complaint(ses_message)[0] current_app.logger.info("Complaint from SES: \n{}".format(ses_message)) try: reference = ses_message['mail']['messageId'] @@ -123,6 +128,7 @@ def handle_complaint(ses_message): complaint_date=ses_complaint.get('timestamp', None) if ses_complaint else None ) save_complaint(complaint) + return complaint, notification, recipient_email def remove_mail_headers(dict_to_edit): @@ -141,13 +147,21 @@ def remove_emails_from_bounce(bounce_dict): def remove_emails_from_complaint(complaint_dict): remove_mail_headers(complaint_dict) complaint_dict['complaint'].pop('complainedRecipients') - complaint_dict['mail'].pop('destination') + return complaint_dict['mail'].pop('destination') def _check_and_queue_callback_task(notification): # queue callback task only if the service_callback_api exists service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) if service_callback_api: - encrypted_notification = create_encrypted_callback_data(notification, service_callback_api) - send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification], + notification_data = create_delivery_status_callback_data(notification, service_callback_api) + send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) + + +def _check_and_queue_complaint_callback_task(complaint, notification, recipient): + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) + send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) diff --git a/app/notifications/process_client_response.py b/app/notifications/process_client_response.py index 2cfe861a6..ce2f023e5 100644 --- a/app/notifications/process_client_response.py +++ b/app/notifications/process_client_response.py @@ -10,7 +10,7 @@ from app.clients.sms.firetext import get_firetext_responses from app.clients.sms.mmg import get_mmg_responses from app.celery.service_callback_tasks import ( send_delivery_status_to_service, - create_encrypted_callback_data, + create_delivery_status_callback_data, ) from app.config import QueueNames from app.dao.notifications_dao import dao_update_notification @@ -98,7 +98,7 @@ def _process_for_status(notification_status, client_name, provider_reference): service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) if service_callback_api: - encrypted_notification = create_encrypted_callback_data(notification, service_callback_api) + encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api) send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification], queue=QueueNames.CALLBACKS) diff --git a/app/schemas.py b/app/schemas.py index db21fef8e..d9bb60e9e 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -232,7 +232,6 @@ class ServiceSchema(BaseSchema): 'service_provider_stats', 'service_notification_stats', 'service_sms_senders', - 'monthly_billing', 'reply_to_email_addresses', 'letter_contacts', 'complaints', diff --git a/migrations/versions/0208_fix_unique_index.py b/migrations/versions/0208_fix_unique_index.py new file mode 100644 index 000000000..52ac87c23 --- /dev/null +++ b/migrations/versions/0208_fix_unique_index.py @@ -0,0 +1,23 @@ +""" + +Revision ID: 0208_fix_unique_index +Revises: 0207_set_callback_history_type +Create Date: 2018-07-25 13:55:24.941794 + +""" +from alembic import op + +revision = '84c3b6eb16b3' +down_revision = '0207_set_callback_history_type' + + +def upgrade(): + op.create_unique_constraint('uix_service_callback_type', 'service_callback_api', ['service_id', 'callback_type']) + op.drop_index('ix_service_callback_api_service_id', table_name='service_callback_api') + op.create_index(op.f('ix_service_callback_api_service_id'), 'service_callback_api', ['service_id'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_service_callback_api_service_id'), table_name='service_callback_api') + op.create_index('ix_service_callback_api_service_id', 'service_callback_api', ['service_id'], unique=True) + op.drop_constraint('uix_service_callback_type', 'service_callback_api', type_='unique') diff --git a/requirements-app.txt b/requirements-app.txt index 369024bac..abc9d8777 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -25,6 +25,6 @@ notifications-python-client==4.8.2 # PaaS awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@29.3.2#egg=notifications-utils==29.3.2 +git+https://github.com/alphagov/notifications-utils.git@29.3.3#egg=notifications-utils==29.3.3 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/requirements.txt b/requirements.txt index c696af14d..2dcc668b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +# pyup: ignore file # This file is autogenerated. Do not edit it manually. # Run `make freeze-requirements` to update requirements.txt # with package version changes made in requirements-app.txt @@ -26,7 +27,7 @@ notifications-python-client==4.8.2 # PaaS awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@29.3.2#egg=notifications-utils==29.3.2 +git+https://github.com/alphagov/notifications-utils.git@29.3.3#egg=notifications-utils==29.3.3 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 @@ -34,12 +35,12 @@ git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 alembic==1.0.0 amqp==1.4.9 anyjson==0.3.3 -awscli==1.15.59 +awscli==1.15.64 bcrypt==3.1.4 billiard==3.3.0.23 bleach==2.1.3 boto3==1.6.16 -botocore==1.10.58 +botocore==1.10.63 certifi==2018.4.16 chardet==3.0.4 click==6.7 @@ -47,7 +48,7 @@ colorama==0.3.9 docutils==0.14 Flask-Redis==0.3.0 future==0.16.0 -greenlet==0.4.13 +greenlet==0.4.14 html5lib==1.0.1 idna==2.7 itsdangerous==0.24 diff --git a/tests/app/billing/test_billing.py b/tests/app/billing/test_billing.py index 02de27604..3ea91bdda 100644 --- a/tests/app/billing/test_billing.py +++ b/tests/app/billing/test_billing.py @@ -4,18 +4,12 @@ import json import pytest -from app.billing.rest import _transform_billing_for_month_sms -from app.dao.monthly_billing_dao import ( - create_or_update_monthly_billing, - get_monthly_billing_by_notification_type, -) -from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, FactBilling +from app.models import FactBilling from app.dao.date_util import get_current_financial_year_start_year, get_month_start_and_end_date_in_utc from app.dao.annual_billing_dao import dao_get_free_sms_fragment_limit_for_year from tests.app.db import ( create_notification, create_rate, - create_monthly_billing_entry, create_annual_billing, create_template, create_service, @@ -36,253 +30,6 @@ def _assert_dict_equals(actual, expected_dict): assert actual == expected_dict -def test_get_yearly_billing_summary_returns_correct_breakdown(client, sample_template): - create_rate(start_date=IN_MAY_2016 - timedelta(days=1), value=0.12, notification_type=SMS_TYPE) - create_notification( - template=sample_template, created_at=IN_MAY_2016, - billable_units=1, rate_multiplier=2, status='delivered' - ) - create_notification( - template=sample_template, created_at=IN_JUN_2016, - billable_units=2, rate_multiplier=3, status='delivered' - ) - - letter_template = create_template(service=sample_template.service, template_type=LETTER_TYPE) - create_notification(template=letter_template, created_at=IN_MAY_2016, status='delivered', billable_units=1) - create_notification(template=letter_template, created_at=IN_JUN_2016, status='delivered', billable_units=1) - - create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_MAY_2016) - create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_JUN_2016) - - response = client.get( - '/service/{}/billing/yearly-usage-summary?year=2016'.format(sample_template.service.id), - headers=[create_authorization_header()] - ) - assert response.status_code == 200 - - resp_json = json.loads(response.get_data(as_text=True)) - assert len(resp_json) == 3 - - _assert_dict_equals(resp_json[0], { - 'notification_type': EMAIL_TYPE, - 'billing_units': 0, - 'rate': 0, - 'letter_total': 0 - }) - _assert_dict_equals(resp_json[1], { - 'notification_type': LETTER_TYPE, - 'billing_units': 2, - 'rate': 0, - 'letter_total': 0.66 - }) - _assert_dict_equals(resp_json[2], { - 'notification_type': SMS_TYPE, - 'billing_units': 8, - 'rate': 0.12, - 'letter_total': 0 - }) - - -def test_get_yearly_billing_usage_breakdown_returns_400_if_missing_year(client, sample_service): - response = client.get( - '/service/{}/billing/yearly-usage-summary'.format(sample_service.id), - headers=[create_authorization_header()] - ) - assert response.status_code == 400 - assert json.loads(response.get_data(as_text=True)) == { - 'message': 'No valid year provided', 'result': 'error' - } - - -def test_get_yearly_usage_by_month_returns_400_if_missing_year(client, sample_service): - response = client.get( - '/service/{}/billing/monthly-usage'.format(sample_service.id), - headers=[create_authorization_header()] - ) - assert response.status_code == 400 - assert json.loads(response.get_data(as_text=True)) == { - 'message': 'No valid year provided', 'result': 'error' - } - - -def test_get_yearly_usage_by_month_returns_empty_list_if_no_usage(client, sample_template): - create_rate(start_date=IN_MAY_2016 - timedelta(days=1), value=0.12, notification_type=SMS_TYPE) - response = client.get( - '/service/{}/billing/monthly-usage?year=2016'.format(sample_template.service.id), - headers=[create_authorization_header()] - ) - assert response.status_code == 200 - - results = json.loads(response.get_data(as_text=True)) - assert results == [] - - -def test_get_yearly_usage_by_month_returns_correctly(client, sample_template): - create_rate(start_date=IN_MAY_2016 - timedelta(days=1), value=0.12, notification_type=SMS_TYPE) - create_notification( - template=sample_template, created_at=IN_MAY_2016, - billable_units=1, rate_multiplier=2, status='delivered' - ) - create_notification( - template=sample_template, created_at=IN_JUN_2016, - billable_units=2, rate_multiplier=3, status='delivered' - ) - - create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_MAY_2016) - create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_JUN_2016) - - response = client.get( - '/service/{}/billing/monthly-usage?year=2016'.format(sample_template.service.id), - headers=[create_authorization_header()] - ) - - assert response.status_code == 200 - - resp_json = json.loads(response.get_data(as_text=True)) - _assert_dict_equals(resp_json[0], { - 'billing_units': 0, - 'month': 'May', - 'notification_type': LETTER_TYPE, - 'rate': 0 - }) - _assert_dict_equals(resp_json[1], { - 'billing_units': 2, - 'month': 'May', - 'notification_type': SMS_TYPE, - 'rate': 0.12 - }) - _assert_dict_equals(resp_json[2], { - 'billing_units': 0, - 'month': 'June', - 'notification_type': LETTER_TYPE, - 'rate': 0 - }) - _assert_dict_equals(resp_json[3], { - 'billing_units': 6, - 'month': 'June', - 'notification_type': SMS_TYPE, - 'rate': 0.12 - }) - - -def test_transform_billing_for_month_returns_empty_if_no_monthly_totals(sample_service): - create_monthly_billing_entry( - service=sample_service, - monthly_totals=[], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type( - sample_service.id, APR_2016_MONTH_START, SMS_TYPE - )) - - _assert_dict_equals(transformed_billing_data, { - 'notification_type': SMS_TYPE, - 'billing_units': 0, - 'month': 'April', - 'rate': 0, - }) - - -def test_transform_billing_for_month_formats_monthly_totals_correctly(sample_service): - create_monthly_billing_entry( - service=sample_service, - monthly_totals=[{ - "billing_units": 12, - "rate": 0.0158, - "rate_multiplier": 5, - "total_cost": 2.1804, - "international": False - }], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type( - sample_service.id, APR_2016_MONTH_START, SMS_TYPE - )) - - _assert_dict_equals(transformed_billing_data, { - 'notification_type': SMS_TYPE, - 'billing_units': 60, - 'month': 'April', - 'rate': 0.0158, - }) - - -def test_transform_billing_sums_billable_units(sample_service): - create_monthly_billing_entry( - service=sample_service, - monthly_totals=[{ - 'billing_units': 1321, - 'international': False, - 'month': 'May', - 'notification_type': SMS_TYPE, - 'rate': 0.12, - 'rate_multiplier': 1 - }, { - 'billing_units': 1, - 'international': False, - 'month': 'May', - 'notification_type': SMS_TYPE, - 'rate': 0.12, - 'rate_multiplier': 1 - }], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type( - sample_service.id, APR_2016_MONTH_START, SMS_TYPE - )) - - _assert_dict_equals(transformed_billing_data, { - 'notification_type': SMS_TYPE, - 'billing_units': 1322, - 'month': 'April', - 'rate': 0.12, - }) - - -def test_transform_billing_calculates_with_different_rate_multipliers(sample_service): - create_monthly_billing_entry( - service=sample_service, - monthly_totals=[{ - 'billing_units': 1321, - 'international': False, - 'month': 'May', - 'notification_type': SMS_TYPE, - 'rate': 0.12, - 'rate_multiplier': 1 - }, { - 'billing_units': 1, - 'international': False, - 'month': 'May', - 'notification_type': SMS_TYPE, - 'rate': 0.12, - 'rate_multiplier': 3 - }], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type( - sample_service.id, APR_2016_MONTH_START, SMS_TYPE - )) - - _assert_dict_equals(transformed_billing_data, { - 'notification_type': SMS_TYPE, - 'billing_units': 1324, - 'month': 'April', - 'rate': 0.12, - }) - - def test_create_update_free_sms_fragment_limit_invalid_schema(client, sample_service): response = client.post('service/{}/billing/free-sms-fragment-limit'.format(sample_service.id), @@ -480,21 +227,6 @@ def test_get_yearly_usage_by_monthly_from_ft_billing(client, notify_db_session): assert len(ft_email) == 0 -def test_compare_ft_billing_to_monthly_billing(client, notify_db_session): - service = set_up_yearly_data() - - monthly_billing_response = client.get('/service/{}/billing/monthly-usage?year=2016'.format(service.id), - headers=[create_authorization_header()]) - - ft_billing_response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id), - headers=[('Content-Type', 'application/json'), create_authorization_header()]) - - monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True)) - ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True)) - - assert monthly_billing_json_resp == ft_billing_json_resp - - def set_up_yearly_data(): service = create_service() sms_template = create_template(service=service, template_type="sms") @@ -527,55 +259,9 @@ def set_up_yearly_data(): notification_type='letter', rate=0.33) start_date, end_date = get_month_start_and_end_date_in_utc(datetime(2016, int(mon), 1)) - create_monthly_billing_entry(service=service, start_date=start_date, - end_date=end_date, - notification_type='sms', - monthly_totals=[ - {"rate": 0.0162, "international": False, - "rate_multiplier": 1, "billing_units": int(d), - "total_cost": 0.0162 * int(d)}, - {"rate": 0.0162, "international": False, - "rate_multiplier": 2, "billing_units": int(d), - "total_cost": 0.0162 * int(d)}] - ) - create_monthly_billing_entry(service=service, start_date=start_date, - end_date=end_date, - notification_type='email', - monthly_totals=[ - {"rate": 0, "international": False, - "rate_multiplier": 1, "billing_units": int(d), - "total_cost": 0}] - ) - create_monthly_billing_entry(service=service, start_date=start_date, - end_date=end_date, - notification_type='letter', - monthly_totals=[ - {"rate": 0.33, "international": False, - "rate_multiplier": 1, "billing_units": int(d), - "total_cost": 0.33 * int(d)}] - ) return service -def test_get_yearly_billing_usage_summary_from_ft_billing_compare_to_monthly_billing( - client, notify_db_session -): - service = set_up_yearly_data() - monthly_billing_response = client.get('/service/{}/billing/yearly-usage-summary?year=2016'.format(service.id), - headers=[create_authorization_header()]) - - ft_billing_response = client.get('service/{}/billing/ft-yearly-usage-summary?year=2016'.format(service.id), - headers=[('Content-Type', 'application/json'), create_authorization_header()]) - - monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True)) - ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True)) - - assert len(monthly_billing_json_resp) == 3 - assert len(ft_billing_json_resp) == 3 - for i in range(0, 3): - assert sorted(monthly_billing_json_resp[i]) == sorted(ft_billing_json_resp[i]) - - def test_get_yearly_billing_usage_summary_from_ft_billing_returns_400_if_missing_year(client, sample_service): response = client.get( '/service/{}/billing/ft-yearly-usage-summary'.format(sample_service.id), diff --git a/tests/app/celery/test_ftp_update_tasks.py b/tests/app/celery/test_ftp_update_tasks.py index de118f5dd..0da3eb541 100644 --- a/tests/app/celery/test_ftp_update_tasks.py +++ b/tests/app/celery/test_ftp_update_tasks.py @@ -308,7 +308,7 @@ def test_check_billable_units_when_billable_units_does_not_match_page_count( mocker, notification_update ): - mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error') + mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') notification = create_notification(sample_letter_template, reference='REFERENCE_ABC', billable_units=3) diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index ecd4002f1..a4a21afd7 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -287,7 +287,7 @@ def test_create_nightly_billing_consolidate_from_3_days_delta( mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) # create records from 11th to 15th - for i in range(0, 5): + for i in range(0, 11): sample_notification( notify_db, notify_db_session, @@ -302,7 +302,7 @@ def test_create_nightly_billing_consolidate_from_3_days_delta( ) notification = Notification.query.order_by(Notification.created_at).all() - assert datetime.date(notification[0].created_at) == date(2018, 1, 11) + assert datetime.date(notification[0].created_at) == date(2018, 1, 5) records = FactBilling.query.all() assert len(records) == 0 @@ -310,8 +310,8 @@ def test_create_nightly_billing_consolidate_from_3_days_delta( create_nightly_billing() records = FactBilling.query.order_by(FactBilling.bst_date).all() - assert len(records) == 3 - assert records[0].bst_date == date(2018, 1, 12) + assert len(records) == 10 + assert records[0].bst_date == date(2018, 1, 5) assert records[-1].bst_date == date(2018, 1, 14) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 573aac282..35b4bf2e7 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -27,7 +27,6 @@ from app.celery.scheduled_tasks import ( run_scheduled_jobs, run_letter_jobs, trigger_letter_pdfs_for_day, - populate_monthly_billing, s3, send_daily_performance_platform_stats, send_scheduled_notifications, @@ -48,7 +47,6 @@ from app.dao.provider_details_dao import ( ) from app.exceptions import NotificationTechnicalFailureException from app.models import ( - MonthlyBilling, NotificationHistory, Service, StatsTemplateUsageByMonth, @@ -60,11 +58,10 @@ from app.models import ( SMS_TYPE ) from app.utils import get_london_midnight_in_utc -from app.celery.service_callback_tasks import create_encrypted_callback_data +from app.celery.service_callback_tasks import create_delivery_status_callback_data from app.v2.errors import JobIncompleteError from tests.app.db import ( - create_notification, create_service, create_template, create_job, create_rate, - create_service_callback_api + create_notification, create_service, create_template, create_job, create_service_callback_api ) from tests.app.conftest import ( @@ -125,8 +122,6 @@ def test_should_have_decorated_tasks_functions(): 'remove_transformed_dvla_files' assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \ 'delete_dvla_response_files_older_than_seven_days' - assert populate_monthly_billing.__wrapped__.__name__ == \ - 'populate_monthly_billing' @pytest.fixture(scope='function') @@ -224,7 +219,7 @@ def test_timeout_notifications_sends_status_update_to_service(client, sample_tem seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10)) timeout_notifications() - encrypted_data = create_encrypted_callback_data(notification, callback_api) + encrypted_data = create_delivery_status_callback_data(notification, callback_api) mocked.assert_called_once_with([str(notification.id), encrypted_data], queue=QueueNames.CALLBACKS) @@ -751,85 +746,6 @@ def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_lett ) -@freeze_time("2017-07-12 02:00:00") -def test_populate_monthly_billing_populates_correctly(sample_template): - yesterday = datetime(2017, 7, 11, 13, 30) - jul_month_start = datetime(2017, 6, 30, 23) - jul_month_end = datetime(2017, 7, 31, 22, 59, 59, 99999) - create_rate(datetime(2016, 1, 1), 0.0123, 'sms') - - create_notification(template=sample_template, status='delivered', created_at=yesterday) - create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1)) - create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1)) - # not included in billing - create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30)) - - populate_monthly_billing() - - monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all() - - assert len(monthly_billing) == 3 - - assert monthly_billing[0].service_id == sample_template.service_id - assert monthly_billing[0].start_date == jul_month_start - assert monthly_billing[0].end_date == jul_month_end - assert monthly_billing[0].notification_type == 'email' - assert monthly_billing[0].monthly_totals == [] - - assert monthly_billing[1].service_id == sample_template.service_id - assert monthly_billing[1].start_date == jul_month_start - assert monthly_billing[1].end_date == jul_month_end - assert monthly_billing[1].notification_type == 'sms' - assert sorted(monthly_billing[1].monthly_totals[0]) == sorted( - { - 'international': False, - 'rate_multiplier': 1, - 'billing_units': 3, - 'rate': 0.0123, - 'total_cost': 0.0369 - } - ) - - assert monthly_billing[2].service_id == sample_template.service_id - assert monthly_billing[2].start_date == jul_month_start - assert monthly_billing[2].end_date == jul_month_end - assert monthly_billing[2].notification_type == 'letter' - assert monthly_billing[2].monthly_totals == [] - - -@freeze_time("2016-04-01 23:00:00") -def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template): - yesterday = datetime.utcnow() - timedelta(days=1) - apr_month_start = datetime(2016, 3, 31, 23) - apr_month_end = datetime(2016, 4, 30, 22, 59, 59, 99999) - create_rate(datetime(2016, 1, 1), 0.0123, 'sms') - create_notification(template=sample_template, status='delivered', created_at=yesterday) - populate_monthly_billing() - - monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all() - - assert len(monthly_billing) == 3 - - assert monthly_billing[0].service_id == sample_template.service_id - assert monthly_billing[0].start_date == apr_month_start - assert monthly_billing[0].end_date == apr_month_end - assert monthly_billing[0].notification_type == 'email' - assert monthly_billing[0].monthly_totals == [] - - assert monthly_billing[1].service_id == sample_template.service_id - assert monthly_billing[1].start_date == apr_month_start - assert monthly_billing[1].end_date == apr_month_end - assert monthly_billing[1].notification_type == 'sms' - assert monthly_billing[1].monthly_totals[0]['billing_units'] == 1 - assert monthly_billing[1].monthly_totals[0]['total_cost'] == 0.0123 - - assert monthly_billing[2].service_id == sample_template.service_id - assert monthly_billing[2].start_date == apr_month_start - assert monthly_billing[2].end_date == apr_month_end - assert monthly_billing[2].notification_type == 'letter' - assert monthly_billing[2].monthly_totals == [] - - def test_run_letter_jobs(client, mocker, sample_letter_template): jobs = [create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND), create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND)] diff --git a/tests/app/celery/test_service_callback_tasks.py b/tests/app/celery/test_service_callback_tasks.py index 4fbc858b1..b17451722 100644 --- a/tests/app/celery/test_service_callback_tasks.py +++ b/tests/app/celery/test_service_callback_tasks.py @@ -3,10 +3,12 @@ from datetime import datetime import pytest import requests_mock +from freezegun import freeze_time from app import (DATETIME_FORMAT, encryption) -from app.celery.service_callback_tasks import send_delivery_status_to_service +from app.celery.service_callback_tasks import send_delivery_status_to_service, send_complaint_to_service from tests.app.db import ( + create_complaint, create_notification, create_service_callback_api, create_service, @@ -19,7 +21,7 @@ from tests.app.db import ( def test_send_delivery_status_to_service_post_https_request_to_service_with_encrypted_data( notify_db_session, notification_type): - callback_api, template = _set_up_test_data(notification_type) + callback_api, template = _set_up_test_data(notification_type, "delivery_status") datestr = datetime(2017, 6, 20) notification = create_notification(template=template, @@ -28,7 +30,7 @@ def test_send_delivery_status_to_service_post_https_request_to_service_with_encr sent_at=datestr, status='sent' ) - encrypted_status_update = _set_up_encrypted_data(callback_api, notification) + encrypted_status_update = _set_up_data_for_status_update(callback_api, notification) with requests_mock.Mocker() as request_mock: request_mock.post(callback_api.url, json={}, @@ -54,12 +56,42 @@ def test_send_delivery_status_to_service_post_https_request_to_service_with_encr assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token) +def test_send_complaint_to_service_posts_https_request_to_service_with_encrypted_data(notify_db_session): + with freeze_time('2001-01-01T12:00:00'): + callback_api, template = _set_up_test_data('email', "complaint") + + notification = create_notification(template=template) + complaint = create_complaint(service=template.service, notification=notification) + complaint_data = _set_up_data_for_complaint(callback_api, complaint, notification) + with requests_mock.Mocker() as request_mock: + request_mock.post(callback_api.url, + json={}, + status_code=200) + send_complaint_to_service(complaint_data) + + mock_data = { + "notification_id": str(notification.id), + "complaint_id": str(complaint.id), + "reference": notification.client_reference, + "to": notification.to, + "complaint_date": datetime.utcnow().strftime( + DATETIME_FORMAT), + } + + assert request_mock.call_count == 1 + assert request_mock.request_history[0].url == callback_api.url + assert request_mock.request_history[0].method == 'POST' + assert request_mock.request_history[0].text == json.dumps(mock_data) + assert request_mock.request_history[0].headers["Content-type"] == "application/json" + assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token) + + @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) -def test_send_delivery_status_to_service_retries_if_request_returns_500_with_encrypted_data( +def test__send_data_to_service_callback_api_retries_if_request_returns_500_with_encrypted_data( notify_db_session, mocker, notification_type ): - callback_api, template = _set_up_test_data(notification_type) + callback_api, template = _set_up_test_data(notification_type, "delivery_status") datestr = datetime(2017, 6, 20) notification = create_notification(template=template, created_at=datestr, @@ -67,7 +99,7 @@ def test_send_delivery_status_to_service_retries_if_request_returns_500_with_enc sent_at=datestr, status='sent' ) - encrypted_data = _set_up_encrypted_data(callback_api, notification) + encrypted_data = _set_up_data_for_status_update(callback_api, notification) mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry') with requests_mock.Mocker() as request_mock: request_mock.post(callback_api.url, @@ -81,12 +113,12 @@ def test_send_delivery_status_to_service_retries_if_request_returns_500_with_enc @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) -def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404_with_encrypted_data( +def test__send_data_to_service_callback_api_does_not_retry_if_request_returns_404_with_encrypted_data( notify_db_session, mocker, notification_type ): - callback_api, template = _set_up_test_data(notification_type) + callback_api, template = _set_up_test_data(notification_type, "delivery_status") datestr = datetime(2017, 6, 20) notification = create_notification(template=template, created_at=datestr, @@ -94,7 +126,7 @@ def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404 sent_at=datestr, status='sent' ) - encrypted_data = _set_up_encrypted_data(callback_api, notification) + encrypted_data = _set_up_data_for_status_update(callback_api, notification) mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry') with requests_mock.Mocker() as request_mock: request_mock.post(callback_api.url, @@ -109,7 +141,7 @@ def test_send_delivery_status_to_service_succeeds_if_sent_at_is_none( notify_db_session, mocker ): - callback_api, template = _set_up_test_data('email') + callback_api, template = _set_up_test_data('email', "delivery_status") datestr = datetime(2017, 6, 20) notification = create_notification(template=template, created_at=datestr, @@ -117,7 +149,7 @@ def test_send_delivery_status_to_service_succeeds_if_sent_at_is_none( sent_at=None, status='technical-failure' ) - encrypted_data = _set_up_encrypted_data(callback_api, notification) + encrypted_data = _set_up_data_for_status_update(callback_api, notification) mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry') with requests_mock.Mocker() as request_mock: request_mock.post(callback_api.url, @@ -128,15 +160,15 @@ def test_send_delivery_status_to_service_succeeds_if_sent_at_is_none( assert mocked.call_count == 0 -def _set_up_test_data(notification_type): +def _set_up_test_data(notification_type, callback_type): service = create_service(restricted=True) template = create_template(service=service, template_type=notification_type, subject='Hello') callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/", - bearer_token="something_unique") + bearer_token="something_unique", callback_type=callback_type) return callback_api, template -def _set_up_encrypted_data(callback_api, notification): +def _set_up_data_for_status_update(callback_api, notification): data = { "notification_id": str(notification.id), "notification_client_reference": notification.client_reference, @@ -152,3 +184,17 @@ def _set_up_encrypted_data(callback_api, notification): } encrypted_status_update = encryption.encrypt(data) return encrypted_status_update + + +def _set_up_data_for_complaint(callback_api, complaint, notification): + data = { + "complaint_id": str(complaint.id), + "notification_id": str(notification.id), + "reference": notification.client_reference, + "to": notification.to, + "complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT), + "service_callback_api_url": callback_api.url, + "service_callback_api_bearer_token": callback_api.bearer_token, + } + obscured_status_update = encryption.encrypt(data) + return obscured_status_update diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index d286da29f..6a21732f1 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -6,9 +6,12 @@ from freezegun import freeze_time from app import db from app.dao.fact_billing_dao import ( - fetch_monthly_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing, - get_rate, + delete_billing_data_for_service_for_day, + fetch_billing_data_for_day, fetch_billing_totals_for_year, + fetch_monthly_billing_for_year, + get_rate, + get_rates_for_billing, ) from app.models import FactBilling, Notification from app.utils import convert_utc_to_bst @@ -353,3 +356,25 @@ def test_fetch_billing_totals_for_year(notify_db_session): assert results[3].notifications_sent == 365 assert results[3].billable_units == 365 assert results[3].rate == Decimal('0.162') + + +def test_delete_billing_data(notify_db_session): + service_1 = create_service(service_name='1') + service_2 = create_service(service_name='2') + sms_template = create_template(service_1, 'sms') + email_template = create_template(service_1, 'email') + other_service_template = create_template(service_2, 'sms') + + existing_rows_to_delete = [ # noqa + create_ft_billing('2018-01-01', 'sms', sms_template, service_1, billable_unit=1), + create_ft_billing('2018-01-01', 'email', email_template, service_1, billable_unit=2) + ] + other_day = create_ft_billing('2018-01-02', 'sms', sms_template, service_1, billable_unit=3) + other_service = create_ft_billing('2018-01-01', 'sms', other_service_template, service_2, billable_unit=4) + + delete_billing_data_for_service_for_day('2018-01-01', service_1.id) + + current_rows = FactBilling.query.all() + assert sorted(x.billable_units for x in current_rows) == sorted( + [other_day.billable_units, other_service.billable_units] + ) diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py deleted file mode 100644 index f422a0b76..000000000 --- a/tests/app/dao/test_monthly_billing.py +++ /dev/null @@ -1,521 +0,0 @@ -from datetime import datetime, timedelta -from dateutil.relativedelta import relativedelta -from freezegun import freeze_time -from functools import partial - -from app.dao.monthly_billing_dao import ( - create_or_update_monthly_billing, - get_monthly_billing_entry, - get_monthly_billing_by_notification_type, - get_service_ids_that_need_billing_populated, - get_billing_data_for_financial_year -) -from app.models import MonthlyBilling, SMS_TYPE, EMAIL_TYPE -from tests.app.conftest import sample_letter_template -from tests.app.db import ( - create_notification, - create_rate, - create_service, - create_template, - create_monthly_billing_entry, -) - -FEB_2016_MONTH_START = datetime(2016, 2, 1) -FEB_2016_MONTH_END = datetime(2016, 2, 29, 23, 59, 59, 99999) - -MAR_2016_MONTH_START = datetime(2016, 3, 1) -MAR_2016_MONTH_END = datetime(2016, 3, 31, 22, 59, 59, 99999) - -APR_2016_MONTH_START = datetime(2016, 3, 31, 23, 00, 00) -APR_2016_MONTH_END = datetime(2016, 4, 30, 22, 59, 59, 99999) - -MAY_2016_MONTH_START = datetime(2016, 5, 31, 23, 00, 00) -MAY_2016_MONTH_END = MAY_2016_MONTH_START + relativedelta(months=1, seconds=-1) - -APR_2017_MONTH_START = datetime(2017, 3, 31, 23, 00, 00) -APR_2017_MONTH_END = datetime(2017, 4, 30, 23, 59, 59, 99999) - -JAN_2017_MONTH_START = datetime(2017, 1, 1) -JAN_2017_MONTH_END = datetime(2017, 1, 31, 23, 59, 59, 99999) - -FEB_2017 = datetime(2017, 2, 15) -APR_2016 = datetime(2016, 4, 10) - -NO_BILLING_DATA = { - "billing_units": 0, - "rate_multiplier": 1, - "international": False, - "rate": 0, - "total_cost": 0 -} - - -def _assert_monthly_billing(monthly_billing, service_id, notification_type, month_start, month_end): - assert monthly_billing.service_id == service_id - assert monthly_billing.notification_type == notification_type - assert monthly_billing.start_date == month_start - assert monthly_billing.end_date == month_end - - -def _assert_monthly_billing_totals(monthly_billing_totals, expected_dict): - assert sorted(monthly_billing_totals.keys()) == sorted(expected_dict.keys()) - assert sorted(monthly_billing_totals.values()) == sorted(expected_dict.values()) - - -def test_get_monthly_billing_by_notification_type_returns_correct_totals(notify_db, notify_db_session): - service = create_service(service_name="Service One") - - create_monthly_billing_entry( - service=service, - monthly_totals=[{ - "billing_units": 12, - "rate": 0.0158, - "rate_multiplier": 5, - "total_cost": 2.1804, - "international": False - }], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016, SMS_TYPE) - - _assert_monthly_billing( - monthly_billing_data, service.id, 'sms', APR_2016_MONTH_START, APR_2016_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing_data.monthly_totals[0], { - "billing_units": 12, - "rate_multiplier": 5, - "international": False, - "rate": 0.0158, - "total_cost": 2.1804 - }) - - -def test_get_monthly_billing_by_notification_type_filters_by_type(notify_db, notify_db_session): - service = create_service(service_name="Service One") - - create_monthly_billing_entry( - service=service, - monthly_totals=[{ - "billing_units": 138, - "rate": 0.0158, - "rate_multiplier": 1, - "total_cost": 2.1804, - "international": None - }], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - create_monthly_billing_entry( - service=service, - monthly_totals=[], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=EMAIL_TYPE - ) - - monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016, EMAIL_TYPE) - - _assert_monthly_billing( - monthly_billing_data, service.id, 'email', APR_2016_MONTH_START, APR_2016_MONTH_END - ) - assert monthly_billing_data.monthly_totals == [] - - -def test_get_monthly_billing_by_notification_type_normalises_start_date(notify_db, notify_db_session): - service = create_service(service_name="Service One") - - create_monthly_billing_entry( - service=service, - monthly_totals=[{ - "billing_units": 321, - "rate": 0.0158, - "rate_multiplier": 1, - "total_cost": 2.1804, - "international": None - }], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016 + timedelta(days=5), SMS_TYPE) - - assert monthly_billing_data.start_date == APR_2016_MONTH_START - assert monthly_billing_data.monthly_totals[0]['billing_units'] == 321 - - -def test_add_monthly_billing_for_single_month_populates_correctly( - sample_template, sample_email_template -): - create_rate(start_date=JAN_2017_MONTH_START, value=0.0158, notification_type=SMS_TYPE) - letter_template = sample_letter_template(sample_template.service) - create_notification( - template=sample_template, created_at=JAN_2017_MONTH_START, - billable_units=1, rate_multiplier=2, status='delivered' - ) - create_notification(template=sample_email_template, created_at=JAN_2017_MONTH_START, - status='delivered') - create_notification(template=letter_template, created_at=JAN_2017_MONTH_START, status='delivered') - - create_or_update_monthly_billing( - service_id=sample_template.service_id, - billing_month=JAN_2017_MONTH_START - ) - - monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all() - - assert len(monthly_billing) == 3 - _assert_monthly_billing( - monthly_billing[0], sample_template.service.id, 'email', JAN_2017_MONTH_START, JAN_2017_MONTH_END - ) - _assert_monthly_billing_totals( - monthly_billing[0].monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 1, - "international": False, - "rate": 0.0, - "total_cost": 0 - }) - - _assert_monthly_billing( - monthly_billing[1], sample_template.service.id, 'sms', JAN_2017_MONTH_START, JAN_2017_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 2, - "international": False, - "rate": 0.0158, - "total_cost": 1 * 2 * 0.0158 - }) - - _assert_monthly_billing( - monthly_billing[2], sample_template.service.id, 'letter', JAN_2017_MONTH_START, JAN_2017_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing[2].monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 1, - "international": False, - "rate": 0.33, - "total_cost": 1 * 0.33 - }) - - -def test_add_monthly_billing_for_multiple_months_populate_correctly( - sample_template -): - create_rate(start_date=FEB_2016_MONTH_START - timedelta(days=1), value=0.12, notification_type=SMS_TYPE) - create_notification( - template=sample_template, created_at=FEB_2016_MONTH_START, - billable_units=1, rate_multiplier=2, status='delivered' - ) - create_notification( - template=sample_template, created_at=MAR_2016_MONTH_START, - billable_units=2, rate_multiplier=3, status='delivered' - ) - - create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=FEB_2016_MONTH_START) - create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=MAR_2016_MONTH_START) - - monthly_billing = MonthlyBilling.query.order_by( - MonthlyBilling.notification_type, - MonthlyBilling.start_date - ).all() - - assert len(monthly_billing) == 6 - _assert_monthly_billing( - monthly_billing[0], sample_template.service.id, 'email', FEB_2016_MONTH_START, FEB_2016_MONTH_END - ) - assert monthly_billing[0].monthly_totals == [] - - _assert_monthly_billing( - monthly_billing[1], sample_template.service.id, 'email', MAR_2016_MONTH_START, MAR_2016_MONTH_END - ) - assert monthly_billing[1].monthly_totals == [] - - _assert_monthly_billing( - monthly_billing[2], sample_template.service.id, 'sms', FEB_2016_MONTH_START, FEB_2016_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing[2].monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 2, - "international": False, - "rate": 0.12, - "total_cost": 0.24 - }) - - _assert_monthly_billing( - monthly_billing[3], sample_template.service.id, 'sms', MAR_2016_MONTH_START, MAR_2016_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing[3].monthly_totals[0], { - "billing_units": 2, - "rate_multiplier": 3, - "international": False, - "rate": 0.12, - "total_cost": 0.72 - }) - - _assert_monthly_billing( - monthly_billing[4], sample_template.service.id, 'letter', FEB_2016_MONTH_START, FEB_2016_MONTH_END - ) - assert monthly_billing[4].monthly_totals == [] - - _assert_monthly_billing( - monthly_billing[5], sample_template.service.id, 'letter', MAR_2016_MONTH_START, MAR_2016_MONTH_END - ) - assert monthly_billing[5].monthly_totals == [] - - -def test_add_monthly_billing_with_multiple_rates_populate_correctly( - sample_template, sample_email_template -): - letter_template = sample_letter_template(sample_template.service) - create_rate(start_date=JAN_2017_MONTH_START, value=0.0158, notification_type=SMS_TYPE) - create_rate(start_date=JAN_2017_MONTH_START + timedelta(days=5), value=0.123, notification_type=SMS_TYPE) - create_notification(template=sample_template, created_at=JAN_2017_MONTH_START, billable_units=1, status='delivered') - create_notification( - template=sample_template, created_at=JAN_2017_MONTH_START + timedelta(days=6), - billable_units=2, status='delivered' - ) - - create_notification(template=sample_email_template, created_at=JAN_2017_MONTH_START, status='delivered') - create_notification(template=letter_template, created_at=JAN_2017_MONTH_START, status='delivered', - billable_units=1) - - create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=JAN_2017_MONTH_START) - - monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all() - - assert len(monthly_billing) == 3 - _assert_monthly_billing( - monthly_billing[0], sample_template.service.id, 'email', JAN_2017_MONTH_START, JAN_2017_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing[0].monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 1, - "international": False, - "rate": 0.0, - "total_cost": 0.0 - }) - - _assert_monthly_billing( - monthly_billing[1], sample_template.service.id, 'sms', JAN_2017_MONTH_START, JAN_2017_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 1, - "international": False, - "rate": 0.0158, - "total_cost": 0.0158 - }) - _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[1], { - "billing_units": 2, - "rate_multiplier": 1, - "international": False, - "rate": 0.123, - "total_cost": 0.246 - }) - - _assert_monthly_billing( - monthly_billing[2], sample_template.service.id, 'letter', JAN_2017_MONTH_START, JAN_2017_MONTH_END - ) - _assert_monthly_billing_totals(monthly_billing[2].monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 1, - "international": False, - "rate": 0.33, - "total_cost": 0.33 - }) - - -def test_update_monthly_billing_overwrites_old_totals(sample_template): - create_rate(APR_2016_MONTH_START, 0.123, SMS_TYPE) - create_notification(template=sample_template, created_at=APR_2016_MONTH_START, billable_units=1, status='delivered') - - create_or_update_monthly_billing(sample_template.service_id, APR_2016_MONTH_END) - first_update = get_monthly_billing_by_notification_type(sample_template.service_id, APR_2016_MONTH_START, SMS_TYPE) - - _assert_monthly_billing( - first_update, sample_template.service.id, 'sms', APR_2016_MONTH_START, APR_2016_MONTH_END - ) - _assert_monthly_billing_totals(first_update.monthly_totals[0], { - "billing_units": 1, - "rate_multiplier": 1, - "international": False, - "rate": 0.123, - "total_cost": 0.123 - }) - - first_updated_at = first_update.updated_at - - with freeze_time(APR_2016_MONTH_START + timedelta(days=3)): - create_notification(template=sample_template, billable_units=2, status='delivered') - create_or_update_monthly_billing(sample_template.service_id, APR_2016_MONTH_END) - - second_update = get_monthly_billing_by_notification_type(sample_template.service_id, APR_2016_MONTH_START, SMS_TYPE) - - _assert_monthly_billing_totals(second_update.monthly_totals[0], { - "billing_units": 3, - "rate_multiplier": 1, - "international": False, - "rate": 0.123, - "total_cost": 0.369 - }) - - assert second_update.updated_at == APR_2016_MONTH_START + timedelta(days=3) - assert first_updated_at != second_update.updated_at - - -def test_get_service_ids_that_need_billing_populated_return_correctly(notify_db_session): - service_1 = create_service(service_name="Service One") - template_1 = create_template(service=service_1) - service_2 = create_service(service_name="Service Two") - template_2 = create_template(service=service_2) - create_notification(template=template_1, created_at=datetime(2017, 6, 30, 13, 30), status='delivered') - create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered') - create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30)) - create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30)) - services = get_service_ids_that_need_billing_populated( - start_date=datetime(2017, 7, 1), end_date=datetime(2017, 7, 16) - ) - expected_services = [service_1.id, service_2.id] - assert sorted([x.service_id for x in services]) == sorted(expected_services) - - -def test_get_monthly_billing_entry_filters_by_service(notify_db, notify_db_session): - service_1 = create_service(service_name="Service One") - service_2 = create_service(service_name="Service Two") - now = datetime.utcnow() - - create_monthly_billing_entry( - service=service_1, - monthly_totals=[], - start_date=now, - end_date=now + timedelta(days=30), - notification_type=SMS_TYPE - ) - - create_monthly_billing_entry( - service=service_2, - monthly_totals=[], - start_date=now, - end_date=now + timedelta(days=30), - notification_type=SMS_TYPE - ) - - entry = get_monthly_billing_entry(service_2.id, now, SMS_TYPE) - - assert entry.start_date == now - assert entry.service_id == service_2.id - - -def test_get_yearly_billing_data_for_year_returns_within_year_only( - sample_template -): - monthly_billing_entry = partial( - create_monthly_billing_entry, service=sample_template.service, notification_type=SMS_TYPE - ) - monthly_billing_entry(start_date=FEB_2016_MONTH_START, end_date=FEB_2016_MONTH_END) - monthly_billing_entry( - monthly_totals=[{ - "billing_units": 138, - "rate": 0.0158, - "rate_multiplier": 1, - "total_cost": 2.1804, - "international": None - }], - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - monthly_billing_entry(start_date=APR_2017_MONTH_START, end_date=APR_2017_MONTH_END) - - billing_data = get_billing_data_for_financial_year(sample_template.service.id, 2016, [SMS_TYPE]) - - assert len(billing_data) == 1 - assert billing_data[0].monthly_totals[0]['billing_units'] == 138 - - -def test_get_yearly_billing_data_for_year_returns_multiple_notification_types(sample_template): - monthly_billing_entry = partial( - create_monthly_billing_entry, service=sample_template.service, - start_date=APR_2016_MONTH_START, end_date=APR_2016_MONTH_END - ) - - monthly_billing_entry( - notification_type=SMS_TYPE, monthly_totals=[] - ) - monthly_billing_entry( - notification_type=EMAIL_TYPE, - monthly_totals=[{ - "billing_units": 2, - "rate": 1.3, - "rate_multiplier": 3, - "total_cost": 2.1804, - "international": False - }] - ) - - billing_data = get_billing_data_for_financial_year( - service_id=sample_template.service.id, - year=2016, - notification_types=[SMS_TYPE, EMAIL_TYPE] - ) - - assert len(billing_data) == 2 - - assert billing_data[0].notification_type == EMAIL_TYPE - assert billing_data[0].monthly_totals[0]['billing_units'] == 2 - assert billing_data[1].notification_type == SMS_TYPE - - -@freeze_time("2016-04-21 11:00:00") -def test_get_yearly_billing_data_for_year_includes_current_day_totals(sample_template): - create_rate(start_date=FEB_2016_MONTH_START, value=0.0158, notification_type=SMS_TYPE) - - create_monthly_billing_entry( - service=sample_template.service, - start_date=APR_2016_MONTH_START, - end_date=APR_2016_MONTH_END, - notification_type=SMS_TYPE - ) - - billing_data = get_billing_data_for_financial_year( - service_id=sample_template.service.id, - year=2016, - notification_types=[SMS_TYPE] - ) - - assert len(billing_data) == 1 - assert billing_data[0].notification_type == SMS_TYPE - assert billing_data[0].monthly_totals == [] - - create_notification( - template=sample_template, - created_at=datetime.utcnow(), - sent_at=datetime.utcnow(), - status='sending', - billable_units=3 - ) - - billing_data = get_billing_data_for_financial_year( - service_id=sample_template.service.id, - year=2016, - notification_types=[SMS_TYPE] - ) - - assert billing_data[0].monthly_totals[0]['billing_units'] == 3 - - -@freeze_time("2017-06-16 13:00:00") -def test_get_billing_data_for_financial_year_updated_monthly_billing_if_today_is_in_current_year( - sample_service, - mocker -): - mock = mocker.patch("app.dao.monthly_billing_dao.create_or_update_monthly_billing") - get_billing_data_for_financial_year(sample_service.id, 2016) - mock.assert_not_called() diff --git a/tests/app/dao/test_service_callback_api_dao.py b/tests/app/dao/test_service_callback_api_dao.py index 16d4ae95d..3b05e2fd1 100644 --- a/tests/app/dao/test_service_callback_api_dao.py +++ b/tests/app/dao/test_service_callback_api_dao.py @@ -56,6 +56,49 @@ def test_save_service_callback_api_fails_if_service_does_not_exist(notify_db, no save_service_callback_api(service_callback_api) +def test_update_service_callback_api_unique_constraint(sample_service): + service_callback_api = ServiceCallbackApi( + service_id=sample_service.id, + url="https://some_service/callback_endpoint", + bearer_token="some_unique_string", + updated_by_id=sample_service.users[0].id, + callback_type='delivery_status' + ) + save_service_callback_api(service_callback_api) + another = ServiceCallbackApi( + service_id=sample_service.id, + url="https://some_service/another_callback_endpoint", + bearer_token="different_string", + updated_by_id=sample_service.users[0].id, + callback_type='delivery_status' + ) + with pytest.raises(expected_exception=SQLAlchemyError): + save_service_callback_api(another) + + +def test_update_service_callback_can_add_two_api_of_different_types(sample_service): + delivery_status = ServiceCallbackApi( + service_id=sample_service.id, + url="https://some_service/callback_endpoint", + bearer_token="some_unique_string", + updated_by_id=sample_service.users[0].id, + callback_type='delivery_status' + ) + save_service_callback_api(delivery_status) + complaint = ServiceCallbackApi( + service_id=sample_service.id, + url="https://some_service/another_callback_endpoint", + bearer_token="different_string", + updated_by_id=sample_service.users[0].id, + callback_type='complaint' + ) + save_service_callback_api(complaint) + results = ServiceCallbackApi.query.order_by(ServiceCallbackApi.callback_type).all() + assert len(results) == 2 + assert results[0].serialize() == complaint.serialize() + assert results[1].serialize() == delivery_status.serialize() + + def test_update_service_callback_api(sample_service): service_callback_api = ServiceCallbackApi( service_id=sample_service.id, diff --git a/tests/app/db.py b/tests/app/db.py index b42203a6b..4e275155d 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -14,7 +14,6 @@ from app.models import ( InboundSms, InboundNumber, Job, - MonthlyBilling, Notification, EmailBranding, Organisation, @@ -382,27 +381,6 @@ def create_inbound_number(number, provider='mmg', active=True, service_id=None): return inbound_number -def create_monthly_billing_entry( - service, - start_date, - end_date, - notification_type, - monthly_totals=[] -): - entry = MonthlyBilling( - service_id=service.id, - notification_type=notification_type, - monthly_totals=monthly_totals, - start_date=start_date, - end_date=end_date - ) - - db.session.add(entry) - db.session.commit() - - return entry - - def create_reply_to_email( service, email_address, diff --git a/tests/app/notifications/test_notifications_ses_callback.py b/tests/app/notifications/test_notifications_ses_callback.py index 701c1a21c..72166b9e8 100644 --- a/tests/app/notifications/test_notifications_ses_callback.py +++ b/tests/app/notifications/test_notifications_ses_callback.py @@ -5,7 +5,7 @@ from flask import json from freezegun import freeze_time from sqlalchemy.exc import SQLAlchemyError -from app import statsd_client +from app import statsd_client, encryption from app.dao.notifications_dao import get_notification_by_id from app.models import Notification, Complaint from app.notifications.notifications_ses_callback import ( @@ -13,7 +13,7 @@ from app.notifications.notifications_ses_callback import ( handle_complaint ) from app.celery.research_mode_tasks import ses_hard_bounce_callback, ses_soft_bounce_callback, ses_notification_callback -from app.celery.service_callback_tasks import create_encrypted_callback_data +from app.celery.service_callback_tasks import create_delivery_status_callback_data from tests.app.conftest import sample_notification as create_sample_notification from tests.app.db import ( @@ -55,7 +55,7 @@ def test_ses_callback_should_update_notification_status( ) statsd_client.incr.assert_any_call("callback.ses.delivered") updated_notification = Notification.query.get(notification.id) - encrypted_data = create_encrypted_callback_data(updated_notification, callback_api) + encrypted_data = create_delivery_status_callback_data(updated_notification, callback_api) send_mock.assert_called_once_with([str(notification.id), encrypted_data], queue="service-callbacks") @@ -220,3 +220,28 @@ def test_process_ses_results_in_complaint_save_complaint_with_null_complaint_typ assert len(complaints) == 1 assert complaints[0].notification_id == notification.id assert not complaints[0].complaint_type + + +def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker): + send_mock = mocker.patch( + 'app.celery.service_callback_tasks.send_complaint_to_service.apply_async' + ) + create_service_callback_api( + service=sample_email_template.service, url="https://original_url.com", callback_type="complaint" + ) + + notification = create_notification(template=sample_email_template, reference='ref1') + response = ses_complaint_callback() + errors = process_ses_response(response) + assert errors is None + + assert send_mock.call_count == 1 + assert encryption.decrypt(send_mock.call_args[0][0][0]) == { + 'complaint_date': '2018-06-05T13:59:58.000000Z', + 'complaint_id': str(Complaint.query.one().id), + 'notification_id': str(notification.id), + 'reference': None, + 'service_callback_api_bearer_token': 'some_super_secret', + 'service_callback_api_url': 'https://original_url.com', + 'to': 'recipient1@example.com' + } diff --git a/tests/app/notifications/test_process_client_response.py b/tests/app/notifications/test_process_client_response.py index 1ea18cb69..6a22a2b03 100644 --- a/tests/app/notifications/test_process_client_response.py +++ b/tests/app/notifications/test_process_client_response.py @@ -7,7 +7,7 @@ from app.notifications.process_client_response import ( validate_callback_data, process_sms_client_response ) -from app.celery.service_callback_tasks import create_encrypted_callback_data +from app.celery.service_callback_tasks import create_delivery_status_callback_data from tests.app.db import create_service_callback_api @@ -64,7 +64,7 @@ def test_outcome_statistics_called_for_successful_callback(sample_notification, success, error = process_sms_client_response(status='3', provider_reference=reference, client_name='MMG') assert success == "MMG callback succeeded. reference {} updated".format(str(reference)) assert error is None - encrypted_data = create_encrypted_callback_data(sample_notification, callback_api) + encrypted_data = create_delivery_status_callback_data(sample_notification, callback_api) send_mock.assert_called_once_with([str(sample_notification.id), encrypted_data], queue="service-callbacks")