Merge pull request #3477 from alphagov/volume-stat-report

Report for total notifications sent per day for each channel.

Makefile

@@ -27,6 +27,10 @@ bootstrap: generate-version-file ## Set up everything to run the app
 	createdb notification_api || true
 	(. environment.sh && flask db upgrade) || true
 
+.PHONY: bootstrap-with-docker
+bootstrap-with-docker: ## Build the image to run the app in Docker
+	docker build -f docker/Dockerfile -t notifications-api .
+
 .PHONY: run-flask
 run-flask: ## Run flask
 	. environment.sh && flask run -p 6011
@@ -39,12 +43,20 @@ run-celery: ## Run celery
 		--loglevel=INFO \
 		--concurrency=4
 
+.PHONY: run-celery-with-docker
+run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
+	./scripts/run_with_docker.sh make run-celery
+
 .PHONY: run-celery-beat
 run-celery-beat: ## Run celery beat
 	. environment.sh && celery \
 		-A run_celery.notify_celery beat \
 		--loglevel=INFO
 
+.PHONY: run-celery-beat-with-docker
+run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)
+	./scripts/run_with_docker.sh make run-celery-beat
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -177,8 +189,3 @@ disable-failwhale: ## Disable the failwhale app and enable api
 	cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api
 	cf stop notify-api-failwhale
 	@echo "Failwhale is disabled"
-
-.PHONY: run-celery-with-docker
-run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
-	docker build -f docker/Dockerfile -t notifications-api .
-	./scripts/run_with_docker.sh make run-celery

README.md

@@ -61,10 +61,19 @@ export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/
 
 ### Redis
 
-To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:
+To switch redis on you'll need to install it locally. On a Mac you can do:
 
-        REDIS_ENABLED = True
+```
+# assuming you use Homebrew
+brew install redis
+brew services start redis
+```
 
+To use redis caching you need to switch it on with an environment variable:
+
+```
+export REDIS_ENABLED=1
+```
 
 ##  To run the application
 
@@ -82,6 +91,19 @@ make run-celery
 make run-celery-beat
 ```
 
+We've had problems running Celery locally due to one of its dependencies: pycurl. Due to the complexity of the issue, we also support running Celery via Docker:
+
+```
+# install dependencies, etc.
+make bootstrap-with-docker
+
+# run the background tasks
+make run-celery-with-docker
+
+# run scheduled tasks
+make run-celery-beat-with-docker
+```
+
 ##  To test the application
 
 ```

app/billing/billing_schemas.py

@@ -6,7 +6,7 @@ create_or_update_free_sms_fragment_limit_schema = {
     "type": "object",
     "title": "Create",
     "properties": {
-        "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
+        "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
     },
     "required": ["free_sms_fragment_limit"]
 }

app/celery/nightly_tasks.py

@@ -255,7 +255,7 @@ def letter_raise_alert_if_no_ack_file_for_zip():
 
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                               subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))
+        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))  # noqa
 
     message = '\n'.join([
         "Letter ack file does not contain all zip files sent."

app/celery/service_callback_tasks.py

@@ -68,7 +68,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                 'Content-Type': 'application/json',
                 'Authorization': 'Bearer {}'.format(token)
             },
-            timeout=60
+            timeout=5
         )
         current_app.logger.info('{} sending {} to {}, response {}'.format(
             function_name,

app/commands.py

@@ -246,14 +246,15 @@ def fix_notification_statuses_not_in_sync():
               one number per line. The number must have the format of 07... not 447....""")
 def insert_inbound_numbers_from_file(file_name):
     print("Inserting inbound numbers from {}".format(file_name))
-    file = open(file_name)
-    sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
+    with open(file_name) as file:
+        sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
 
-    for line in file:
-        print(line)
-        db.session.execute(sql.format(uuid.uuid4(), line.strip()))
-        db.session.commit()
-    file.close()
+        for line in file:
+            line = line.strip()
+            if line:
+                print(line)
+                db.session.execute(sql.format(uuid.uuid4(), line))
+                db.session.commit()
 
 
 @notify_command(name='replay-create-pdf-for-templated-letter')

app/config.py

@@ -307,7 +307,7 @@ class Config(object):
             },
             'raise-alert-if-letter-notifications-still-sending': {
                 'task': 'raise-alert-if-letter-notifications-still-sending',
-                'schedule': crontab(hour=15, minute=30),
+                'schedule': crontab(hour=17, minute=00),
                 'options': {'queue': QueueNames.PERIODIC}
             },
             # The collate-letter-pdf does assume it is called in an hour that BST does not make a
@@ -428,7 +428,7 @@ class Development(Config):
     NOTIFY_EMAIL_DOMAIN = "notify.tools"
 
     SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api')
-    REDIS_URL = 'redis://localhost:6379/0'
+    REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
 
     ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
 

app/dao/fact_billing_dao.py

@@ -716,3 +716,151 @@ def fetch_billing_details_for_all_services():
     ).all()
 
     return billing_details
+
+
+def fetch_daily_volumes_for_platform(start_date, end_date):
+    # query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
+
+    daily_volume_stats = db.session.query(
+        FactBilling.bst_date,
+        func.sum(case(
+            [
+                (FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
+            ], else_=0
+        )).label('sms_totals'),
+        func.sum(case(
+            [
+                (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units)
+            ], else_=0
+        )).label('sms_fragment_totals'),
+        func.sum(case(
+            [
+                (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units * FactBilling.rate_multiplier)
+            ], else_=0
+        )).label('sms_fragments_times_multiplier'),
+        func.sum(case(
+            [
+                (FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
+            ], else_=0
+        )).label('email_totals'),
+        func.sum(case(
+            [
+                (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
+            ], else_=0
+        )).label('letter_totals'),
+        func.sum(case(
+            [
+                (FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
+            ], else_=0
+        )).label('letter_sheet_totals')
+    ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date
+    ).group_by(
+        FactBilling.bst_date,
+        FactBilling.notification_type
+    ).subquery()
+
+    aggregated_totals = db.session.query(
+        daily_volume_stats.c.bst_date.cast(db.Text).label('bst_date'),
+        func.sum(daily_volume_stats.c.sms_totals).label('sms_totals'),
+        func.sum(daily_volume_stats.c.sms_fragment_totals).label('sms_fragment_totals'),
+        func.sum(
+            daily_volume_stats.c.sms_fragments_times_multiplier).label('sms_chargeable_units'),
+        func.sum(daily_volume_stats.c.email_totals).label('email_totals'),
+        func.sum(daily_volume_stats.c.letter_totals).label('letter_totals'),
+        func.sum(daily_volume_stats.c.letter_sheet_totals).label('letter_sheet_totals')
+    ).group_by(
+        daily_volume_stats.c.bst_date
+    ).order_by(
+        daily_volume_stats.c.bst_date
+    ).all()
+
+    return aggregated_totals
+
+
+def fetch_volumes_by_service(start_date, end_date):
+    # query to return the volume totals by service aggregated for the date range given
+    # start and end dates are inclusive.
+    year_end_date = int(end_date.strftime('%Y'))
+
+    volume_stats = db.session.query(
+        FactBilling.bst_date,
+        FactBilling.service_id,
+        func.sum(case([
+            (FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
+        ], else_=0)).label('sms_totals'),
+        func.sum(case([
+            (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units * FactBilling.rate_multiplier)
+        ], else_=0)).label('sms_fragments_times_multiplier'),
+        func.sum(case([
+            (FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
+        ], else_=0)).label('email_totals'),
+        func.sum(case([
+            (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
+        ], else_=0)).label('letter_totals'),
+        func.sum(case([
+            (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent * FactBilling.rate)
+        ], else_=0)).label("letter_cost"),
+        func.sum(case(
+            [
+                (FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
+            ], else_=0
+        )).label('letter_sheet_totals')
+    ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date
+    ).group_by(
+        FactBilling.bst_date,
+        FactBilling.service_id,
+        FactBilling.notification_type
+    ).subquery()
+
+    annual_billing = db.session.query(
+        func.max(AnnualBilling.financial_year_start).label('financial_year_start'),
+        AnnualBilling.service_id,
+        AnnualBilling.free_sms_fragment_limit
+    ).filter(
+        AnnualBilling.financial_year_start <= year_end_date
+    ).group_by(
+        AnnualBilling.service_id,
+        AnnualBilling.free_sms_fragment_limit
+    ).subquery()
+
+    results = db.session.query(
+        Service.name.label("service_name"),
+        Service.id.label("service_id"),
+        Service.organisation_id.label("organisation_id"),
+        Organisation.name.label("organisation_name"),
+        annual_billing.c.free_sms_fragment_limit.label("free_allowance"),
+        func.coalesce(func.sum(volume_stats.c.sms_totals), 0).label("sms_notifications"),
+        func.coalesce(func.sum(volume_stats.c.sms_fragments_times_multiplier), 0
+                      ).label("sms_chargeable_units"),
+        func.coalesce(func.sum(volume_stats.c.email_totals), 0).label("email_totals"),
+        func.coalesce(func.sum(volume_stats.c.letter_totals), 0).label("letter_totals"),
+        func.coalesce(func.sum(volume_stats.c.letter_cost), 0).label("letter_cost"),
+        func.coalesce(func.sum(volume_stats.c.letter_sheet_totals), 0).label("letter_sheet_totals")
+    ).select_from(
+        Service
+    ).outerjoin(
+        Organisation, Service.organisation_id == Organisation.id
+    ).join(
+        annual_billing, Service.id == annual_billing.c.service_id
+    ).outerjoin(  # include services without volume
+        volume_stats, Service.id == volume_stats.c.service_id
+    ).filter(
+        Service.restricted.is_(False),
+        Service.count_as_live.is_(True),
+        Service.active.is_(True)
+    ).group_by(
+        Service.id,
+        Service.name,
+        Service.organisation_id,
+        Organisation.name,
+        annual_billing.c.free_sms_fragment_limit
+    ).order_by(
+        Organisation.name,
+        Service.name,
+    ).all()
+
+    return results

app/dao/services_dao.py

@@ -544,8 +544,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
         Notification.notification_type == SMS_TYPE,
         func.substr(Notification.normalised_to, 3, 7) == '7700900',
         Service.restricted == False,  # noqa
-        Service.research_mode == False,
-        Service.active == True,
+        Service.research_mode == False,  # noqa
+        Service.active == True,  # noqa
     ).group_by(
         Notification.service_id,
     ).having(
@@ -553,7 +553,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
     ).all()
 
 
-def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=1000):
+def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
     subquery = db.session.query(
         func.count(Notification.id).label('total_count'),
         Notification.service_id.label('service_id')
@@ -564,8 +564,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.key_type != KEY_TYPE_TEST,
         Notification.notification_type == SMS_TYPE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,
-        Service.active == True,
+        Service.research_mode == False,  # noqa
+        Service.active == True,  # noqa
     ).group_by(
         Notification.service_id,
     ).having(
@@ -590,8 +590,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.notification_type == SMS_TYPE,
         Notification.status == NOTIFICATION_PERMANENT_FAILURE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,
-        Service.active == True,
+        Service.research_mode == False,  # noqa
+        Service.active == True,  # noqa
     ).group_by(
         Notification.service_id,
         subquery.c.total_count

app/dao/uploads_dao.py

@@ -84,7 +84,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Notification.notification_type == LETTER_TYPE,
         Notification.api_key_id == None,  # noqa
         Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden == True,
+        Template.hidden == True,  # noqa
         Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
     ]
     if limit_days is not None:

app/letters/rest.py

@@ -9,11 +9,19 @@ from app.v2.errors import register_errors
 letter_job = Blueprint("letter-job", __name__)
 register_errors(letter_job)
 
+# too many references will make SQS error (as the task can only be 256kb)
+MAX_REFERENCES_PER_TASK = 5000
+
 
 @letter_job.route('/letters/returned', methods=['POST'])
 def create_process_returned_letters_job():
-    references = validate(request.get_json(), letter_references)
+    references = validate(request.get_json(), letter_references)['references']
 
-    process_returned_letters_list.apply_async([references['references']], queue=QueueNames.DATABASE)
+    for start_index in range(0, len(references), MAX_REFERENCES_PER_TASK):
+        process_returned_letters_list.apply_async(
+            args=(references[start_index:start_index + MAX_REFERENCES_PER_TASK], ),
+            queue=QueueNames.DATABASE,
+            compression='zlib'
+        )
 
-    return jsonify(references=references['references']), 200
+    return jsonify(references=references), 200

app/platform_stats/rest.py

@@ -5,9 +5,11 @@ from flask import Blueprint, jsonify, request
 from app.dao.date_util import get_financial_year_for_datetime
 from app.dao.fact_billing_dao import (
     fetch_billing_details_for_all_services,
+    fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
     fetch_sms_billing_for_all_services,
+    fetch_volumes_by_service,
 )
 from app.dao.fact_notification_status_dao import (
     fetch_notification_status_totals_for_all_services,
@@ -40,12 +42,17 @@ def get_platform_stats():
     return jsonify(stats)
 
 
-def validate_date_range_is_within_a_financial_year(start_date, end_date):
+def validate_date_format(date_to_validate):
     try:
-        start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
-        end_date = datetime.strptime(end_date, "%Y-%m-%d").date()
+        validated_date = datetime.strptime(date_to_validate, "%Y-%m-%d").date()
     except ValueError:
         raise InvalidRequest(message="Input must be a date in the format: YYYY-MM-DD", status_code=400)
+    return validated_date
+
+
+def validate_date_range_is_within_a_financial_year(start_date, end_date):
+    start_date = validate_date_format(start_date)
+    end_date = validate_date_format(end_date)
     if end_date < start_date:
         raise InvalidRequest(message="Start date must be before end date", status_code=400)
 
@@ -133,6 +140,53 @@ def get_data_for_billing_report():
     return jsonify(result)
 
 
+@platform_stats_blueprint.route('daily-volumes-report')
+def daily_volumes_report():
+    start_date = validate_date_format(request.args.get('start_date'))
+    end_date = validate_date_format(request.args.get('end_date'))
+
+    daily_volumes = fetch_daily_volumes_for_platform(start_date, end_date)
+    report = []
+
+    for row in daily_volumes:
+        report.append({
+            "day": row.bst_date,
+            "sms_totals": int(row.sms_totals),
+            "sms_fragment_totals": int(row.sms_fragment_totals),
+            "sms_chargeable_units": int(row.sms_chargeable_units),
+            "email_totals": int(row.email_totals),
+            "letter_totals": int(row.letter_totals),
+            "letter_sheet_totals": int(row.letter_sheet_totals)
+        })
+    return jsonify(report)
+
+
+@platform_stats_blueprint.route('volumes-by-service')
+def volumes_by_service_report():
+    start_date = validate_date_format(request.args.get('start_date'))
+    end_date = validate_date_format(request.args.get('end_date'))
+
+    volumes_by_service = fetch_volumes_by_service(start_date, end_date)
+    report = []
+
+    for row in volumes_by_service:
+        report.append({
+            "service_name": row.service_name,
+            "service_id": str(row.service_id),
+            "organisation_name": row.organisation_name if row.organisation_name else '',
+            "organisation_id": str(row.organisation_id) if row.organisation_id else '',
+            "free_allowance": int(row.free_allowance),
+            "sms_notifications": int(row.sms_notifications),
+            "sms_chargeable_units": int(row.sms_chargeable_units),
+            "email_totals": int(row.email_totals),
+            "letter_totals": int(row.letter_totals),
+            "letter_sheet_totals": int(row.letter_sheet_totals),
+            "letter_cost": float(row.letter_cost),
+        })
+
+    return jsonify(report)
+
+
 def postage_description(postage):
     if postage in UK_POSTAGE_TYPES:
         return f'{postage} class'

app/user/rest.py

@@ -374,6 +374,8 @@ def send_user_confirm_new_email(user_id):
 
 @user_blueprint.route('/<uuid:user_id>/email-verification', methods=['POST'])
 def send_new_user_email_verification(user_id):
+    request_json = request.get_json()
+
     # when registering, we verify all users' email addresses using this function
     user_to_send_to = get_user_by_id(user_id=user_id)
 
@@ -387,7 +389,10 @@ def send_new_user_email_verification(user_id):
         service=service,
         personalisation={
             'name': user_to_send_to.name,
-            'url': _create_verification_url(user_to_send_to)
+            'url': _create_verification_url(
+                user_to_send_to,
+                base_url=request_json.get('admin_base_url'),
+            ),
         },
         notification_type=template.template_type,
         api_key_id=None,
@@ -556,10 +561,10 @@ def _create_reset_password_url(email, next_redirect, base_url=None):
     return full_url
 
 
-def _create_verification_url(user):
+def _create_verification_url(user, base_url):
     data = json.dumps({'user_id': str(user.id), 'email': user.email_address})
     url = '/verify-email/'
-    return url_with_token(data, url, current_app.config)
+    return url_with_token(data, url, current_app.config, base_url=base_url)
 
 
 def _create_confirmation_url(user, email_address):

migrations/versions/0366_letter_rates_2022.py

@@ -0,0 +1,63 @@
+"""
+
+Revision ID: 0366_letter_rates_2022
+Revises: 0365_add_nhs_branding
+Create Date: 2022-03-01 14:00:00
+
+"""
+import itertools
+import uuid
+from datetime import datetime
+
+from alembic import op
+from sqlalchemy.sql import text
+
+from app.models import LetterRate
+
+
+revision = '0366_letter_rates_2022'
+down_revision = '0365_add_nhs_branding'
+
+
+CHANGEOVER_DATE = datetime(2022, 3, 1, 0, 0)
+
+
+def get_new_rate(sheet_count, post_class):
+    base_prices = {
+        'second': 36,
+        'first': 58,
+        'europe': 88,
+        'rest-of-world': 88,
+    }
+    multiplier = 5 if post_class in ('first', 'second') else 8
+
+    return (base_prices[post_class] + (multiplier * sheet_count)) / 100.0
+
+
+def upgrade():
+    conn = op.get_bind()
+    conn.execute(text("UPDATE letter_rates SET end_date = :start WHERE end_date IS NULL"), start=CHANGEOVER_DATE)
+
+    op.bulk_insert(LetterRate.__table__, [
+        {
+            'id': uuid.uuid4(),
+            'start_date': CHANGEOVER_DATE,
+            'end_date': None,
+            'sheet_count': sheet_count,
+            'rate': get_new_rate(sheet_count, post_class),
+            'crown': crown,
+            'post_class': post_class,
+        }
+        for sheet_count, crown, post_class in itertools.product(
+            range(1, 6),
+            [True, False],
+            ['first', 'second', 'europe', 'rest-of-world']
+        )
+    ])
+
+
+def downgrade():
+    # Make sure you've thought about billing implications etc before downgrading!
+    conn = op.get_bind()
+    conn.execute(text("DELETE FROM letter_rates WHERE start_date = :start"), start=CHANGEOVER_DATE)
+    conn.execute(text("UPDATE letter_rates SET end_date = NULL WHERE end_date = :start"), start=CHANGEOVER_DATE)

requirements.in

@@ -1,32 +1,32 @@
 # Run `make freeze-requirements` to update requirements.txt
 # with package version changes made in requirements-app.txt
 
-cffi==1.14.5
+cffi==1.15.0
 celery[sqs]==5.2.3
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.14.0
-Flask-Migrate==2.7.0
+Flask-Migrate==3.1.0
 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
 Flask==1.1.2
 click-datetime==0.2
-eventlet==0.30.2 # pyup: ignore # 0.31 breaks Gunicorn
-gunicorn==20.1.0
-iso8601==0.1.14
+# Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
+git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64#egg=gunicorn[eventlet]==20.1.0
+iso8601==1.0.2
 itsdangerous==1.1.0
 jsonschema==3.2.0
 marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.0 # marshmallow v3 throws errors
 marshmallow==2.21.0 # pyup: <3 # v3 throws errors
-psycopg2-binary==2.8.6
+psycopg2-binary==2.9.3
 PyJWT==2.0.1
 SQLAlchemy==1.4.10
 strict-rfc3339==0.7
 rfc3987==1.3.8
 cachetools==4.2.1
 beautifulsoup4==4.9.3
-lxml==4.7.1
+lxml==4.8.0
 Werkzeug==2.0.2
 
-notifications-python-client==6.0.2
+notifications-python-client==6.3.0
 
 # PaaS
 awscli-cwlogs==1.4.6

requirements.txt

@@ -43,7 +43,7 @@ certifi==2021.10.8
     # via
     #   pyproj
     #   requests
-cffi==1.14.5
+cffi==1.15.0
     # via
     #   -r requirements.in
     #   bcrypt
@@ -67,14 +67,14 @@ click-repl==0.2.0
     # via celery
 colorama==0.4.3
     # via awscli
-dnspython==1.16.0
+dnspython==2.2.0
     # via eventlet
 docopt==0.6.2
     # via notifications-python-client
 docutils==0.15.2
     # via awscli
-eventlet==0.30.2
-    # via -r requirements.in
+eventlet==0.33.0
+    # via gunicorn
 flask==1.1.2
     # via
     #   -r requirements.in
@@ -88,7 +88,7 @@ flask-bcrypt==0.7.1
     # via -r requirements.in
 flask-marshmallow==0.14.0
     # via -r requirements.in
-flask-migrate==2.7.0
+flask-migrate==3.1.0
     # via -r requirements.in
 flask-redis==0.4.0
     # via notifications-utils
@@ -106,11 +106,11 @@ greenlet==1.1.2
     # via
     #   eventlet
     #   sqlalchemy
-gunicorn==20.1.0
+gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64
     # via -r requirements.in
 idna==3.3
     # via requests
-iso8601==0.1.14
+iso8601==1.0.2
     # via -r requirements.in
 itsdangerous==1.1.0
     # via
@@ -129,7 +129,7 @@ jsonschema==3.2.0
     # via -r requirements.in
 kombu==5.2.3
     # via celery
-lxml==4.7.1
+lxml==4.8.0
     # via -r requirements.in
 mako==1.1.5
     # via alembic
@@ -146,7 +146,7 @@ marshmallow-sqlalchemy==0.23.1
     # via -r requirements.in
 mistune==0.8.4
     # via notifications-utils
-notifications-python-client==6.0.2
+notifications-python-client==6.3.0
     # via -r requirements.in
 notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@53.0.0
     # via -r requirements.in
@@ -162,7 +162,7 @@ prometheus-client==0.10.1
     #   gds-metrics
 prompt-toolkit==3.0.21
     # via click-repl
-psycopg2-binary==2.8.6
+psycopg2-binary==2.9.3
     # via -r requirements.in
 pyasn1==0.4.8
     # via rsa

requirements_for_test.txt

@@ -1,14 +1,14 @@
 -r requirements.txt
-flake8==3.8.4
-flake8-bugbear==20.11.1
-isort==5.7.0
-moto==2.0.11
-pytest==6.1.2
+flake8==4.0.1
+flake8-bugbear==22.1.11
+isort==5.10.1
+moto==3.0.5
+pytest==7.0.1
 pytest-env==0.6.2
-pytest-mock==3.3.1
-pytest-cov==2.10.1
-pytest-xdist==2.1.0
+pytest-mock==3.7.0
+pytest-cov==3.0.0
+pytest-xdist==2.5.0
 freezegun==1.1.0
-requests-mock==1.8.0
+requests-mock==1.9.3
 # used for creating manifest file locally
-jinja2-cli[yaml]==0.7.0
+jinja2-cli[yaml]==0.8.1

scripts/run_with_docker.sh

@@ -9,11 +9,14 @@ source environment.sh
 AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"$(aws configure get aws_access_key_id)"}
 AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"$(aws configure get aws_secret_access_key)"}
 : "${SQLALCHEMY_DATABASE_URI:=postgresql://postgres@host.docker.internal/notification_api}"
+REDIS_URL="redis://host.docker.internal:6379"
 
 docker run -it --rm \
   -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
   -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
   -e SQLALCHEMY_DATABASE_URI=$SQLALCHEMY_DATABASE_URI \
+  -e REDIS_ENABLED=${REDIS_ENABLED:-0} \
+  -e REDIS_URL=$REDIS_URL \
   -v $(pwd):/home/vcap/app \
   ${DOCKER_IMAGE_NAME} \
   ${@}

tests/app/billing/test_billing.py

@@ -29,10 +29,6 @@ IN_MAY_2016 = datetime(2016, 5, 10, 23, 00, 00)
 IN_JUN_2016 = datetime(2016, 6, 3, 23, 00, 00)
 
 
-def _assert_dict_equals(actual, expected_dict):
-    assert actual == expected_dict
-
-
 def test_create_update_free_sms_fragment_limit_invalid_schema(client, sample_service):
 
     response = client.post('service/{}/billing/free-sms-fragment-limit'.format(sample_service.id),

tests/app/celery/test_reporting_tasks.py

@@ -605,10 +605,7 @@ def test_create_nightly_notification_status_for_service_and_day_overwrites_old_d
     notification = create_notification(template=first_template, status='sending')
     create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
 
-    new_fact_data = FactNotificationStatus.query.order_by(
-        FactNotificationStatus.bst_date,
-        FactNotificationStatus.notification_type
-    ).all()
+    new_fact_data = FactNotificationStatus.query.all()
 
     assert len(new_fact_data) == 1
     assert new_fact_data[0].notification_count == 1
@@ -620,8 +617,7 @@ def test_create_nightly_notification_status_for_service_and_day_overwrites_old_d
     create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
 
     updated_fact_data = FactNotificationStatus.query.order_by(
-        FactNotificationStatus.bst_date,
-        FactNotificationStatus.notification_type
+        FactNotificationStatus.notification_status
     ).all()
 
     assert len(updated_fact_data) == 2

tests/app/dao/test_ft_billing_dao.py

@@ -10,12 +10,14 @@ from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
     fetch_billing_totals_for_year,
+    fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
     fetch_monthly_billing_for_year,
     fetch_sms_billing_for_all_services,
     fetch_sms_free_allowance_remainder_until_date,
     fetch_usage_year_for_organisation,
+    fetch_volumes_by_service,
     get_rate,
     get_rates_for_billing,
 )
@@ -812,3 +814,90 @@ def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(n
     assert len(results) == 1
     assert results[str(live_service.id)]['sms_billable_units'] == 19
     assert results[str(live_service.id)]['emails_sent'] == 0
+
+
+def test_fetch_daily_volumes_for_platform(
+        notify_db_session, sample_template, sample_email_template, sample_letter_template
+):
+    create_ft_billing(bst_date='2022-02-03', template=sample_template,
+                      notifications_sent=10, billable_unit=10)
+    create_ft_billing(bst_date='2022-02-03', template=sample_template,
+                      notifications_sent=10, billable_unit=30, international=True)
+    create_ft_billing(bst_date='2022-02-03', template=sample_email_template, notifications_sent=10)
+    create_ft_billing(bst_date='2022-02-03', template=sample_letter_template, notifications_sent=5,
+                      billable_unit=5, rate=0.39)
+    create_ft_billing(bst_date='2022-02-03', template=sample_letter_template, notifications_sent=5,
+                      billable_unit=10, rate=0.44)
+
+    create_ft_billing(bst_date='2022-02-04', template=sample_template,
+                      notifications_sent=20, billable_unit=40)
+    create_ft_billing(bst_date='2022-02-04', template=sample_template,
+                      notifications_sent=10, billable_unit=20, rate_multiplier=3)
+    create_ft_billing(bst_date='2022-02-04', template=sample_email_template, notifications_sent=50)
+    create_ft_billing(bst_date='2022-02-04', template=sample_letter_template, notifications_sent=20, billable_unit=40)
+
+    results = fetch_daily_volumes_for_platform(start_date='2022-02-03', end_date='2022-02-04')
+
+    assert len(results) == 2
+    assert results[0].bst_date == '2022-02-03'
+    assert results[0].sms_totals == 20
+    assert results[0].sms_fragment_totals == 40
+    assert results[0].sms_chargeable_units == 40
+    assert results[0].email_totals == 10
+    assert results[0].letter_totals == 10
+    assert results[0].letter_sheet_totals == 15
+
+    assert results[1].bst_date == '2022-02-04'
+    assert results[1].sms_totals == 30
+    assert results[1].sms_fragment_totals == 60
+    assert results[1].sms_chargeable_units == 100
+    assert results[1].email_totals == 50
+    assert results[1].letter_totals == 20
+    assert results[1].letter_sheet_totals == 40
+
+
+def test_fetch_volumes_by_service(notify_db_session):
+    set_up_usage_data(datetime(2022, 2, 1))
+
+    results = fetch_volumes_by_service(start_date=datetime(2022, 2, 1), end_date=datetime(2022, 2, 28))
+
+    assert len(results) == 4
+    assert results[0].service_name == 'a - with sms and letter'
+    assert results[0].organisation_name == 'Org for a - with sms and letter'
+    assert results[0].free_allowance == 10
+    assert results[0].sms_notifications == 2
+    assert results[0].sms_chargeable_units == 3
+    assert results[0].email_totals == 0
+    assert results[0].letter_totals == 4
+    assert results[0].letter_sheet_totals == 6
+    assert float(results[0].letter_cost) == 1.6
+
+    assert results[1].service_name == 'f - without ft_billing'
+    assert results[1].organisation_name == 'Org for a - with sms and letter'
+    assert results[1].free_allowance == 10
+    assert results[1].sms_notifications == 0
+    assert results[1].sms_chargeable_units == 0
+    assert results[1].email_totals == 0
+    assert results[1].letter_totals == 0
+    assert results[1].letter_sheet_totals == 0
+    assert float(results[1].letter_cost) == 0
+
+    assert results[2].service_name == 'b - chargeable sms'
+    assert not results[2].organisation_name
+    assert results[2].free_allowance == 10
+    assert results[2].sms_notifications == 2
+    assert results[2].sms_chargeable_units == 3
+    assert results[2].email_totals == 0
+    assert results[2].letter_totals == 0
+    assert results[2].letter_sheet_totals == 0
+    assert float(results[2].letter_cost) == 0
+
+    assert results[3].service_name == 'e - sms within allowance'
+    assert not results[3].organisation_name
+    assert results[3].free_allowance == 10
+    assert results[3].sms_notifications == 1
+    assert results[3].sms_chargeable_units == 2
+    assert results[3].email_totals == 0
+    assert results[3].letter_totals == 0
+    assert results[3].letter_sheet_totals == 0
+    assert float(results[3].letter_cost) == 0

tests/app/db.py

@@ -260,7 +260,6 @@ def create_notification(
         rate_multiplier=None,
         international=False,
         phone_prefix=None,
-        scheduled_for=None,
         normalised_to=None,
         one_off=False,
         reply_to_text=None,
@@ -980,11 +979,11 @@ def set_up_usage_data(start_date):
     create_ft_billing(bst_date=two_days_later, template=sms_template_1, billable_unit=1, rate=0.11)
 
     create_ft_billing(bst_date=one_week_later, template=letter_template_1,
-                      notifications_sent=2, billable_unit=1, rate=.35, postage='first')
+                      notifications_sent=2, billable_unit=2, rate=.35, postage='first')
     create_ft_billing(bst_date=one_month_later, template=letter_template_1,
-                      notifications_sent=4, billable_unit=2, rate=.45, postage='second')
+                      notifications_sent=4, billable_unit=8, rate=.45, postage='second')
     create_ft_billing(bst_date=one_week_later, template=letter_template_1,
-                      notifications_sent=2, billable_unit=2, rate=.45, postage='second')
+                      notifications_sent=2, billable_unit=4, rate=.45, postage='second')
 
     # service with emails only:
     service_with_emails = create_service(service_name='b - emails')

tests/app/letters/test_returned_letters.py

@@ -20,4 +20,23 @@ def test_process_returned_letters(status, references, admin_request, mocker):
     if status != 200:
         assert '{} does not match'.format(references[0]) in response['errors'][0]['message']
     else:
-        mock_celery.assert_called_once_with([references], queue='database-tasks')
+        mock_celery.assert_called_once_with(args=(references,), queue='database-tasks', compression='zlib')
+
+
+def test_process_returned_letters_splits_tasks_up(admin_request, mocker):
+    mock_celery = mocker.patch("app.letters.rest.process_returned_letters_list.apply_async")
+    mocker.patch("app.letters.rest.MAX_REFERENCES_PER_TASK", 3)
+
+    references = [f'{x:016}' for x in range(10)]
+
+    admin_request.post(
+        'letter-job.create_process_returned_letters_job',
+        _data={"references": references},
+    )
+
+    assert mock_celery.call_count == 4
+
+    assert mock_celery.call_args_list[0][1]['args'][0] == ['0000000000000000', '0000000000000001', '0000000000000002']
+    assert mock_celery.call_args_list[1][1]['args'][0] == ['0000000000000003', '0000000000000004', '0000000000000005']
+    assert mock_celery.call_args_list[2][1]['args'][0] == ['0000000000000006', '0000000000000007', '0000000000000008']
+    assert mock_celery.call_args_list[3][1]['args'][0] == ['0000000000000009']

tests/app/platform_stats/test_rest.py

@@ -183,3 +183,58 @@ def test_get_data_for_billing_report(notify_db_session, admin_request):
         "2 second class letters at 35p\n1 first class letters at 50p\n15 international letters at £1.55\n"
     )
     assert response[3]["purchase_order_number"] is None
+
+
+def test_daily_volumes_report(
+        notify_db_session, sample_template, sample_email_template, sample_letter_template, admin_request
+):
+    set_up_usage_data(datetime(2022, 3, 1))
+    response = admin_request.get(
+        "platform_stats.daily_volumes_report",
+        start_date='2022-03-01',
+        end_date='2022-03-31'
+    )
+
+    assert len(response) == 3
+    assert response[0] == {'day': '2022-03-01', 'email_totals': 10, 'letter_sheet_totals': 3,
+                           'letter_totals': 2, 'sms_chargeable_units': 2, 'sms_fragment_totals': 2, 'sms_totals': 1}
+    assert response[1] == {'day': '2022-03-03', 'email_totals': 0, 'letter_sheet_totals': 10, 'letter_totals': 18,
+                           'sms_chargeable_units': 2, 'sms_fragment_totals': 2, 'sms_totals': 2}
+    assert response[2] == {'day': '2022-03-08', 'email_totals': 0, 'letter_sheet_totals': 11, 'letter_totals': 12,
+                           'sms_chargeable_units': 4, 'sms_fragment_totals': 4, 'sms_totals': 2}
+
+
+def test_volumes_by_service_report(
+        notify_db_session, sample_template, sample_email_template, sample_letter_template, admin_request
+):
+    fixture = set_up_usage_data(datetime(2022, 3, 1))
+    response = admin_request.get(
+        "platform_stats.volumes_by_service_report",
+        start_date='2022-03-01',
+        end_date='2022-03-01'
+    )
+
+    assert len(response) == 4
+    assert response[0] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0,
+                           'letter_sheet_totals': 0, 'letter_totals': 0,
+                           'organisation_id': str(fixture['org_1'].id),
+                           'organisation_name': fixture['org_1'].name,
+                           'service_id': str(fixture['service_1_sms_and_letter'].id),
+                           'service_name': fixture['service_1_sms_and_letter'].name,
+                           'sms_chargeable_units': 2, 'sms_notifications': 1}
+    assert response[1] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
+                           'letter_totals': 0, 'organisation_id': str(fixture['org_1'].id),
+                           'organisation_name': fixture['org_1'].name,
+                           'service_id': str(fixture['service_with_out_ft_billing_this_year'].id),
+                           'service_name': fixture['service_with_out_ft_billing_this_year'].name,
+                           'sms_chargeable_units': 0, 'sms_notifications': 0}
+    assert response[2] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
+                           'letter_totals': 0, 'organisation_id': '', 'organisation_name': '',
+                           'service_id': str(fixture['service_with_sms_without_org'].id),
+                           'service_name': fixture['service_with_sms_without_org'].name,
+                           'sms_chargeable_units': 0, 'sms_notifications': 0}
+    assert response[3] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
+                           'letter_totals': 0, 'organisation_id': '', 'organisation_name': '',
+                           'service_id': str(fixture['service_with_sms_within_allowance'].id),
+                           'service_name': fixture['service_with_sms_within_allowance'].name,
+                           'sms_chargeable_units': 0, 'sms_notifications': 0}

tests/app/test_commands.py

@@ -1,8 +1,23 @@
-from app.commands import local_dev_broadcast_permissions
+from app.commands import (
+    insert_inbound_numbers_from_file,
+    local_dev_broadcast_permissions,
+)
+from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers
 from app.dao.services_dao import dao_add_user_to_service
 from tests.app.db import create_user
 
 
+def test_insert_inbound_numbers_from_file(notify_db_session, notify_api, tmpdir):
+    numbers_file = tmpdir.join("numbers.txt")
+    numbers_file.write("07700900373\n07700900473\n07700900375\n\n\n\n")
+
+    notify_api.test_cli_runner().invoke(insert_inbound_numbers_from_file, ['-f', numbers_file])
+
+    inbound_numbers = dao_get_available_inbound_numbers()
+    assert len(inbound_numbers) == 3
+    assert set(x.number for x in inbound_numbers) == {'07700900373', '07700900473', '07700900375'}
+
+
 def test_local_dev_broadcast_permissions(
     sample_service,
     sample_broadcast_service,

tests/app/user/test_rest_verify.py

@@ -292,15 +292,29 @@ def test_send_sms_code_returns_204_when_too_many_codes_already_created(client, s
     assert VerifyCode.query.count() == 5
 
 
-def test_send_new_user_email_verification(client,
-                                          sample_user,
-                                          mocker,
-                                          email_verification_template):
+@pytest.mark.parametrize('post_data, expected_url_starts_with', (
+    (
+        {},
+        'http://localhost',
+    ),
+    (
+        {'admin_base_url': 'https://example.com'},
+        'https://example.com',
+    ),
+))
+def test_send_new_user_email_verification(
+    client,
+    sample_user,
+    mocker,
+    email_verification_template,
+    post_data,
+    expected_url_starts_with,
+):
     mocked = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
     auth_header = create_admin_authorization_header()
     resp = client.post(
         url_for('user.send_new_user_email_verification', user_id=str(sample_user.id)),
-        data=json.dumps({}),
+        data=json.dumps(post_data),
         headers=[('Content-Type', 'application/json'), auth_header])
     notify_service = email_verification_template.service
     assert resp.status_code == 204
@@ -308,6 +322,8 @@ def test_send_new_user_email_verification(client,
     assert VerifyCode.query.count() == 0
     mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks")
     assert notification.reply_to_text == notify_service.get_default_reply_to_email_address()
+    assert notification.personalisation['name'] == 'Test User'
+    assert notification.personalisation['url'].startswith(expected_url_starts_with)
 
 
 def test_send_email_verification_returns_404_for_bad_input_data(client, notify_db_session, mocker):

tests/app/v2/notifications/test_get_notifications.py

@@ -20,7 +20,6 @@ def test_get_notification_by_id_returns_200(
         template=sample_template,
         billable_units=billable_units,
         sent_by=provider,
-        scheduled_for="2017-05-12 15:15"
     )
 
     # another
@@ -28,7 +27,6 @@ def test_get_notification_by_id_returns_200(
         template=sample_template,
         billable_units=billable_units,
         sent_by=provider,
-        scheduled_for="2017-06-12 15:15"
     )
 
     auth_header = create_service_authorization_header(service_id=sample_notification.service_id)