bump celery

Makefile

@@ -27,10 +27,6 @@ bootstrap: generate-version-file ## Set up everything to run the app
 	createdb notification_api || true
 	(. environment.sh && flask db upgrade) || true
 
-.PHONY: bootstrap-with-docker
-bootstrap-with-docker: ## Build the image to run the app in Docker
-	docker build -f docker/Dockerfile -t notifications-api .
-
 .PHONY: run-flask
 run-flask: ## Run flask
 	. environment.sh && flask run -p 6011
@@ -43,20 +39,12 @@ run-celery: ## Run celery
 		--loglevel=INFO \
 		--concurrency=4
 
-.PHONY: run-celery-with-docker
-run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery
-
 .PHONY: run-celery-beat
 run-celery-beat: ## Run celery beat
 	. environment.sh && celery \
 		-A run_celery.notify_celery beat \
 		--loglevel=INFO
 
-.PHONY: run-celery-beat-with-docker
-run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery-beat
-
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -189,3 +177,8 @@ disable-failwhale: ## Disable the failwhale app and enable api
 	cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api
 	cf stop notify-api-failwhale
 	@echo "Failwhale is disabled"
+
+.PHONY: run-celery-with-docker
+run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
+	docker build -f docker/Dockerfile -t notifications-api .
+	./scripts/run_with_docker.sh make run-celery

README.md

@@ -61,19 +61,10 @@ export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/
 
 ### Redis
 
-To switch redis on you'll need to install it locally. On a Mac you can do:
+To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:
 
-```
-# assuming you use Homebrew
-brew install redis
-brew services start redis
-```
+        REDIS_ENABLED = True
 
-To use redis caching you need to switch it on with an environment variable:
-
-```
-export REDIS_ENABLED=1
-```
 
 ##  To run the application
 
@@ -91,19 +82,6 @@ make run-celery
 make run-celery-beat
 ```
 
-We've had problems running Celery locally due to one of its dependencies: pycurl. Due to the complexity of the issue, we also support running Celery via Docker:
-
-```
-# install dependencies, etc.
-make bootstrap-with-docker
-
-# run the background tasks
-make run-celery-with-docker
-
-# run scheduled tasks
-make run-celery-beat-with-docker
-```
-
 ##  To test the application
 
 ```

app/billing/billing_schemas.py

@@ -6,7 +6,7 @@ create_or_update_free_sms_fragment_limit_schema = {
     "type": "object",
     "title": "Create",
     "properties": {
-        "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
+        "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
     },
     "required": ["free_sms_fragment_limit"]
 }

app/celery/nightly_tasks.py

@@ -255,7 +255,7 @@ def letter_raise_alert_if_no_ack_file_for_zip():
 
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                               subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))  # noqa
+        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))
 
     message = '\n'.join([
         "Letter ack file does not contain all zip files sent."

app/celery/service_callback_tasks.py

@@ -68,7 +68,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                 'Content-Type': 'application/json',
                 'Authorization': 'Bearer {}'.format(token)
             },
-            timeout=5
+            timeout=60
         )
         current_app.logger.info('{} sending {} to {}, response {}'.format(
             function_name,

app/commands.py

@@ -246,15 +246,14 @@ def fix_notification_statuses_not_in_sync():
               one number per line. The number must have the format of 07... not 447....""")
 def insert_inbound_numbers_from_file(file_name):
     print("Inserting inbound numbers from {}".format(file_name))
-    with open(file_name) as file:
-        sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
+    file = open(file_name)
+    sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
 
-        for line in file:
-            line = line.strip()
-            if line:
-                print(line)
-                db.session.execute(sql.format(uuid.uuid4(), line))
-                db.session.commit()
+    for line in file:
+        print(line)
+        db.session.execute(sql.format(uuid.uuid4(), line.strip()))
+        db.session.commit()
+    file.close()
 
 
 @notify_command(name='replay-create-pdf-for-templated-letter')

app/config.py

@@ -307,7 +307,7 @@ class Config(object):
             },
             'raise-alert-if-letter-notifications-still-sending': {
                 'task': 'raise-alert-if-letter-notifications-still-sending',
-                'schedule': crontab(hour=17, minute=00),
+                'schedule': crontab(hour=15, minute=30),
                 'options': {'queue': QueueNames.PERIODIC}
             },
             # The collate-letter-pdf does assume it is called in an hour that BST does not make a
@@ -428,7 +428,7 @@ class Development(Config):
     NOTIFY_EMAIL_DOMAIN = "notify.tools"
 
     SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api')
-    REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
+    REDIS_URL = 'redis://localhost:6379/0'
 
     ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
 

app/dao/fact_billing_dao.py

@@ -716,151 +716,3 @@ def fetch_billing_details_for_all_services():
     ).all()
 
     return billing_details
-
-
-def fetch_daily_volumes_for_platform(start_date, end_date):
-    # query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
-
-    daily_volume_stats = db.session.query(
-        FactBilling.bst_date,
-        func.sum(case(
-            [
-                (FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
-            ], else_=0
-        )).label('sms_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units)
-            ], else_=0
-        )).label('sms_fragment_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units * FactBilling.rate_multiplier)
-            ], else_=0
-        )).label('sms_fragments_times_multiplier'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
-            ], else_=0
-        )).label('email_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
-            ], else_=0
-        )).label('letter_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
-            ], else_=0
-        )).label('letter_sheet_totals')
-    ).filter(
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date
-    ).group_by(
-        FactBilling.bst_date,
-        FactBilling.notification_type
-    ).subquery()
-
-    aggregated_totals = db.session.query(
-        daily_volume_stats.c.bst_date.cast(db.Text).label('bst_date'),
-        func.sum(daily_volume_stats.c.sms_totals).label('sms_totals'),
-        func.sum(daily_volume_stats.c.sms_fragment_totals).label('sms_fragment_totals'),
-        func.sum(
-            daily_volume_stats.c.sms_fragments_times_multiplier).label('sms_chargeable_units'),
-        func.sum(daily_volume_stats.c.email_totals).label('email_totals'),
-        func.sum(daily_volume_stats.c.letter_totals).label('letter_totals'),
-        func.sum(daily_volume_stats.c.letter_sheet_totals).label('letter_sheet_totals')
-    ).group_by(
-        daily_volume_stats.c.bst_date
-    ).order_by(
-        daily_volume_stats.c.bst_date
-    ).all()
-
-    return aggregated_totals
-
-
-def fetch_volumes_by_service(start_date, end_date):
-    # query to return the volume totals by service aggregated for the date range given
-    # start and end dates are inclusive.
-    year_end_date = int(end_date.strftime('%Y'))
-
-    volume_stats = db.session.query(
-        FactBilling.bst_date,
-        FactBilling.service_id,
-        func.sum(case([
-            (FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
-        ], else_=0)).label('sms_totals'),
-        func.sum(case([
-            (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units * FactBilling.rate_multiplier)
-        ], else_=0)).label('sms_fragments_times_multiplier'),
-        func.sum(case([
-            (FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
-        ], else_=0)).label('email_totals'),
-        func.sum(case([
-            (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
-        ], else_=0)).label('letter_totals'),
-        func.sum(case([
-            (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent * FactBilling.rate)
-        ], else_=0)).label("letter_cost"),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
-            ], else_=0
-        )).label('letter_sheet_totals')
-    ).filter(
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date
-    ).group_by(
-        FactBilling.bst_date,
-        FactBilling.service_id,
-        FactBilling.notification_type
-    ).subquery()
-
-    annual_billing = db.session.query(
-        func.max(AnnualBilling.financial_year_start).label('financial_year_start'),
-        AnnualBilling.service_id,
-        AnnualBilling.free_sms_fragment_limit
-    ).filter(
-        AnnualBilling.financial_year_start <= year_end_date
-    ).group_by(
-        AnnualBilling.service_id,
-        AnnualBilling.free_sms_fragment_limit
-    ).subquery()
-
-    results = db.session.query(
-        Service.name.label("service_name"),
-        Service.id.label("service_id"),
-        Service.organisation_id.label("organisation_id"),
-        Organisation.name.label("organisation_name"),
-        annual_billing.c.free_sms_fragment_limit.label("free_allowance"),
-        func.coalesce(func.sum(volume_stats.c.sms_totals), 0).label("sms_notifications"),
-        func.coalesce(func.sum(volume_stats.c.sms_fragments_times_multiplier), 0
-                      ).label("sms_chargeable_units"),
-        func.coalesce(func.sum(volume_stats.c.email_totals), 0).label("email_totals"),
-        func.coalesce(func.sum(volume_stats.c.letter_totals), 0).label("letter_totals"),
-        func.coalesce(func.sum(volume_stats.c.letter_cost), 0).label("letter_cost"),
-        func.coalesce(func.sum(volume_stats.c.letter_sheet_totals), 0).label("letter_sheet_totals")
-    ).select_from(
-        Service
-    ).outerjoin(
-        Organisation, Service.organisation_id == Organisation.id
-    ).join(
-        annual_billing, Service.id == annual_billing.c.service_id
-    ).outerjoin(  # include services without volume
-        volume_stats, Service.id == volume_stats.c.service_id
-    ).filter(
-        Service.restricted.is_(False),
-        Service.count_as_live.is_(True),
-        Service.active.is_(True)
-    ).group_by(
-        Service.id,
-        Service.name,
-        Service.organisation_id,
-        Organisation.name,
-        annual_billing.c.free_sms_fragment_limit
-    ).order_by(
-        Organisation.name,
-        Service.name,
-    ).all()
-
-    return results

app/dao/services_dao.py

@@ -544,8 +544,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
         Notification.notification_type == SMS_TYPE,
         func.substr(Notification.normalised_to, 3, 7) == '7700900',
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -553,7 +553,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
     ).all()
 
 
-def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
+def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=1000):
     subquery = db.session.query(
         func.count(Notification.id).label('total_count'),
         Notification.service_id.label('service_id')
@@ -564,8 +564,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.key_type != KEY_TYPE_TEST,
         Notification.notification_type == SMS_TYPE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -590,8 +590,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.notification_type == SMS_TYPE,
         Notification.status == NOTIFICATION_PERMANENT_FAILURE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
         subquery.c.total_count

app/dao/uploads_dao.py

@@ -84,7 +84,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Notification.notification_type == LETTER_TYPE,
         Notification.api_key_id == None,  # noqa
         Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden == True,  # noqa
+        Template.hidden == True,
         Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
     ]
     if limit_days is not None:

app/letters/rest.py

@@ -9,19 +9,11 @@ from app.v2.errors import register_errors
 letter_job = Blueprint("letter-job", __name__)
 register_errors(letter_job)
 
-# too many references will make SQS error (as the task can only be 256kb)
-MAX_REFERENCES_PER_TASK = 5000
-
 
 @letter_job.route('/letters/returned', methods=['POST'])
 def create_process_returned_letters_job():
-    references = validate(request.get_json(), letter_references)['references']
+    references = validate(request.get_json(), letter_references)
 
-    for start_index in range(0, len(references), MAX_REFERENCES_PER_TASK):
-        process_returned_letters_list.apply_async(
-            args=(references[start_index:start_index + MAX_REFERENCES_PER_TASK], ),
-            queue=QueueNames.DATABASE,
-            compression='zlib'
-        )
+    process_returned_letters_list.apply_async([references['references']], queue=QueueNames.DATABASE)
 
-    return jsonify(references=references), 200
+    return jsonify(references=references['references']), 200

app/platform_stats/rest.py

@@ -5,11 +5,9 @@ from flask import Blueprint, jsonify, request
 from app.dao.date_util import get_financial_year_for_datetime
 from app.dao.fact_billing_dao import (
     fetch_billing_details_for_all_services,
-    fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
     fetch_sms_billing_for_all_services,
-    fetch_volumes_by_service,
 )
 from app.dao.fact_notification_status_dao import (
     fetch_notification_status_totals_for_all_services,
@@ -42,17 +40,12 @@ def get_platform_stats():
     return jsonify(stats)
 
 
-def validate_date_format(date_to_validate):
+def validate_date_range_is_within_a_financial_year(start_date, end_date):
     try:
-        validated_date = datetime.strptime(date_to_validate, "%Y-%m-%d").date()
+        start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end_date = datetime.strptime(end_date, "%Y-%m-%d").date()
     except ValueError:
         raise InvalidRequest(message="Input must be a date in the format: YYYY-MM-DD", status_code=400)
-    return validated_date
-
-
-def validate_date_range_is_within_a_financial_year(start_date, end_date):
-    start_date = validate_date_format(start_date)
-    end_date = validate_date_format(end_date)
     if end_date < start_date:
         raise InvalidRequest(message="Start date must be before end date", status_code=400)
 
@@ -140,53 +133,6 @@ def get_data_for_billing_report():
     return jsonify(result)
 
 
-@platform_stats_blueprint.route('daily-volumes-report')
-def daily_volumes_report():
-    start_date = validate_date_format(request.args.get('start_date'))
-    end_date = validate_date_format(request.args.get('end_date'))
-
-    daily_volumes = fetch_daily_volumes_for_platform(start_date, end_date)
-    report = []
-
-    for row in daily_volumes:
-        report.append({
-            "day": row.bst_date,
-            "sms_totals": int(row.sms_totals),
-            "sms_fragment_totals": int(row.sms_fragment_totals),
-            "sms_chargeable_units": int(row.sms_chargeable_units),
-            "email_totals": int(row.email_totals),
-            "letter_totals": int(row.letter_totals),
-            "letter_sheet_totals": int(row.letter_sheet_totals)
-        })
-    return jsonify(report)
-
-
-@platform_stats_blueprint.route('volumes-by-service')
-def volumes_by_service_report():
-    start_date = validate_date_format(request.args.get('start_date'))
-    end_date = validate_date_format(request.args.get('end_date'))
-
-    volumes_by_service = fetch_volumes_by_service(start_date, end_date)
-    report = []
-
-    for row in volumes_by_service:
-        report.append({
-            "service_name": row.service_name,
-            "service_id": str(row.service_id),
-            "organisation_name": row.organisation_name if row.organisation_name else '',
-            "organisation_id": str(row.organisation_id) if row.organisation_id else '',
-            "free_allowance": int(row.free_allowance),
-            "sms_notifications": int(row.sms_notifications),
-            "sms_chargeable_units": int(row.sms_chargeable_units),
-            "email_totals": int(row.email_totals),
-            "letter_totals": int(row.letter_totals),
-            "letter_sheet_totals": int(row.letter_sheet_totals),
-            "letter_cost": float(row.letter_cost),
-        })
-
-    return jsonify(report)
-
-
 def postage_description(postage):
     if postage in UK_POSTAGE_TYPES:
         return f'{postage} class'

app/user/rest.py

@@ -374,8 +374,6 @@ def send_user_confirm_new_email(user_id):
 
 @user_blueprint.route('/<uuid:user_id>/email-verification', methods=['POST'])
 def send_new_user_email_verification(user_id):
-    request_json = request.get_json()
-
     # when registering, we verify all users' email addresses using this function
     user_to_send_to = get_user_by_id(user_id=user_id)
 
@@ -389,10 +387,7 @@ def send_new_user_email_verification(user_id):
         service=service,
         personalisation={
             'name': user_to_send_to.name,
-            'url': _create_verification_url(
-                user_to_send_to,
-                base_url=request_json.get('admin_base_url'),
-            ),
+            'url': _create_verification_url(user_to_send_to)
         },
         notification_type=template.template_type,
         api_key_id=None,
@@ -561,10 +556,10 @@ def _create_reset_password_url(email, next_redirect, base_url=None):
     return full_url
 
 
-def _create_verification_url(user, base_url):
+def _create_verification_url(user):
     data = json.dumps({'user_id': str(user.id), 'email': user.email_address})
     url = '/verify-email/'
-    return url_with_token(data, url, current_app.config, base_url=base_url)
+    return url_with_token(data, url, current_app.config)
 
 
 def _create_confirmation_url(user, email_address):

migrations/versions/0366_letter_rates_2022.py

@@ -1,63 +0,0 @@
-"""
-
-Revision ID: 0366_letter_rates_2022
-Revises: 0365_add_nhs_branding
-Create Date: 2022-03-01 14:00:00
-
-"""
-import itertools
-import uuid
-from datetime import datetime
-
-from alembic import op
-from sqlalchemy.sql import text
-
-from app.models import LetterRate
-
-
-revision = '0366_letter_rates_2022'
-down_revision = '0365_add_nhs_branding'
-
-
-CHANGEOVER_DATE = datetime(2022, 3, 1, 0, 0)
-
-
-def get_new_rate(sheet_count, post_class):
-    base_prices = {
-        'second': 36,
-        'first': 58,
-        'europe': 88,
-        'rest-of-world': 88,
-    }
-    multiplier = 5 if post_class in ('first', 'second') else 8
-
-    return (base_prices[post_class] + (multiplier * sheet_count)) / 100.0
-
-
-def upgrade():
-    conn = op.get_bind()
-    conn.execute(text("UPDATE letter_rates SET end_date = :start WHERE end_date IS NULL"), start=CHANGEOVER_DATE)
-
-    op.bulk_insert(LetterRate.__table__, [
-        {
-            'id': uuid.uuid4(),
-            'start_date': CHANGEOVER_DATE,
-            'end_date': None,
-            'sheet_count': sheet_count,
-            'rate': get_new_rate(sheet_count, post_class),
-            'crown': crown,
-            'post_class': post_class,
-        }
-        for sheet_count, crown, post_class in itertools.product(
-            range(1, 6),
-            [True, False],
-            ['first', 'second', 'europe', 'rest-of-world']
-        )
-    ])
-
-
-def downgrade():
-    # Make sure you've thought about billing implications etc before downgrading!
-    conn = op.get_bind()
-    conn.execute(text("DELETE FROM letter_rates WHERE start_date = :start"), start=CHANGEOVER_DATE)
-    conn.execute(text("UPDATE letter_rates SET end_date = NULL WHERE end_date = :start"), start=CHANGEOVER_DATE)

requirements.in

@@ -1,32 +1,32 @@
 # Run `make freeze-requirements` to update requirements.txt
 # with package version changes made in requirements-app.txt
 
-cffi==1.15.0
+cffi==1.14.5
 celery[sqs]==5.2.3
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.14.0
-Flask-Migrate==3.1.0
+Flask-Migrate==2.7.0
 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
 Flask==1.1.2
 click-datetime==0.2
-# Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
-git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64#egg=gunicorn[eventlet]==20.1.0
-iso8601==1.0.2
+eventlet==0.30.2 # pyup: ignore # 0.31 breaks Gunicorn
+gunicorn==20.1.0
+iso8601==0.1.14
 itsdangerous==1.1.0
 jsonschema==3.2.0
 marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.0 # marshmallow v3 throws errors
 marshmallow==2.21.0 # pyup: <3 # v3 throws errors
-psycopg2-binary==2.9.3
+psycopg2-binary==2.8.6
 PyJWT==2.0.1
 SQLAlchemy==1.4.10
 strict-rfc3339==0.7
 rfc3987==1.3.8
 cachetools==4.2.1
 beautifulsoup4==4.9.3
-lxml==4.8.0
+lxml==4.7.1
 Werkzeug==2.0.2
 
-notifications-python-client==6.3.0
+notifications-python-client==6.0.2
 
 # PaaS
 awscli-cwlogs==1.4.6

requirements.txt

@@ -43,7 +43,7 @@ certifi==2021.10.8
     # via
     #   pyproj
     #   requests
-cffi==1.15.0
+cffi==1.14.5
     # via
     #   -r requirements.in
     #   bcrypt
@@ -67,14 +67,14 @@ click-repl==0.2.0
     # via celery
 colorama==0.4.3
     # via awscli
-dnspython==2.2.0
+dnspython==1.16.0
     # via eventlet
 docopt==0.6.2
     # via notifications-python-client
 docutils==0.15.2
     # via awscli
-eventlet==0.33.0
-    # via gunicorn
+eventlet==0.30.2
+    # via -r requirements.in
 flask==1.1.2
     # via
     #   -r requirements.in
@@ -88,7 +88,7 @@ flask-bcrypt==0.7.1
     # via -r requirements.in
 flask-marshmallow==0.14.0
     # via -r requirements.in
-flask-migrate==3.1.0
+flask-migrate==2.7.0
     # via -r requirements.in
 flask-redis==0.4.0
     # via notifications-utils
@@ -106,11 +106,11 @@ greenlet==1.1.2
     # via
     #   eventlet
     #   sqlalchemy
-gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64
+gunicorn==20.1.0
     # via -r requirements.in
 idna==3.3
     # via requests
-iso8601==1.0.2
+iso8601==0.1.14
     # via -r requirements.in
 itsdangerous==1.1.0
     # via
@@ -129,7 +129,7 @@ jsonschema==3.2.0
     # via -r requirements.in
 kombu==5.2.3
     # via celery
-lxml==4.8.0
+lxml==4.7.1
     # via -r requirements.in
 mako==1.1.5
     # via alembic
@@ -146,7 +146,7 @@ marshmallow-sqlalchemy==0.23.1
     # via -r requirements.in
 mistune==0.8.4
     # via notifications-utils
-notifications-python-client==6.3.0
+notifications-python-client==6.0.2
     # via -r requirements.in
 notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@53.0.0
     # via -r requirements.in
@@ -162,7 +162,7 @@ prometheus-client==0.10.1
     #   gds-metrics
 prompt-toolkit==3.0.21
     # via click-repl
-psycopg2-binary==2.9.3
+psycopg2-binary==2.8.6
     # via -r requirements.in
 pyasn1==0.4.8
     # via rsa

requirements_for_test.txt

@@ -1,14 +1,14 @@
 -r requirements.txt
-flake8==4.0.1
-flake8-bugbear==22.1.11
-isort==5.10.1
-moto==3.0.5
-pytest==7.0.1
+flake8==3.8.4
+flake8-bugbear==20.11.1
+isort==5.7.0
+moto==2.0.11
+pytest==6.1.2
 pytest-env==0.6.2
-pytest-mock==3.7.0
-pytest-cov==3.0.0
-pytest-xdist==2.5.0
+pytest-mock==3.3.1
+pytest-cov==2.10.1
+pytest-xdist==2.1.0
 freezegun==1.1.0
-requests-mock==1.9.3
+requests-mock==1.8.0
 # used for creating manifest file locally
-jinja2-cli[yaml]==0.8.1
+jinja2-cli[yaml]==0.7.0

scripts/run_with_docker.sh

@@ -9,14 +9,11 @@ source environment.sh
 AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"$(aws configure get aws_access_key_id)"}
 AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"$(aws configure get aws_secret_access_key)"}
 : "${SQLALCHEMY_DATABASE_URI:=postgresql://postgres@host.docker.internal/notification_api}"
-REDIS_URL="redis://host.docker.internal:6379"
 
 docker run -it --rm \
   -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
   -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
   -e SQLALCHEMY_DATABASE_URI=$SQLALCHEMY_DATABASE_URI \
-  -e REDIS_ENABLED=${REDIS_ENABLED:-0} \
-  -e REDIS_URL=$REDIS_URL \
   -v $(pwd):/home/vcap/app \
   ${DOCKER_IMAGE_NAME} \
   ${@}

tests/app/billing/test_billing.py

@@ -29,6 +29,10 @@ IN_MAY_2016 = datetime(2016, 5, 10, 23, 00, 00)
 IN_JUN_2016 = datetime(2016, 6, 3, 23, 00, 00)
 
 
+def _assert_dict_equals(actual, expected_dict):
+    assert actual == expected_dict
+
+
 def test_create_update_free_sms_fragment_limit_invalid_schema(client, sample_service):
 
     response = client.post('service/{}/billing/free-sms-fragment-limit'.format(sample_service.id),

tests/app/celery/test_reporting_tasks.py

@@ -605,7 +605,10 @@ def test_create_nightly_notification_status_for_service_and_day_overwrites_old_d
     notification = create_notification(template=first_template, status='sending')
     create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
 
-    new_fact_data = FactNotificationStatus.query.all()
+    new_fact_data = FactNotificationStatus.query.order_by(
+        FactNotificationStatus.bst_date,
+        FactNotificationStatus.notification_type
+    ).all()
 
     assert len(new_fact_data) == 1
     assert new_fact_data[0].notification_count == 1
@@ -617,7 +620,8 @@ def test_create_nightly_notification_status_for_service_and_day_overwrites_old_d
     create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
 
     updated_fact_data = FactNotificationStatus.query.order_by(
-        FactNotificationStatus.notification_status
+        FactNotificationStatus.bst_date,
+        FactNotificationStatus.notification_type
     ).all()
 
     assert len(updated_fact_data) == 2

tests/app/dao/test_ft_billing_dao.py

@@ -10,14 +10,12 @@ from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
     fetch_billing_totals_for_year,
-    fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
     fetch_monthly_billing_for_year,
     fetch_sms_billing_for_all_services,
     fetch_sms_free_allowance_remainder_until_date,
     fetch_usage_year_for_organisation,
-    fetch_volumes_by_service,
     get_rate,
     get_rates_for_billing,
 )
@@ -814,90 +812,3 @@ def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(n
     assert len(results) == 1
     assert results[str(live_service.id)]['sms_billable_units'] == 19
     assert results[str(live_service.id)]['emails_sent'] == 0
-
-
-def test_fetch_daily_volumes_for_platform(
-        notify_db_session, sample_template, sample_email_template, sample_letter_template
-):
-    create_ft_billing(bst_date='2022-02-03', template=sample_template,
-                      notifications_sent=10, billable_unit=10)
-    create_ft_billing(bst_date='2022-02-03', template=sample_template,
-                      notifications_sent=10, billable_unit=30, international=True)
-    create_ft_billing(bst_date='2022-02-03', template=sample_email_template, notifications_sent=10)
-    create_ft_billing(bst_date='2022-02-03', template=sample_letter_template, notifications_sent=5,
-                      billable_unit=5, rate=0.39)
-    create_ft_billing(bst_date='2022-02-03', template=sample_letter_template, notifications_sent=5,
-                      billable_unit=10, rate=0.44)
-
-    create_ft_billing(bst_date='2022-02-04', template=sample_template,
-                      notifications_sent=20, billable_unit=40)
-    create_ft_billing(bst_date='2022-02-04', template=sample_template,
-                      notifications_sent=10, billable_unit=20, rate_multiplier=3)
-    create_ft_billing(bst_date='2022-02-04', template=sample_email_template, notifications_sent=50)
-    create_ft_billing(bst_date='2022-02-04', template=sample_letter_template, notifications_sent=20, billable_unit=40)
-
-    results = fetch_daily_volumes_for_platform(start_date='2022-02-03', end_date='2022-02-04')
-
-    assert len(results) == 2
-    assert results[0].bst_date == '2022-02-03'
-    assert results[0].sms_totals == 20
-    assert results[0].sms_fragment_totals == 40
-    assert results[0].sms_chargeable_units == 40
-    assert results[0].email_totals == 10
-    assert results[0].letter_totals == 10
-    assert results[0].letter_sheet_totals == 15
-
-    assert results[1].bst_date == '2022-02-04'
-    assert results[1].sms_totals == 30
-    assert results[1].sms_fragment_totals == 60
-    assert results[1].sms_chargeable_units == 100
-    assert results[1].email_totals == 50
-    assert results[1].letter_totals == 20
-    assert results[1].letter_sheet_totals == 40
-
-
-def test_fetch_volumes_by_service(notify_db_session):
-    set_up_usage_data(datetime(2022, 2, 1))
-
-    results = fetch_volumes_by_service(start_date=datetime(2022, 2, 1), end_date=datetime(2022, 2, 28))
-
-    assert len(results) == 4
-    assert results[0].service_name == 'a - with sms and letter'
-    assert results[0].organisation_name == 'Org for a - with sms and letter'
-    assert results[0].free_allowance == 10
-    assert results[0].sms_notifications == 2
-    assert results[0].sms_chargeable_units == 3
-    assert results[0].email_totals == 0
-    assert results[0].letter_totals == 4
-    assert results[0].letter_sheet_totals == 6
-    assert float(results[0].letter_cost) == 1.6
-
-    assert results[1].service_name == 'f - without ft_billing'
-    assert results[1].organisation_name == 'Org for a - with sms and letter'
-    assert results[1].free_allowance == 10
-    assert results[1].sms_notifications == 0
-    assert results[1].sms_chargeable_units == 0
-    assert results[1].email_totals == 0
-    assert results[1].letter_totals == 0
-    assert results[1].letter_sheet_totals == 0
-    assert float(results[1].letter_cost) == 0
-
-    assert results[2].service_name == 'b - chargeable sms'
-    assert not results[2].organisation_name
-    assert results[2].free_allowance == 10
-    assert results[2].sms_notifications == 2
-    assert results[2].sms_chargeable_units == 3
-    assert results[2].email_totals == 0
-    assert results[2].letter_totals == 0
-    assert results[2].letter_sheet_totals == 0
-    assert float(results[2].letter_cost) == 0
-
-    assert results[3].service_name == 'e - sms within allowance'
-    assert not results[3].organisation_name
-    assert results[3].free_allowance == 10
-    assert results[3].sms_notifications == 1
-    assert results[3].sms_chargeable_units == 2
-    assert results[3].email_totals == 0
-    assert results[3].letter_totals == 0
-    assert results[3].letter_sheet_totals == 0
-    assert float(results[3].letter_cost) == 0

tests/app/db.py

@@ -260,6 +260,7 @@ def create_notification(
         rate_multiplier=None,
         international=False,
         phone_prefix=None,
+        scheduled_for=None,
         normalised_to=None,
         one_off=False,
         reply_to_text=None,
@@ -979,11 +980,11 @@ def set_up_usage_data(start_date):
     create_ft_billing(bst_date=two_days_later, template=sms_template_1, billable_unit=1, rate=0.11)
 
     create_ft_billing(bst_date=one_week_later, template=letter_template_1,
-                      notifications_sent=2, billable_unit=2, rate=.35, postage='first')
+                      notifications_sent=2, billable_unit=1, rate=.35, postage='first')
     create_ft_billing(bst_date=one_month_later, template=letter_template_1,
-                      notifications_sent=4, billable_unit=8, rate=.45, postage='second')
+                      notifications_sent=4, billable_unit=2, rate=.45, postage='second')
     create_ft_billing(bst_date=one_week_later, template=letter_template_1,
-                      notifications_sent=2, billable_unit=4, rate=.45, postage='second')
+                      notifications_sent=2, billable_unit=2, rate=.45, postage='second')
 
     # service with emails only:
     service_with_emails = create_service(service_name='b - emails')

tests/app/letters/test_returned_letters.py

@@ -20,23 +20,4 @@ def test_process_returned_letters(status, references, admin_request, mocker):
     if status != 200:
         assert '{} does not match'.format(references[0]) in response['errors'][0]['message']
     else:
-        mock_celery.assert_called_once_with(args=(references,), queue='database-tasks', compression='zlib')
-
-
-def test_process_returned_letters_splits_tasks_up(admin_request, mocker):
-    mock_celery = mocker.patch("app.letters.rest.process_returned_letters_list.apply_async")
-    mocker.patch("app.letters.rest.MAX_REFERENCES_PER_TASK", 3)
-
-    references = [f'{x:016}' for x in range(10)]
-
-    admin_request.post(
-        'letter-job.create_process_returned_letters_job',
-        _data={"references": references},
-    )
-
-    assert mock_celery.call_count == 4
-
-    assert mock_celery.call_args_list[0][1]['args'][0] == ['0000000000000000', '0000000000000001', '0000000000000002']
-    assert mock_celery.call_args_list[1][1]['args'][0] == ['0000000000000003', '0000000000000004', '0000000000000005']
-    assert mock_celery.call_args_list[2][1]['args'][0] == ['0000000000000006', '0000000000000007', '0000000000000008']
-    assert mock_celery.call_args_list[3][1]['args'][0] == ['0000000000000009']
+        mock_celery.assert_called_once_with([references], queue='database-tasks')

tests/app/platform_stats/test_rest.py

@@ -183,58 +183,3 @@ def test_get_data_for_billing_report(notify_db_session, admin_request):
         "2 second class letters at 35p\n1 first class letters at 50p\n15 international letters at £1.55\n"
     )
     assert response[3]["purchase_order_number"] is None
-
-
-def test_daily_volumes_report(
-        notify_db_session, sample_template, sample_email_template, sample_letter_template, admin_request
-):
-    set_up_usage_data(datetime(2022, 3, 1))
-    response = admin_request.get(
-        "platform_stats.daily_volumes_report",
-        start_date='2022-03-01',
-        end_date='2022-03-31'
-    )
-
-    assert len(response) == 3
-    assert response[0] == {'day': '2022-03-01', 'email_totals': 10, 'letter_sheet_totals': 3,
-                           'letter_totals': 2, 'sms_chargeable_units': 2, 'sms_fragment_totals': 2, 'sms_totals': 1}
-    assert response[1] == {'day': '2022-03-03', 'email_totals': 0, 'letter_sheet_totals': 10, 'letter_totals': 18,
-                           'sms_chargeable_units': 2, 'sms_fragment_totals': 2, 'sms_totals': 2}
-    assert response[2] == {'day': '2022-03-08', 'email_totals': 0, 'letter_sheet_totals': 11, 'letter_totals': 12,
-                           'sms_chargeable_units': 4, 'sms_fragment_totals': 4, 'sms_totals': 2}
-
-
-def test_volumes_by_service_report(
-        notify_db_session, sample_template, sample_email_template, sample_letter_template, admin_request
-):
-    fixture = set_up_usage_data(datetime(2022, 3, 1))
-    response = admin_request.get(
-        "platform_stats.volumes_by_service_report",
-        start_date='2022-03-01',
-        end_date='2022-03-01'
-    )
-
-    assert len(response) == 4
-    assert response[0] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0,
-                           'letter_sheet_totals': 0, 'letter_totals': 0,
-                           'organisation_id': str(fixture['org_1'].id),
-                           'organisation_name': fixture['org_1'].name,
-                           'service_id': str(fixture['service_1_sms_and_letter'].id),
-                           'service_name': fixture['service_1_sms_and_letter'].name,
-                           'sms_chargeable_units': 2, 'sms_notifications': 1}
-    assert response[1] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
-                           'letter_totals': 0, 'organisation_id': str(fixture['org_1'].id),
-                           'organisation_name': fixture['org_1'].name,
-                           'service_id': str(fixture['service_with_out_ft_billing_this_year'].id),
-                           'service_name': fixture['service_with_out_ft_billing_this_year'].name,
-                           'sms_chargeable_units': 0, 'sms_notifications': 0}
-    assert response[2] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
-                           'letter_totals': 0, 'organisation_id': '', 'organisation_name': '',
-                           'service_id': str(fixture['service_with_sms_without_org'].id),
-                           'service_name': fixture['service_with_sms_without_org'].name,
-                           'sms_chargeable_units': 0, 'sms_notifications': 0}
-    assert response[3] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
-                           'letter_totals': 0, 'organisation_id': '', 'organisation_name': '',
-                           'service_id': str(fixture['service_with_sms_within_allowance'].id),
-                           'service_name': fixture['service_with_sms_within_allowance'].name,
-                           'sms_chargeable_units': 0, 'sms_notifications': 0}

tests/app/test_commands.py

@@ -1,23 +1,8 @@
-from app.commands import (
-    insert_inbound_numbers_from_file,
-    local_dev_broadcast_permissions,
-)
-from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers
+from app.commands import local_dev_broadcast_permissions
 from app.dao.services_dao import dao_add_user_to_service
 from tests.app.db import create_user
 
 
-def test_insert_inbound_numbers_from_file(notify_db_session, notify_api, tmpdir):
-    numbers_file = tmpdir.join("numbers.txt")
-    numbers_file.write("07700900373\n07700900473\n07700900375\n\n\n\n")
-
-    notify_api.test_cli_runner().invoke(insert_inbound_numbers_from_file, ['-f', numbers_file])
-
-    inbound_numbers = dao_get_available_inbound_numbers()
-    assert len(inbound_numbers) == 3
-    assert set(x.number for x in inbound_numbers) == {'07700900373', '07700900473', '07700900375'}
-
-
 def test_local_dev_broadcast_permissions(
     sample_service,
     sample_broadcast_service,

tests/app/user/test_rest_verify.py

@@ -292,29 +292,15 @@ def test_send_sms_code_returns_204_when_too_many_codes_already_created(client, s
     assert VerifyCode.query.count() == 5
 
 
-@pytest.mark.parametrize('post_data, expected_url_starts_with', (
-    (
-        {},
-        'http://localhost',
-    ),
-    (
-        {'admin_base_url': 'https://example.com'},
-        'https://example.com',
-    ),
-))
-def test_send_new_user_email_verification(
-    client,
-    sample_user,
-    mocker,
-    email_verification_template,
-    post_data,
-    expected_url_starts_with,
-):
+def test_send_new_user_email_verification(client,
+                                          sample_user,
+                                          mocker,
+                                          email_verification_template):
     mocked = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
     auth_header = create_admin_authorization_header()
     resp = client.post(
         url_for('user.send_new_user_email_verification', user_id=str(sample_user.id)),
-        data=json.dumps(post_data),
+        data=json.dumps({}),
         headers=[('Content-Type', 'application/json'), auth_header])
     notify_service = email_verification_template.service
     assert resp.status_code == 204
@@ -322,8 +308,6 @@ def test_send_new_user_email_verification(
     assert VerifyCode.query.count() == 0
     mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks")
     assert notification.reply_to_text == notify_service.get_default_reply_to_email_address()
-    assert notification.personalisation['name'] == 'Test User'
-    assert notification.personalisation['url'].startswith(expected_url_starts_with)
 
 
 def test_send_email_verification_returns_404_for_bad_input_data(client, notify_db_session, mocker):

tests/app/v2/notifications/test_get_notifications.py

@@ -20,6 +20,7 @@ def test_get_notification_by_id_returns_200(
         template=sample_template,
         billable_units=billable_units,
         sent_by=provider,
+        scheduled_for="2017-05-12 15:15"
     )
 
     # another
@@ -27,6 +28,7 @@ def test_get_notification_by_id_returns_200(
         template=sample_template,
         billable_units=billable_units,
         sent_by=provider,
+        scheduled_for="2017-06-12 15:15"
     )
 
     auth_header = create_service_authorization_header(service_id=sample_notification.service_id)