disable cache persistence & env updates

Bump utils to version 56.0.0

.gitignore

@@ -69,6 +69,8 @@ target/
 *.DS_Store
 environment.sh
 .envrc
+.env
+.env*
 
 celerybeat-schedule
 

Makefile

@@ -25,7 +25,7 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 bootstrap: generate-version-file ## Set up everything to run the app
 	pip3 install -r requirements_for_test.txt
 	createdb notification_api || true
-	(. environment.sh && flask db upgrade) || true
+	(flask db upgrade) || true
 
 .PHONY: bootstrap-with-docker
 bootstrap-with-docker: ## Build the image to run the app in Docker
@@ -33,11 +33,12 @@ bootstrap-with-docker: ## Build the image to run the app in Docker
 
 .PHONY: run-flask
 run-flask: ## Run flask
-	. environment.sh && flask run -p 6011
+	flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery
-run-celery: ## Run celery
-	. environment.sh && celery \
+run-celery: ## Run celery, TODO remove purge for staging/prod
+	celery -A run_celery.notify_celery purge -f
+	celery \
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
@@ -49,9 +50,9 @@ run-celery-with-docker: ## Run celery in Docker container (useful if you can't i
 
 .PHONY: run-celery-beat
 run-celery-beat: ## Run celery beat
-	. environment.sh && celery \
-		-A run_celery.notify_celery beat \
-		--loglevel=INFO
+	celery \
+	-A run_celery.notify_celery beat \
+	--loglevel=INFO
 
 .PHONY: run-celery-beat-with-docker
 run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)

app/celery/research_mode_tasks.py

@@ -164,7 +164,7 @@ def ses_notification_callback(reference):
             'processingTimeMillis': 2003,
             'recipients': ['success@simulator.amazonses.com'],
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
+            'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
             'smtpResponse': '250 2.6.0 Message received',
             'timestamp': '2017-11-17T12:14:03.646Z'
         },
@@ -201,7 +201,7 @@ def ses_notification_callback(reference):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:01.643Z'
         },
@@ -211,14 +211,14 @@ def ses_notification_callback(reference):
     return {
         'Type': 'Notification',
         'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
-        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:03.710Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',
-        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
-        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
+        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
+        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
         'MessageAttributes': {}
     }
 
@@ -244,7 +244,7 @@ def _ses_bounce_callback(reference, bounce_type):
             }],
             'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
+            'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
             'timestamp': '2017-11-17T12:14:05.131Z'
         },
         'mail': {
@@ -280,7 +280,7 @@ def _ses_bounce_callback(reference, bounce_type):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:03.000Z'
         },
@@ -289,13 +289,13 @@ def _ses_bounce_callback(reference, bounce_type):
     return {
         'Type': 'Notification',
         'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
-        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:05.149Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',  # noqa
-        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
-        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
+        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
+        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
         'MessageAttributes': {}
     }

app/clients/cbc_proxy.py

@@ -39,7 +39,7 @@ class CBCProxyClient:
         if app.config.get('CBC_PROXY_ENABLED'):
             self._lambda_client = boto3.client(
                 'lambda',
-                region_name='eu-west-2',
+                region_name='us-west-2',
                 aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
                 aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
             )

app/config.py

@@ -137,7 +137,7 @@ class Config(object):
     ###########################
 
     NOTIFY_ENVIRONMENT = 'development'
-    AWS_REGION = 'eu-west-1'
+    AWS_REGION = 'us-west-2'
     INVITATION_EXPIRATION_DAYS = 2
     NOTIFY_APP_NAME = 'api'
     SQLALCHEMY_RECORD_QUERIES = False
@@ -152,6 +152,8 @@ class Config(object):
     ONE_OFF_MESSAGE_FILENAME = 'Report'
     MAX_VERIFY_CODE_COUNT = 5
     MAX_FAILED_LOGIN_COUNT = 10
+    
+    SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
 
     # be careful increasing this size without being sure that we won't see slowness in pysftp
     MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
@@ -381,9 +383,8 @@ class Config(object):
     # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
     MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
     FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
-    SES_STUB_URL = os.environ.get("SES_STUB_URL")
 
-    AWS_REGION = 'eu-west-1'
+    AWS_REGION = 'us-west-2'
 
     CBC_PROXY_ENABLED = True
     CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '')
@@ -428,9 +429,9 @@ class Development(Config):
 
     NOTIFY_ENVIRONMENT = 'development'
     NOTIFY_LOG_PATH = 'application.log'
-    NOTIFY_EMAIL_DOMAIN = "notify.tools"
+    NOTIFY_EMAIL_DOMAIN = "dispostable.com"
 
-    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api')
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
     REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
 
     ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
@@ -466,7 +467,7 @@ class Test(Development):
     LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
 
     # this is overriden in jenkins and on cloudfoundry
-    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
 
     CELERY = {
         **Config.CELERY,

app/dao/fact_billing_dao.py

@@ -34,52 +34,84 @@ from app.models import (
 from app.utils import get_london_midnight_in_utc
 
 
+def fetch_sms_free_allowance_remainder_until_date(end_date):
+    # ASSUMPTION: AnnualBilling has been populated for year.
+    billing_year = get_financial_year_for_datetime(end_date)
+    start_of_year = date(billing_year, 4, 1)
+
+    billable_units = func.coalesce(func.sum(FactBilling.billable_units * FactBilling.rate_multiplier), 0)
+
+    query = db.session.query(
+        AnnualBilling.service_id.label("service_id"),
+        AnnualBilling.free_sms_fragment_limit,
+        billable_units.label('billable_units'),
+        func.greatest((AnnualBilling.free_sms_fragment_limit - billable_units).cast(Integer), 0).label('sms_remainder')
+    ).outerjoin(
+        # if there are no ft_billing rows for a service we still want to return the annual billing so we can use the
+        # free_sms_fragment_limit)
+        FactBilling, and_(
+            AnnualBilling.service_id == FactBilling.service_id,
+            FactBilling.bst_date >= start_of_year,
+            FactBilling.bst_date < end_date,
+            FactBilling.notification_type == SMS_TYPE,
+        )
+    ).filter(
+        AnnualBilling.financial_year_start == billing_year,
+    ).group_by(
+        AnnualBilling.service_id,
+        AnnualBilling.free_sms_fragment_limit,
+    )
+    return query
+
+
 def fetch_sms_billing_for_all_services(start_date, end_date):
-    # ASSUMPTION: start and end date are in the same financial year
-    financial_year = get_financial_year_for_datetime(get_london_midnight_in_utc(start_date))
 
     # ASSUMPTION: AnnualBilling has been populated for year.
-    ft_billing_subquery = query_sms_usage_for_year_per_service(financial_year).subquery()
+    allowance_left_at_start_date_query = fetch_sms_free_allowance_remainder_until_date(start_date).subquery()
 
-    sms_billable_units = func.sum(func.coalesce(ft_billing_subquery.c.chargeable_units, 0))
+    sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
 
-    # get the lowest value allowance (which will be the last date within our filter range)
-    sms_allowance_left = func.greatest(
-        func.min(AnnualBilling.free_sms_fragment_limit - ft_billing_subquery.c.free_allowance_used_to_date),
-        0
-    )
+    # subtract sms_billable_units units accrued since report's start date to get up-to-date
+    # allowance remainder
+    sms_allowance_left = func.greatest(allowance_left_at_start_date_query.c.sms_remainder - sms_billable_units, 0)
+
+    # billable units here are for period between start date and end date only, so to see
+    # how many are chargeable, we need to see how much free allowance was used up in the
+    # period up until report's start date and then do a subtraction
+    chargeable_sms = func.greatest(sms_billable_units - allowance_left_at_start_date_query.c.sms_remainder, 0)
+    sms_cost = chargeable_sms * FactBilling.rate
 
-    chargeable_sms = func.sum(ft_billing_subquery.c.charged_units)
-    sms_cost = func.sum(ft_billing_subquery.c.cost)
     query = db.session.query(
         Organisation.name.label('organisation_name'),
         Organisation.id.label('organisation_id'),
         Service.name.label("service_name"),
         Service.id.label("service_id"),
-        AnnualBilling.free_sms_fragment_limit,
-        func.coalesce(sms_allowance_left, 0).label("sms_remainder"),
-        func.coalesce(sms_billable_units, 0).label('sms_billable_units'),
-        func.coalesce(chargeable_sms, 0).label("chargeable_billable_sms"),
-        func.coalesce(sms_cost, 0).label('sms_cost'),
+        allowance_left_at_start_date_query.c.free_sms_fragment_limit,
+        FactBilling.rate.label('sms_rate'),
+        sms_allowance_left.label("sms_remainder"),
+        sms_billable_units.label('sms_billable_units'),
+        chargeable_sms.label("chargeable_billable_sms"),
+        sms_cost.label('sms_cost'),
     ).select_from(
         Service
+    ).outerjoin(
+        allowance_left_at_start_date_query, Service.id == allowance_left_at_start_date_query.c.service_id
     ).outerjoin(
         Service.organisation
-    ).outerjoin(
-        AnnualBilling,
-        and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == financial_year)
-    ).outerjoin(
-        ft_billing_subquery, Service.id == ft_billing_subquery.c.service_id
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
     ).filter(
-        Service.restricted.is_(False),
-        ft_billing_subquery.c.bst_date >= start_date,
-        ft_billing_subquery.c.bst_date <= end_date,
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == SMS_TYPE,
     ).group_by(
         Organisation.name,
         Organisation.id,
         Service.id,
         Service.name,
-        AnnualBilling.free_sms_fragment_limit
+        allowance_left_at_start_date_query.c.free_sms_fragment_limit,
+        allowance_left_at_start_date_query.c.sms_remainder,
+        FactBilling.rate,
     ).order_by(
         Organisation.name,
         Service.name
@@ -730,12 +762,6 @@ def fetch_sms_billing_for_organisation(organisation_id, financial_year):
 
 
 def query_organisation_sms_usage_for_year(organisation_id, year):
-    return query_sms_usage_for_year_per_service(year).filter(
-        Service.organisation_id == organisation_id
-    )
-
-
-def query_sms_usage_for_year_per_service(year):
     """
     See docstring for query_service_sms_usage_for_year()
     """
@@ -769,15 +795,12 @@ def query_sms_usage_for_year_per_service(year):
     # for, after taking any remaining free allowance into account.
     charged_units = func.greatest(this_rows_chargeable_units - remaining_free_allowance_before_this_row, 0)
 
-    free_allowance_used_to_date = chargeable_units_used_before_this_row + this_rows_chargeable_units
-
     return db.session.query(
         Service.id.label('service_id'),
         FactBilling.bst_date,
         this_rows_chargeable_units.label("chargeable_units"),
         (charged_units * FactBilling.rate).label("cost"),
         charged_units.label("charged_units"),
-        free_allowance_used_to_date.label("free_allowance_used_to_date"),
     ).join(
         AnnualBilling,
         AnnualBilling.service_id == Service.id
@@ -790,6 +813,7 @@ def query_sms_usage_for_year_per_service(year):
             FactBilling.notification_type == SMS_TYPE,
         )
     ).filter(
+        Service.organisation_id == organisation_id,
         AnnualBilling.financial_year_start == year,
     )
 

app/models.py

@@ -122,7 +122,7 @@ class User(db.Model):
     state = db.Column(db.String, nullable=False, default='pending')
     platform_admin = db.Column(db.Boolean, nullable=False, default=False)
     current_session_id = db.Column(UUID(as_uuid=True), nullable=True)
-    auth_type = db.Column(db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=SMS_AUTH_TYPE)
+    auth_type = db.Column(db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE)
     email_access_validated_at = db.Column(
         db.DateTime, index=False, unique=False, nullable=False, default=datetime.datetime.utcnow
     )
@@ -1841,7 +1841,7 @@ class InvitedUser(db.Model):
         db.ForeignKey('auth_type.name'),
         index=True,
         nullable=False,
-        default=SMS_AUTH_TYPE
+        default=EMAIL_AUTH_TYPE
     )
     folder_permissions = db.Column(JSONB(none_as_null=True), nullable=False, default=[])
 

app/platform_stats/rest.py

@@ -66,6 +66,7 @@ def validate_date_range_is_within_a_financial_year(start_date, end_date):
     return start_date, end_date
 
 
+@platform_stats_blueprint.route('usage-for-all-services')
 @platform_stats_blueprint.route('data-for-billing-report')
 def get_data_for_billing_report():
     start_date = request.args.get('start_date')

app/template/rest.py

@@ -9,7 +9,7 @@ from notifications_utils.template import (
     BroadcastMessageTemplate,
     SMSMessageTemplate,
 )
-from PyPDF2.utils import PdfReadError
+from PyPDF2.errors import PdfReadError
 from requests import post as requests_post
 from sqlalchemy.orm.exc import NoResultFound
 

devcontainer-api/.devcontainer.json

@@ -0,0 +1,42 @@
+{
+	"name": "notification-api",
+	"dockerComposeFile": "../docker-compose.devcontainer.yml",
+	"service": "dev",
+	"workspaceFolder": "/workspace",
+	"shutdownAction": "stopCompose",
+	"remoteEnv": {
+		"PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence
+	},
+	"settings": {
+		"[python]": {
+			"editor.formatOnSave": true
+		},
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.pythonPath": "/usr/local/bin/python"
+	},
+	"features": {
+		"docker-from-docker": {
+			"version": "latest",
+			"moby": true
+		}
+	},
+	"extensions": [
+		"ms-python.black-formatter",
+		"donjayamanne.python-extension-pack",
+		"ms-azuretools.vscode-docker",
+		"ms-python.vscode-pylance",
+		"eamodio.gitlens",
+		"wholroyd.jinja",
+		"pmbenjamin.vscode-snyk",
+		"visualstudioexptteam.vscodeintellicode",
+		"yzhang.markdown-all-in-one",
+		"ms-ossdata.vscode-postgresql",
+		"GitHub.copilot",
+		"ms-vsliveshare.vsliveshare",
+		"mtxr.sqltools",
+		"mtxr.sqltools-driver-pg",
+	],
+	"postCreateCommand": "notify-dev-entrypoint.sh",
+	"remoteUser": "vscode"
+}

devcontainer-api/Dockerfile

@@ -0,0 +1,40 @@
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.9
+
+RUN apt-get update \
+    && apt-get -y install --no-install-recommends \
+    apt-utils \
+    postgresql-client \
+    2>&1 \
+    && apt-get -y install \
+    curl \
+    emacs \
+    exa \
+    fd-find \
+    git \
+    iproute2 \
+    less \
+    libsodium-dev \
+    lsb-release \
+    man-db \
+    manpages \
+    net-tools \
+    openssh-client \
+    procps \
+    sudo \
+    tldr \
+    unzip \
+    vim \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Upgrade pip
+RUN pip install --upgrade pip
+
+COPY devcontainer-api/scripts/notify-dev-entrypoint.sh /usr/local/bin/
+COPY devcontainer-api/scripts/notify-worker-entrypoint.sh /usr/local/bin/
+
+ENV SHELL /bin/zsh
+
+EXPOSE 8000
+EXPOSE 6011

devcontainer-api/initdb/notify-db-entrypoint.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -ex
+
+###################################################################
+# This script will get executed *once* the Docker container has 
+# been built. Commands that need to be executed with all available
+# tools and the filesystem mount enabled should be located here. 
+#
+# The PostgreSQL Docker image has an extension mechanism that does 
+# not necessitate to override the entrypoint or main command. One
+# simply has to copy a shell script into the 
+# /docker-entrypoint-initdb.d/ initialization folder. 
+###################################################################
+
+# Notify database setup.
+createdb --user=postgres notification_api
+
+wall "The db container entrypoint setup is complete!"

devcontainer-api/scripts/notify-dev-entrypoint.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -ex 
+
+###################################################################
+# This script will get executed *once* the Docker container has 
+# been built. Commands that need to be executed with all available
+# tools and the filesystem mount enabled should be located here. 
+###################################################################
+
+# Define aliases
+echo -e "\n\n# User's Aliases" >> ~/.zshrc
+echo -e "alias fd=fdfind" >> ~/.zshrc
+echo -e "alias l='ls -al --color'" >> ~/.zshrc
+echo -e "alias ls='exa'" >> ~/.zshrc
+echo -e "alias l='exa -alh'" >> ~/.zshrc
+echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
+echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
+
+# # Kubectl aliases and command autocomplete
+# echo -e "alias k='kubectl'" >> ~/.zshrc
+# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc
+# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc
+# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc
+# echo -e "complete -F __start_kubectl k" >> ~/.zshrc
+
+cd /workspace 
+
+# Warm up git index prior to display status in prompt else it will 
+# be quite slow on every invocation of starship.
+git status
+
+make generate-version-file
+pip3 install -r requirements.txt
+pip3 install -r requirements_for_test.txt
+
+# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer
+pip3 install virtualenv
+
+# Upgrade schema of the notification_api database
+flask db upgrade
+
+# Run flask server
+# make run-flask

devcontainer-api/scripts/notify-worker-entrypoint.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -ex 
+
+###################################################################
+# This script will get executed *once* the Docker container has 
+# been built. Commands that need to be executed with all available
+# tools and the filesystem mount enabled should be located here. 
+###################################################################
+
+# Define aliases
+echo -e "\n\n# User's Aliases" >> ~/.zshrc
+echo -e "alias fd=fdfind" >> ~/.zshrc
+echo -e "alias l='ls -al --color'" >> ~/.zshrc
+echo -e "alias ls='exa'" >> ~/.zshrc
+echo -e "alias l='exa -alh'" >> ~/.zshrc
+echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
+echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
+
+# # Kubectl aliases and command autocomplete
+# echo -e "alias k='kubectl'" >> ~/.zshrc
+# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc
+# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc
+# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc
+# echo -e "complete -F __start_kubectl k" >> ~/.zshrc
+
+cd /workspace 
+
+# Warm up git index prior to display status in prompt else it will 
+# be quite slow on every invocation of starship.
+git status
+
+make generate-version-file
+pip3 install -r requirements.txt
+pip3 install -r requirements_for_test.txt
+
+# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer
+pip3 install virtualenv
+
+# Run Celery worker
+# make run-celery

devcontainer-worker/.devcontainer.json

@@ -0,0 +1,42 @@
+{
+	"name": "notification-api",
+	"dockerComposeFile": "../docker-compose.devcontainer.yml",
+	"service": "worker",
+	"workspaceFolder": "/workspace",
+	"shutdownAction": "stopCompose",
+	"remoteEnv": {
+		"PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence
+	},
+	"settings": {
+		"[python]": {
+			"editor.formatOnSave": true
+		},
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.pythonPath": "/usr/local/bin/python"
+	},
+	"features": {
+		"docker-from-docker": {
+			"version": "latest",
+			"moby": true
+		}
+	},
+	"extensions": [
+		"ms-python.black-formatter",
+		"donjayamanne.python-extension-pack",
+		"ms-azuretools.vscode-docker",
+		"ms-python.vscode-pylance",
+		"eamodio.gitlens",
+		"wholroyd.jinja",
+		"pmbenjamin.vscode-snyk",
+		"visualstudioexptteam.vscodeintellicode",
+		"yzhang.markdown-all-in-one",
+		"ms-ossdata.vscode-postgresql",
+		"GitHub.copilot",
+		"ms-vsliveshare.vsliveshare",
+		"mtxr.sqltools",
+		"mtxr.sqltools-driver-pg",
+	],
+	"postCreateCommand": "notify-worker-entrypoint.sh",
+	"remoteUser": "vscode"
+}

docker-compose.devcontainer.yml

@@ -0,0 +1,72 @@
+version: '3'
+
+services:
+  db:
+    container_name: db
+    image: postgres:13.4
+    volumes:
+      - ./devcontainer-api/initdb:/docker-entrypoint-initdb.d
+    environment:
+      PGGSSENCMODE: disable
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: chummy
+      POSTGRES_HOST_AUTH_METHOD: trust
+    ports:
+      - "5432:5432"
+    expose:
+      - "5432"
+    command:
+      - "postgres"
+      - "-c"
+      - "listen_addresses=*"
+    restart: always
+  redis:
+    container_name: redis
+    image: redis:6.2
+    restart: always
+    command: sh -c "rm -f /data/dump.rdb && redis-server --port 6380" # disable persistence
+    ports:
+      - "6380:6380"
+    expose:
+      - "6380"
+  dev:
+    container_name: dev
+    image: dev-notification-api
+    build:
+      context: .
+      dockerfile: devcontainer-api/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/workspace:cached
+    command: sleep infinity
+    ports:
+      - 6011:6011
+    expose:
+      - "6011"
+    depends_on:
+      - db
+      - redis
+    links:
+      - db
+    restart: always
+  worker:
+    container_name: worker
+    image: dev-notification-api
+    build:
+      context: .
+      dockerfile: devcontainer-api/Dockerfile
+    env_file: .env
+    volumes:
+      - .:/workspace:cached
+    command: sleep infinity
+    depends_on:
+      - db
+      - redis
+      - dev
+    links:
+      - db
+
+networks:
+  default:
+    name: notify-network
+    external: true

migrations/versions/0025_notify_service_data.py

@@ -25,18 +25,18 @@ def upgrade():
     password = hashpw(str(uuid.uuid4()))
     op.get_bind()
     user_insert = """INSERT INTO users (id, name, email_address, created_at, failed_login_count, _password, mobile_number, state, platform_admin)
-                     VALUES ('{}', 'Notify service user', 'notify-service-user@digital.cabinet-office', '{}', 0,'{}', '+441234123412', 'active', False)
+                     VALUES ('{}', 'Notify service user', 'testsender@dispostable.com', '{}', 0,'{}', '+441234123412', 'active', False)
                   """
     op.execute(user_insert.format(user_id, datetime.utcnow(), password))
     service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
-                        VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk',
-                        '{}', 'notify@digital.cabinet-office.gov.uk', 1)
+                        VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com',
+                        '{}', 'testsender@dispostable.com', 1)
 
                      """
     op.execute(service_history_insert.format(service_id, datetime.utcnow(), user_id))
     service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
-                        VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk',
-                        '{}', 'notify@digital.cabinet-office.gov.uk', 1)
+                        VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com',
+                        '{}', 'testsender@dispostable.com', 1)
                     """
     op.execute(service_insert.format(service_id, datetime.utcnow(), user_id))
     user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES ('{}', '{}')"""

migrations/versions/0029_fix_email_from.py

@@ -16,8 +16,8 @@ import sqlalchemy as sa
 service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
 def upgrade():
     op.get_bind()
-    op.execute("update services set email_from = 'gov.uk.notify' where id = '{}'".format(service_id))
-    op.execute("update services_history set email_from = 'gov.uk.notify' where id = '{}'".format(service_id))
+    op.execute("update services set email_from = 'testsender' where id = '{}'".format(service_id))
+    op.execute("update services_history set email_from = 'testsender' where id = '{}'".format(service_id))
 
 
 def downgrade():

migrations/versions/0070_fix_notify_user_email.py

@@ -17,7 +17,7 @@ import sqlalchemy as sa
 def upgrade():
     op.execute("""
         UPDATE users
-        SET email_address = 'notify-service-user@digital.cabinet-office.gov.uk'
+        SET email_address = 'testsender@dispostable.com'
         WHERE email_address = 'notify-service-user@digital.cabinet-office'
     """)
 
@@ -26,5 +26,5 @@ def downgrade():
     op.execute("""
         UPDATE users
         SET email_address = 'notify-service-user@digital.cabinet-office'
-        WHERE email_address = 'notify-service-user@digital.cabinet-office.gov.uk'
+        WHERE email_address = 'testsender@dispostable.com'
     """)

migrations/versions/0082_add_golive_template.py

@@ -64,7 +64,7 @@ It’s only an emergency if:
 *   a 500 response code appears when you try to send messages using the API
 
 If you have one of these emergencies, email details to:
-ooh-gov-uk-notify-support@digital.cabinet-office.gov.uk
+testsender@dispostable.com
 
 ^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team.
 

migrations/versions/0130_service_email_reply_to_row.py

@@ -22,7 +22,7 @@ def upgrade():
         INSERT INTO service_email_reply_to
         (id, service_id, email_address, is_default, created_at)
         VALUES
-        ('{}','{}', 'notify+1@digital.cabinet-office.gov.uk', 'f', NOW())
+        ('{}','{}', 'testsender@dispostable.com', 'f', NOW())
     """.format(EMAIL_REPLY_TO_ID, NOTIFY_SERVICE_ID))
 
 

requirements.in

@@ -23,13 +23,14 @@ cachetools==5.1.0
 beautifulsoup4==4.11.1
 lxml==4.8.0
 Werkzeug==2.0.3  # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned
+python-dotenv==0.20.0
 
 notifications-python-client==6.3.0
 
 # PaaS
 awscli-cwlogs==1.4.6
 
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@55.1.6
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0
 
 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
 prometheus-client==0.14.1

requirements.txt

@@ -77,6 +77,8 @@ docopt==0.6.2
     # via notifications-python-client
 docutils==0.16
     # via awscli
+python-dotenv==0.20.0
+    # via -r requirements.in
 eventlet==0.33.1
     # via gunicorn
 flask==2.1.2
@@ -160,7 +162,7 @@ mistune==0.8.4
     # via notifications-utils
 notifications-python-client==6.3.0
     # via -r requirements.in
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@55.1.6
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0
     # via -r requirements.in
 orderedset==2.0.3
     # via notifications-utils
@@ -189,7 +191,7 @@ pyjwt==2.4.0
     #   notifications-python-client
 pyparsing==3.0.9
     # via packaging
-pypdf2==1.28.2
+pypdf2==2.0.0
     # via notifications-utils
 pyproj==3.3.1
     # via notifications-utils
@@ -250,6 +252,8 @@ sqlalchemy==1.4.36
     #   marshmallow-sqlalchemy
 statsd==3.3.0
     # via notifications-utils
+typing-extensions==4.2.0
+    # via pypdf2
 uri-template==1.2.0
     # via jsonschema
 urllib3==1.26.9

sample.env

@@ -0,0 +1,39 @@
+# Debug
+DEBUG=True
+ANTIVIRUS_ENABLED=0
+NOTIFY_ENVIRONMENT=development
+NOTIFICATION_QUEUE_PREFIX=local_dev_10x
+STATSD_HOST=localhost
+SES_STUB_URL=None
+NOTIFY_APP_NAME=api
+NOTIFY_EMAIL_DOMAIN=dispostable.com
+
+# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
+ADMIN_CLIENT_ID=notify-admin
+GOVUK_ALERTS_CLIENT_ID=govuk-alerts
+
+# Flask
+FLASK_APP=application.py
+FLASK_ENV=development
+WERKZEUG_DEBUG_PIN=off
+SECRET_KEY=dev-notify-secret-key
+DANGEROUS_SALT=dev-notify-salt
+
+# URL of admin app
+ADMIN_BASE_URL=http://0.0.0.0:6012
+
+# URL of api app (on AWS this is the internal api endpoint)
+API_HOST_NAME=http://0.0.0.0:6011
+
+# URL of redis instance
+REDIS_URL=redis://redis:6380
+REDIS_ENABLED=1
+
+# DB conection string, this is overriden in jenkins and on cloudfoundry
+SQLALCHEMY_DATABASE_URI=postgresql://postgres:chummy@db:5432/notification_api
+SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api
+
+# AWS
+AWS_REGION=us-west-2
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=

scripts/run_app_paas.sh

@@ -65,7 +65,7 @@ function start_application {
 }
 
 function start_aws_logs_agent {
-  exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
+  exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
   AWSLOGS_AGENT_PID=$!
   echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
 }

scripts/run_multi_worker_app_paas.sh

@@ -102,7 +102,7 @@ function start_application {
 
 function start_aws_logs_agent {
   echo "Starting aws logs agent..."
-  exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
+  exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
   AWSLOGS_AGENT_PID=$!
   echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
 }

tests/app/dao/test_fact_billing_dao.py

@@ -16,12 +16,12 @@ from app.dao.fact_billing_dao import (
     fetch_letter_line_items_for_all_services,
     fetch_monthly_billing_for_year,
     fetch_sms_billing_for_all_services,
+    fetch_sms_free_allowance_remainder_until_date,
     fetch_usage_year_for_organisation,
     fetch_volumes_by_service,
     get_rate,
     get_rates_for_billing,
     query_organisation_sms_usage_for_year,
-    query_sms_usage_for_year_per_service,
 )
 from app.dao.organisation_dao import dao_add_service_to_organisation
 from app.models import NOTIFICATION_STATUS_TYPES, FactBilling
@@ -635,6 +635,32 @@ def test_delete_billing_data(notify_db_session):
     )
 
 
+def test_fetch_sms_free_allowance_remainder_until_date_with_two_services(notify_db_session):
+    service = create_service(service_name='has free allowance')
+    template = create_template(service=service)
+    org = create_organisation(name="Org for {}".format(service.name))
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2016)
+    create_ft_billing(template=template, bst_date=datetime(2016, 4, 20), billable_unit=2, rate=0.11)
+    create_ft_billing(template=template, bst_date=datetime(2016, 5, 20), billable_unit=3, rate=0.11)
+
+    service_2 = create_service(service_name='used free allowance')
+    template_2 = create_template(service=service_2)
+    org_2 = create_organisation(name="Org for {}".format(service_2.name))
+    dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id)
+    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=20, financial_year_start=2016)
+    create_ft_billing(template=template_2, bst_date=datetime(2016, 4, 20), billable_unit=12, rate=0.11)
+    create_ft_billing(template=template_2, bst_date=datetime(2016, 4, 22), billable_unit=10, rate=0.11)
+    create_ft_billing(template=template_2, bst_date=datetime(2016, 5, 20), billable_unit=3, rate=0.11)
+
+    results = fetch_sms_free_allowance_remainder_until_date(datetime(2016, 5, 1)).all()
+    assert len(results) == 2
+    service_result = [row for row in results if row[0] == service.id]
+    assert service_result[0] == (service.id, 10, 2, 8)
+    service_2_result = [row for row in results if row[0] == service_2.id]
+    assert service_2_result[0] == (service_2.id, 20, 22, 0)
+
+
 def test_fetch_sms_billing_for_all_services_for_first_quarter(notify_db_session):
     # This test is useful because the inner query resultset is empty.
     service = create_service(service_name='a - has free allowance')
@@ -645,15 +671,8 @@ def test_fetch_sms_billing_for_all_services_for_first_quarter(notify_db_session)
     create_ft_billing(template=template, bst_date=datetime(2019, 4, 20), billable_unit=44, rate=0.11)
     results = fetch_sms_billing_for_all_services(datetime(2019, 4, 1), datetime(2019, 5, 30))
     assert len(results) == 1
-    assert results[0].organisation_id == org.id
-    assert results[0].organisation_name == org.name
-    assert results[0].service_id == service.id
-    assert results[0].service_name == service.name
-    assert results[0].chargeable_billable_sms == 0
-    assert results[0].sms_billable_units == 44
-    assert results[0].free_sms_fragment_limit == 25000
-    assert results[0].sms_remainder == 24956
-    assert results[0].sms_cost == 0
+    assert results[0] == (org.name, org.id, service.name, service.id, 25000, Decimal('0.11'), 24956, 44, 0,
+                          Decimal('0'))
 
 
 def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
@@ -682,7 +701,6 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
     create_ft_billing(template=template_3, bst_date=datetime(2019, 4, 20), billable_unit=5, rate=0.11)
     create_ft_billing(template=template_3, bst_date=datetime(2019, 5, 20), billable_unit=7, rate=0.11)
 
-    # this isn't included in results as it doesn't have any SMS rows
     service_4 = create_service(service_name='d - email only')
     email_template = create_template(service=service_4, template_type='email')
     org_4 = create_organisation(name="Org for {}".format(service_4.name))
@@ -699,21 +717,21 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
         # the requested report's start date.
         {
             "organisation_name": org.name, "organisation_id": org.id, "service_name": service_1.name,
-            "service_id": service_1.id, "free_sms_fragment_limit": 10, "sms_remainder": 5,
+            "service_id": service_1.id, "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 5,
             "sms_billable_units": 3, "chargeable_billable_sms": 0, "sms_cost": Decimal('0.00')
         },
         # sms remainder is 0, because this service sent SMS worth 15 billable units, 12 of which were sent
         # before requested report's start date
         {
             "organisation_name": org_2.name, "organisation_id": org_2.id, "service_name": service_2.name,
-            "service_id": service_2.id, "free_sms_fragment_limit": 10, "sms_remainder": 0,
+            "service_id": service_2.id, "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 0,
             "sms_billable_units": 3, "chargeable_billable_sms": 3, "sms_cost": Decimal('0.33')
         },
         # sms remainder is 0, because this service sent SMS worth 12 billable units, 5 of which were sent
         # before requested report's start date
         {
             "organisation_name": org_3.name, "organisation_id": org_3.id, "service_name": service_3.name,
-            "service_id": service_3.id, "free_sms_fragment_limit": 10, "sms_remainder": 0,
+            "service_id": service_3.id, "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 0,
             "sms_billable_units": 7, "chargeable_billable_sms": 2, "sms_cost": Decimal('0.22')
         },
     ]
@@ -733,7 +751,7 @@ def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(noti
             "organisation_name": fixtures["org_1"].name, "organisation_id": fixtures["org_1"].id,
             "service_name": fixtures["service_1_sms_and_letter"].name,
             "service_id": fixtures["service_1_sms_and_letter"].id,
-            "free_sms_fragment_limit": 10, "sms_remainder": 5,
+            "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 5,
             "sms_billable_units": 3, "chargeable_billable_sms": 0, "sms_cost": Decimal('0.00')
         },
         # sms remainder is 0, because this service sent SMS worth 15 billable units, 12 of which were sent
@@ -742,14 +760,14 @@ def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(noti
             "organisation_name": None, "organisation_id": None,
             "service_name": fixtures["service_with_sms_without_org"].name,
             "service_id": fixtures["service_with_sms_without_org"].id, "free_sms_fragment_limit": 10,
-            "sms_remainder": 0,
+            "sms_rate": Decimal('0.11'), "sms_remainder": 0,
             "sms_billable_units": 3, "chargeable_billable_sms": 3, "sms_cost": Decimal('0.33')
         },
         {
             "organisation_name": None, "organisation_id": None,
             "service_name": fixtures["service_with_sms_within_allowance"].name,
             "service_id": fixtures["service_with_sms_within_allowance"].id, "free_sms_fragment_limit": 10,
-            "sms_remainder": 8,
+            "sms_rate": Decimal('0.11'), "sms_remainder": 8,
             "sms_billable_units": 2, "chargeable_billable_sms": 0, "sms_cost": Decimal('0.00')
         },
     ]
@@ -1008,12 +1026,15 @@ def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(n
 
 
 @freeze_time('2022-04-27 13:30')
-def test_query_sms_usage_for_year_per_service_handles_multiple_services(notify_db_session):
+def test_query_organisation_sms_usage_for_year_handles_multiple_services(notify_db_session):
     today = datetime.utcnow().date()
     yesterday = datetime.utcnow().date() - timedelta(days=1)
     current_year = datetime.utcnow().year
 
+    org = create_organisation(name='Organisation 1')
+
     service_1 = create_service(restricted=False, service_name="Service 1")
+    dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
     sms_template_1 = create_template(service=service_1)
     create_ft_billing(
         bst_date=yesterday, template=sms_template_1, rate=1,
@@ -1026,6 +1047,7 @@ def test_query_sms_usage_for_year_per_service_handles_multiple_services(notify_d
     create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=5, financial_year_start=current_year)
 
     service_2 = create_service(restricted=False, service_name="Service 2")
+    dao_add_service_to_organisation(service=service_2, organisation_id=org.id)
     sms_template_2 = create_template(service=service_2)
     create_ft_billing(
         bst_date=yesterday, template=sms_template_2, rate=1,
@@ -1039,7 +1061,7 @@ def test_query_sms_usage_for_year_per_service_handles_multiple_services(notify_d
 
     # ----------
 
-    result = query_sms_usage_for_year_per_service(2022).all()
+    result = query_organisation_sms_usage_for_year(org.id, 2022).all()
 
     service_1_rows = [row for row in result if row.service_id == service_1.id]
     service_2_rows = [row for row in result if row.service_id == service_2.id]
@@ -1073,12 +1095,15 @@ def test_query_sms_usage_for_year_per_service_handles_multiple_services(notify_d
 
 
 @freeze_time('2022-05-01 13:30')
-def test_query_sms_usage_for_year_per_service_handles_multiple_rates(notify_db_session):
+def test_query_organisation_sms_usage_for_year_handles_multiple_rates(notify_db_session):
     old_rate_date = date(2022, 4, 29)
     new_rate_date = date(2022, 5, 1)
     current_year = datetime.utcnow().year
 
+    org = create_organisation(name='Organisation 1')
+
     service_1 = create_service(restricted=False, service_name="Service 1")
+    dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
     sms_template_1 = create_template(service=service_1)
     create_ft_billing(
         bst_date=old_rate_date, template=sms_template_1, rate=2,
@@ -1090,7 +1115,7 @@ def test_query_sms_usage_for_year_per_service_handles_multiple_rates(notify_db_s
     )
     create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=3, financial_year_start=current_year)
 
-    result = query_sms_usage_for_year_per_service(2022).all()
+    result = query_organisation_sms_usage_for_year(org.id, 2022).all()
 
     # al lthe free allowance is used on the first day
     assert result[0]['bst_date'] == date(2022, 4, 29)
@@ -1102,37 +1127,6 @@ def test_query_sms_usage_for_year_per_service_handles_multiple_rates(notify_db_s
     assert result[1]['cost'] == 6
 
 
-@freeze_time('2022-05-01 13:30')
-def test_query_organisation_sms_usage_for_year_filters_on_organisation(notify_db_session):
-    bst_date = date(2022, 5, 1)
-    current_year = datetime.utcnow().year
-
-    service_1 = create_service(restricted=False, service_name="Service 1")
-    service_2 = create_service(restricted=False, service_name="Service 2")
-
-    org_1 = create_organisation(name="Org 1")
-    org_2 = create_organisation(name="Org 2")
-
-    dao_add_service_to_organisation(service=service_1, organisation_id=org_1.id)
-    dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id)
-
-    sms_template_1 = create_template(service=service_1)
-    sms_template_2 = create_template(service=service_2)
-
-    create_ft_billing(bst_date=bst_date, template=sms_template_1, billable_unit=2)
-    create_ft_billing(bst_date=bst_date, template=sms_template_2, billable_unit=4)
-
-    create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=0, financial_year_start=current_year)
-    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=0, financial_year_start=current_year)
-
-    result = query_organisation_sms_usage_for_year(org_1.id, 2022).all()
-
-    assert len(result) == 1
-    assert result[0]['bst_date'] == date(2022, 5, 1)
-    assert result[0]['charged_units'] == 2
-    assert result[0]['service_id'] == service_1.id
-
-
 def test_fetch_daily_volumes_for_platform(
         notify_db_session, sample_template, sample_email_template, sample_letter_template
 ):

tests/app/template/test_rest.py

@@ -10,7 +10,7 @@ import pytest
 import requests_mock
 from freezegun import freeze_time
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
-from PyPDF2.utils import PdfReadError
+from PyPDF2.errors import PdfReadError
 
 from app.dao.templates_dao import (
     dao_get_template_by_id,