diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index 43bedbb35..0a6d3d7be 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -107,6 +107,12 @@ class AwsCloudwatchClient(Client): provider_response = self._aws_value_or_default( event, "delivery", "providerResponse" ) + message_cost = self._aws_value_or_default(event, "delivery", "priceInUSD") + if message_cost is None or message_cost == "": + message_cost = 0.0 + else: + message_cost = float(message_cost) + my_timestamp = self._aws_value_or_default(event, "notification", "timestamp") return { "notification.messageId": event["notification"]["messageId"], @@ -114,6 +120,7 @@ class AwsCloudwatchClient(Client): "delivery.phoneCarrier": phone_carrier, "delivery.providerResponse": provider_response, "@timestamp": my_timestamp, + "delivery.priceInUSD": message_cost, } # Here is an example of how to get the events with log insights diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index ed84218d6..52823f7d6 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -507,7 +507,7 @@ def insert_notification_history_delete_notifications( SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id, key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units, client_reference, international, phone_prefix, rate_multiplier, notification_status, - created_by_id, document_download_count + created_by_id, document_download_count, message_cost FROM notifications WHERE service_id = :service_id AND notification_type = :notification_type @@ -842,7 +842,6 @@ def dao_update_delivery_receipts(receipts, delivered): new_receipts.append(r) receipts = new_receipts - id_to_carrier = { r["notification.messageId"]: r["delivery.phoneCarrier"] for r in receipts } @@ -851,9 +850,13 @@ def dao_update_delivery_receipts(receipts, delivered): } id_to_timestamp = {r["notification.messageId"]: r["@timestamp"] for r in receipts} + id_to_message_cost = { + r["notification.messageId"]: r["delivery.priceInUSD"] for r in receipts + } status_to_update_with = NotificationStatus.DELIVERED if not delivered: status_to_update_with = NotificationStatus.FAILED + stmt = ( update(Notification) .where(Notification.message_id.in_(id_to_carrier.keys())) @@ -877,6 +880,12 @@ def dao_update_delivery_receipts(receipts, delivered): for key, value in id_to_provider_response.items() ] ), + message_cost=case( + *[ + (Notification.message_id == key, value) + for key, value in id_to_message_cost.items() + ] + ), ) ) db.session.execute(stmt) @@ -908,7 +917,6 @@ def dao_close_out_delivery_receipts(): def dao_batch_insert_notifications(batch): - db.session.bulk_save_objects(batch) db.session.commit() current_app.logger.info(f"Batch inserted notifications: {len(batch)}") diff --git a/app/models.py b/app/models.py index f78f630ea..d9a50a025 100644 --- a/app/models.py +++ b/app/models.py @@ -1508,6 +1508,7 @@ class Notification(db.Model): created_at = db.Column(db.DateTime, index=True, unique=False, nullable=False) sent_at = db.Column(db.DateTime, index=False, unique=False, nullable=True) sent_by = db.Column(db.String, nullable=True) + message_cost = db.Column(db.Float, nullable=True, default=0.0) updated_at = db.Column( db.DateTime, index=False, @@ -1813,6 +1814,7 @@ class NotificationHistory(db.Model, HistoryModel): created_at = db.Column(db.DateTime, unique=False, nullable=False) sent_at = db.Column(db.DateTime, index=False, unique=False, nullable=True) sent_by = db.Column(db.String, nullable=True) + message_cost = db.Column(db.Float, nullable=True, default=0.0) updated_at = db.Column( db.DateTime, index=False, diff --git a/migrations/versions/0415_add_message_cost.py b/migrations/versions/0415_add_message_cost.py new file mode 100644 index 000000000..1a86170d7 --- /dev/null +++ b/migrations/versions/0415_add_message_cost.py @@ -0,0 +1,23 @@ +""" + +Revision ID: 0415_add_message_cost +Revises: 0414_change_total_message_limit +Create Date: 2025-02-28 11:35:22.873930 + +""" + +import sqlalchemy as sa +from alembic import op + +down_revision = "0414_change_total_message_limit" +revision = "0415_add_message_cost" + + +def upgrade(): + op.add_column("notifications", sa.Column("message_cost", sa.Float)) + op.add_column("notification_history", sa.Column("message_cost", sa.Float)) + + +def downgrade(): + op.drop_column("notifications", "message_cost") + op.add_column("notification_history", sa.Column("message_cost", sa.Float)) diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index 57673e6b4..e18e31f1b 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -571,9 +571,9 @@ def test_purge_bucket(mocker): mock_s3_resource = MagicMock() mock_bucket = MagicMock() mock_s3_resource.Bucket.return_value = mock_bucket - mocker.patch('app.aws.s3.get_s3_resource', return_value=mock_s3_resource) + mocker.patch("app.aws.s3.get_s3_resource", return_value=mock_s3_resource) - purge_bucket('my-bucket', 'access-key', 'secret-key', 'region') + purge_bucket("my-bucket", "access-key", "secret-key", "region") # Assert that the bucket's objects.all().delete() method was called mock_bucket.objects.all.return_value.delete.assert_called_once() @@ -613,8 +613,7 @@ def test_get_s3_files_handles_exception(mocker): # Make the first call succeed, second call should fail. mock_read_s3_file = mocker.patch( - "app.aws.s3.read_s3_file", - side_effect=[None, Exception("exception here")] + "app.aws.s3.read_s3_file", side_effect=[None, Exception("exception here")] ) mock_thread_pool_executor = mocker.patch("app.aws.s3.ThreadPoolExecutor") diff --git a/tests/app/clients/test_aws_cloudwatch.py b/tests/app/clients/test_aws_cloudwatch.py index 7a0379454..36de2c1b1 100644 --- a/tests/app/clients/test_aws_cloudwatch.py +++ b/tests/app/clients/test_aws_cloudwatch.py @@ -31,7 +31,7 @@ def side_effect(filterPattern, logGroupName, startTime, endTime): { "logStreamName": "89db9712-c6d1-49f9-be7c-4caa7ed9efb1", "message": '{"delivery":{"destination":"+1661","phoneCarrier":"ATT Mobility", ' - '"providerResponse":"Invalid phone number"}}', + '"providerResponse":"Invalid phone number", "priceInUSD": "0.00881"}}', "eventId": "37535432778099870001723210579798865345508698025292922880", } ] @@ -44,7 +44,7 @@ def side_effect(filterPattern, logGroupName, startTime, endTime): "logStreamName": "89db9712-c6d1-49f9-be7c-4caa7ed9efb1", "timestamp": 1683147017911, "message": '{"delivery":{"destination":"+1661","phoneCarrier":"ATT Mobility",' - '"providerResponse":"Phone accepted msg"}}', + '"providerResponse":"Phone accepted msg", "priceInUSD": "0.00881"}}', "ingestionTime": 1683147018026, "eventId": "37535432778099870001723210579798865345508698025292922880", } @@ -131,6 +131,7 @@ def test_event_to_db_format_with_missing_fields(): "status": "UNKNOWN", "delivery.phoneCarrier": "", "delivery.providerResponse": "", + "delivery.priceInUSD": 0.0, "@timestamp": "", } @@ -140,7 +141,11 @@ def test_event_to_db_format_with_string_input(): { "notification": {"messageId": "67890", "timestamp": "2024-01-01T14:00:00Z"}, "status": "FAILED", - "delivery": {"phoneCarrier": "Verizon", "providerResponse": "Error"}, + "delivery": { + "phoneCarrier": "Verizon", + "providerResponse": "Error", + "priceInUSD": "0.00881", + }, } ) result = aws_cloudwatch_client.event_to_db_format(event) @@ -149,5 +154,6 @@ def test_event_to_db_format_with_string_input(): "status": "FAILED", "delivery.phoneCarrier": "Verizon", "delivery.providerResponse": "Error", + "delivery.priceInUSD": 0.00881, "@timestamp": "2024-01-01T14:00:00Z", } diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index facf23d9f..4df57ec07 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -2019,8 +2019,8 @@ def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_ def test_update_delivery_receipts(mocker): mock_session = mocker.patch("app.dao.notifications_dao.db.session") receipts = [ - '{"notification.messageId": "msg1", "delivery.phoneCarrier": "carrier1", "delivery.providerResponse": "resp1", "@timestamp": "2024-01-01T12:00:00"}', # noqa - '{"notification.messageId": "msg2", "delivery.phoneCarrier": "carrier2", "delivery.providerResponse": "resp2", "@timestamp": "2024-01-01T13:00:00"}', # noqa + '{"notification.messageId": "msg1", "delivery.phoneCarrier": "carrier1", "delivery.providerResponse": "resp1", "@timestamp": "2024-01-01T12:00:00", "delivery.priceInUSD": "0.00881"}', # noqa + '{"notification.messageId": "msg2", "delivery.phoneCarrier": "carrier2", "delivery.providerResponse": "resp2", "@timestamp": "2024-01-01T13:00:00", "delivery.priceInUSD": "0.00881"}', # noqa ] delivered = True mock_update = MagicMock() @@ -2136,7 +2136,9 @@ def test_sanitize_successful_notification_by_id(): ) -def test_dao_get_notifications_by_recipient_or_reference_covers_sms_search_by_reference(notify_db_session): +def test_dao_get_notifications_by_recipient_or_reference_covers_sms_search_by_reference( + notify_db_session, +): """ This test: 1. Creates a service and an SMS template. @@ -2153,7 +2155,7 @@ def test_dao_get_notifications_by_recipient_or_reference_covers_sms_search_by_re data = { "id": uuid.uuid4(), "to": "1", - "normalised_to": "1", # phone is irrelevant here + "normalised_to": "1", # phone is irrelevant here "service_id": service.id, "service": service, "template_id": template.id, diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index 5b9a7d695..bad3a5162 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -47,14 +47,19 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session): created_at=datetime(2018, 1, 1, 1, x, 0), status=NotificationStatus.DELIVERED, ) - create_notification( + whats_this = create_notification( service_1.templates[0], created_at=datetime(2018, 1, 1, 1, 1, 0) ) - create_notification( + print(f"WTN status = {whats_this.status} type = {whats_this.notification_type}") + questionable_notification = create_notification( service_1.templates[1], created_at=datetime(2018, 1, 1, 1, 1, 0), status=NotificationStatus.DELIVERED, ) + print( + f"QN status = {questionable_notification.status} type = {questionable_notification.notification_type}" + ) + create_notification( service_1.templates[0], created_at=datetime(2018, 2, 1, 1, 1, 0), diff --git a/tests/app/organization/test_rest.py b/tests/app/organization/test_rest.py index 445a47297..d128630bd 100644 --- a/tests/app/organization/test_rest.py +++ b/tests/app/organization/test_rest.py @@ -831,7 +831,9 @@ def test_get_organization_users_returns_users_for_organization( ) assert len(response["data"]) == 2 - assert response["data"][0]["id"] == str(first.id) + response_ids = [response["data"][0]["id"], response["data"][1]["id"]] + assert str(first.id) in response_ids + assert str(second.id) in response_ids @freeze_time("2019-12-24 13:30")