darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-01 04:08:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

imports cleanup

Browse Source
This commit is contained in:
stvnrlly
2022-12-07 15:20:28 -05:00
parent 3130feacf2
commit d6f35cf6da
12 changed files with 11 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
app/celery/nightly_tasks.py
View File

@@ -1,15 +1,10 @@
from datetime import datetime, timedelta
import pytz
from flask import current_app
from notifications_utils.clients.zendesk.zendesk_client import (
NotifySupportTicket,
)
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import func
from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery, statsd_client, zendesk_client
from app import notify_celery, statsd_client
from app.aws import s3
from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
from app.config import QueueNames
@@ -29,15 +24,7 @@ from app.dao.notifications_dao import (
from app.dao.service_data_retention_dao import (
fetch_service_data_retention_for_all_services_by_notification_type,
)
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
LETTER_TYPE,
NOTIFICATION_SENDING,
SMS_TYPE,
FactProcessingTime,
Notification,
)
from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE, FactProcessingTime
from app.utils import get_local_midnight_in_utc

5
app/celery/research_mode_tasks.py
View File

@@ -1,13 +1,8 @@
import json
import random
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.s3 import s3upload
from requests import HTTPError, request
from app import notify_celery
from app.aws.s3 import file_exists
from app.celery.process_ses_receipts_tasks import process_ses_results
from app.config import QueueNames
from app.models import SMS_TYPE

11
app/celery/scheduled_tasks.py
View File

@@ -8,14 +8,13 @@ from sqlalchemy import between
from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery, zendesk_client
from app.aws import s3
from app.celery.tasks import (
get_recipient_csv_and_template_and_sender_id,
process_incomplete_jobs,
process_job,
process_row,
)
from app.config import QueueNames, TaskNames
from app.config import QueueNames
from app.dao.invited_org_user_dao import (
delete_org_invitations_created_more_than_two_days_ago,
)
@@ -28,12 +27,7 @@ from app.dao.jobs_dao import (
find_jobs_with_missing_rows,
find_missing_row_for_job,
)
from app.dao.notifications_dao import (
dao_old_letters_with_created_status,
dao_precompiled_letters_still_pending_virus_check,
letters_missing_from_sending_bucket,
notifications_not_yet_sent,
)
from app.dao.notifications_dao import notifications_not_yet_sent
from app.dao.provider_details_dao import (
dao_adjust_provider_priority_back_to_resting_points,
)
@@ -42,7 +36,6 @@ from app.dao.services_dao import (
dao_find_services_with_high_failure_rates,
)
from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
from app.letters.utils import generate_letter_pdf_filename
from app.models import (
EMAIL_TYPE,
JOB_STATUS_ERROR,

18
app/celery/tasks.py
View File

@@ -1,16 +1,12 @@
import json
from collections import defaultdict, namedtuple
from datetime import datetime
from flask import current_app
from notifications_utils.insensitive_dict import InsensitiveDict
from notifications_utils.postal_address import PostalAddress
from notifications_utils.recipients import RecipientCSV
from notifications_utils.timezones import convert_utc_to_local_timezone
from requests import HTTPError, RequestException, request
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
from app import create_random_identifier, create_uuid, encryption, notify_celery
from app import create_uuid, encryption, notify_celery
from app.aws import s3
from app.celery import provider_tasks
from app.config import QueueNames
@@ -18,10 +14,8 @@ from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
from app.dao.notifications_dao import (
dao_get_last_notification_added_for_job_id,
dao_get_notification_history_by_reference,
dao_update_notifications_by_reference,
get_notification_by_id,
update_notification_status_by_reference,
)
from app.dao.returned_letters_dao import insert_or_update_returned_letters
from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
@@ -29,28 +23,20 @@ from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
from app.dao.templates_dao import dao_get_template_by_id
from app.models import (
DVLA_RESPONSE_STATUS_SENT,
EMAIL_TYPE,
JOB_STATUS_CANCELLED,
JOB_STATUS_FINISHED,
JOB_STATUS_IN_PROGRESS,
JOB_STATUS_PENDING,
KEY_TYPE_NORMAL,
LETTER_TYPE,
NOTIFICATION_CREATED,
NOTIFICATION_DELIVERED,
NOTIFICATION_RETURNED_LETTER,
NOTIFICATION_SENDING,
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
DailySortedLetter,
)
from app.notifications.process_notifications import persist_notification
from app.notifications.validators import check_service_over_daily_message_limit
from app.serialised_models import SerialisedService, SerialisedTemplate
from app.service.utils import service_allowed_to_send_to
from app.utils import DATETIME_FORMAT, get_reference_from_personalisation
from app.utils import DATETIME_FORMAT
from app.v2.errors import TooManyRequestsError

1
app/commands.py
View File

@@ -20,7 +20,6 @@ from sqlalchemy.orm.exc import NoResultFound
from app import db
from app.aws import s3
from app.celery.tasks import process_row
from app.config import QueueNames
from app.dao.annual_billing_dao import (
dao_create_or_update_annual_billing_for_year,
set_default_free_allowance_for_service,

48
app/dao/notifications_dao.py
View File

@@ -10,10 +10,7 @@ from notifications_utils.recipients import (
try_validate_and_format_phone_number,
validate_and_format_email_address,
)
from notifications_utils.timezones import (
convert_local_timezone_to_utc,
convert_utc_to_local_timezone,
)
from notifications_utils.timezones import convert_local_timezone_to_utc
from sqlalchemy import asc, desc, func, or_, union
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound
@@ -654,49 +651,6 @@ def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
return notifications
def dao_old_letters_with_created_status():
yesterday_bst = convert_utc_to_local_timezone(datetime.utcnow()) - timedelta(days=1)
last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)
notifications = Notification.query.filter(
Notification.created_at < convert_local_timezone_to_utc(last_processing_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED
).order_by(
Notification.created_at
).all()
return notifications
def letters_missing_from_sending_bucket(seconds_to_subtract):
older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract)
# We expect letters to have a `created` status, updated_at timestamp and billable units greater than zero.
notifications = Notification.query.filter(
Notification.billable_units == 0,
Notification.updated_at == None, # noqa
Notification.status == NOTIFICATION_CREATED,
Notification.created_at <= older_than_date,
Notification.notification_type == LETTER_TYPE,
Notification.key_type == KEY_TYPE_NORMAL
).order_by(
Notification.created_at
).all()
return notifications
def dao_precompiled_letters_still_pending_virus_check():
ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)
notifications = Notification.query.filter(
Notification.created_at < ninety_minutes_ago,
Notification.status == NOTIFICATION_PENDING_VIRUS_CHECK
).order_by(
Notification.created_at
).all()
return notifications
def _duplicate_update_warning(notification, status):
current_app.logger.info(
(

2
app/models.py
View File

@@ -1368,8 +1368,6 @@ NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notif
NOTIFICATION_STATUS_LETTER_ACCEPTED = 'accepted'
NOTIFICATION_STATUS_LETTER_RECEIVED = 'received'
DVLA_RESPONSE_STATUS_SENT = 'Sent'
FIRST_CLASS = 'first'
SECOND_CLASS = 'second'
EUROPE = 'europe'

2
app/v2/notifications/post_notifications.py
View File

@@ -368,8 +368,6 @@ def process_letter_notification(
status = NOTIFICATION_DELIVERED
updated_at = datetime.utcnow()
queue = QueueNames.CREATE_LETTERS_PDF if not test_key else QueueNames.RESEARCH_MODE
notification = create_letter_notification(letter_data=letter_data,
service=service,
template=template,

57
tests/app/celery/test_nightly_tasks.py
View File

@@ -2,12 +2,7 @@ from datetime import date, datetime, timedelta
from unittest.mock import ANY, call
import pytest
import pytz
from flask import current_app
from freezegun import freeze_time
from notifications_utils.clients.zendesk.zendesk_client import (
NotifySupportTicket,
)
from app.celery import nightly_tasks
from app.celery.nightly_tasks import (
@@ -192,58 +187,6 @@ def test_delete_inbound_sms_calls_child_task(notify_api, mocker):
assert nightly_tasks.delete_inbound_sms_older_than_retention.call_count == 1
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match)
letter_raise_alert_if_no_ack_file_for_zip()
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # Datatime format on AWS
subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent'
assert mock_file_list.call_count == 2
assert mock_file_list.call_args_list == [
call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'),
call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch',
suffix='.ACK.txt', last_modified=yesterday),
]
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
mock_create_ticket.assert_called_once_with(
ANY,
subject="Letter acknowledge error",
message=ANY,
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
assert "['NOTIFY.2018-01-11175009', 'NOTIFY.2018-01-11175010']" in mock_create_ticket.call_args[1]['message']
assert '2018-01-11/zips_sent' in mock_create_ticket.call_args[1]['message']
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
@freeze_time('2021-01-18T02:00')
@pytest.mark.parametrize('date_provided', [None, '2021-1-17'])
def test_save_daily_notification_processing_time(mocker, sample_template, date_provided):

8
tests/app/celery/test_research_mode_tasks.py
View File

@@ -1,10 +1,8 @@
import uuid
from unittest.mock import ANY, call
from unittest.mock import ANY
import pytest
import requests_mock
from flask import current_app, json
from freezegun import freeze_time
from flask import json
from app.celery.research_mode_tasks import (
HTTPError,
@@ -15,7 +13,7 @@ from app.celery.research_mode_tasks import (
ses_notification_callback,
)
from app.config import QueueNames
from tests.conftest import Matcher, set_config_values
from tests.conftest import Matcher
dvla_response_file_matcher = Matcher(
'dvla_response_file',

158
tests/app/celery/test_scheduled_tasks.py
View File

@@ -4,7 +4,6 @@ from unittest import mock
from unittest.mock import ANY, call
import pytest
from freezegun import freeze_time
from notifications_utils.clients.zendesk.zendesk_client import (
NotifySupportTicket,
)
@@ -19,15 +18,13 @@ from app.celery.scheduled_tasks import (
replay_created_notifications,
run_scheduled_jobs,
)
from app.config import QueueNames, TaskNames, Test
from app.config import QueueNames, Test
from app.dao.jobs_dao import dao_get_job_by_id
from app.models import (
JOB_STATUS_ERROR,
JOB_STATUS_FINISHED,
JOB_STATUS_IN_PROGRESS,
JOB_STATUS_PENDING,
NOTIFICATION_DELIVERED,
NOTIFICATION_PENDING_VIRUS_CHECK,
)
from tests.app import load_example_csv
from tests.app.db import create_job, create_notification, create_template
@@ -275,159 +272,6 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
check_job_status()
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_pending_virus_check_restarts_scan_for_stuck_letters(
mocker,
sample_letter_template
):
mock_file_exists = mocker.patch('app.aws.s3.file_exists', return_value=True)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task')
create_notification(
template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=5401),
reference='one'
)
expected_filename = 'NOTIFY.ONE.D.2.C.20190530122959.PDF'
check_if_letters_still_pending_virus_check()
mock_file_exists.assert_called_once_with('test-letters-scan', expected_filename)
mock_celery.assert_called_once_with(
name=TaskNames.SCAN_FILE,
kwargs={'filename': expected_filename},
queue=QueueNames.ANTIVIRUS
)
assert mock_create_ticket.called is False
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_pending_virus_check_raises_zendesk_if_files_cant_be_found(
mocker,
sample_letter_template
):
mock_file_exists = mocker.patch('app.aws.s3.file_exists', return_value=False)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
create_notification(template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=5400))
create_notification(template=sample_letter_template,
status=NOTIFICATION_DELIVERED,
created_at=datetime.utcnow() - timedelta(seconds=6000))
notification_1 = create_notification(template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=5401),
reference='one')
notification_2 = create_notification(template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=70000),
reference='two')
check_if_letters_still_pending_virus_check()
assert mock_file_exists.call_count == 2
mock_file_exists.assert_has_calls([
call('test-letters-scan', 'NOTIFY.ONE.D.2.C.20190530122959.PDF'),
call('test-letters-scan', 'NOTIFY.TWO.D.2.C.20190529183320.PDF'),
], any_order=True)
assert mock_celery.called is False
mock_create_ticket.assert_called_once_with(
ANY,
subject='[test] Letters still pending virus check',
message=ANY,
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
assert '2 precompiled letters have been pending-virus-check' in mock_create_ticket.call_args.kwargs['message']
assert f'{(str(notification_1.id), notification_1.reference)}' in mock_create_ticket.call_args.kwargs['message']
assert f'{(str(notification_2.id), notification_2.reference)}' in mock_create_ticket.call_args.kwargs['message']
mock_send_ticket_to_zendesk.assert_called_once()
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_in_created_during_bst(mocker, sample_letter_template):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29))
create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))
check_if_letters_still_in_created()
message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
"Follow runbook to resolve: " \
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created."
mock_logger.assert_called_once_with(message)
mock_create_ticket.assert_called_with(
ANY,
message=message,
subject="[test] Letters still in 'created' status",
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
@freeze_time("2019-01-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_in_created_during_utc(mocker, sample_letter_template):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29))
create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))
check_if_letters_still_in_created()
message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
"Follow runbook to resolve: " \
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created."
mock_logger.assert_called_once_with(message)
mock_create_ticket.assert_called_once_with(
ANY,
message=message,
subject="[test] Letters still in 'created' status",
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
@pytest.mark.parametrize('offset', (
timedelta(days=1),
pytest.param(timedelta(hours=23, minutes=59), marks=pytest.mark.xfail),

2
tests/app/celery/test_tasks.py
View File

@@ -66,7 +66,6 @@ from tests.app.db import (
create_template,
create_user,
)
from tests.conftest import set_config_values
class AnyStringWith(str):
@@ -931,7 +930,6 @@ def test_save_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample
assert not retry.called
def test_save_sms_uses_sms_sender_reply_to_text(mocker, notify_db_session):
service = create_service_with_defined_sms_sender(sms_sender_value='07123123123')
template = create_template(service=service)