From d6f35cf6daf57bbf8fc2fee522fa18f1635560b3 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 7 Dec 2022 15:20:28 -0500 Subject: [PATCH] imports cleanup --- app/celery/nightly_tasks.py | 17 +- app/celery/research_mode_tasks.py | 5 - app/celery/scheduled_tasks.py | 11 +- app/celery/tasks.py | 18 +-- app/commands.py | 1 - app/dao/notifications_dao.py | 48 +----- app/models.py | 2 - app/v2/notifications/post_notifications.py | 2 - tests/app/celery/test_nightly_tasks.py | 57 ------- tests/app/celery/test_research_mode_tasks.py | 8 +- tests/app/celery/test_scheduled_tasks.py | 158 +------------------ tests/app/celery/test_tasks.py | 2 - 12 files changed, 11 insertions(+), 318 deletions(-) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index 13b5a5e25..f71f7fb14 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -1,15 +1,10 @@ from datetime import datetime, timedelta -import pytz from flask import current_app -from notifications_utils.clients.zendesk.zendesk_client import ( - NotifySupportTicket, -) from notifications_utils.timezones import convert_utc_to_local_timezone -from sqlalchemy import func from sqlalchemy.exc import SQLAlchemyError -from app import notify_celery, statsd_client, zendesk_client +from app import notify_celery, statsd_client from app.aws import s3 from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task from app.config import QueueNames @@ -29,15 +24,7 @@ from app.dao.notifications_dao import ( from app.dao.service_data_retention_dao import ( fetch_service_data_retention_for_all_services_by_notification_type, ) -from app.models import ( - EMAIL_TYPE, - KEY_TYPE_NORMAL, - LETTER_TYPE, - NOTIFICATION_SENDING, - SMS_TYPE, - FactProcessingTime, - Notification, -) +from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE, FactProcessingTime from app.utils import get_local_midnight_in_utc diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py index 8d4e802c4..6ed35c1fa 100644 --- a/app/celery/research_mode_tasks.py +++ b/app/celery/research_mode_tasks.py @@ -1,13 +1,8 @@ import json -import random -from datetime import datetime, timedelta from flask import current_app -from notifications_utils.s3 import s3upload from requests import HTTPError, request -from app import notify_celery -from app.aws.s3 import file_exists from app.celery.process_ses_receipts_tasks import process_ses_results from app.config import QueueNames from app.models import SMS_TYPE diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index b337f2397..d3e76e584 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -8,14 +8,13 @@ from sqlalchemy import between from sqlalchemy.exc import SQLAlchemyError from app import notify_celery, zendesk_client -from app.aws import s3 from app.celery.tasks import ( get_recipient_csv_and_template_and_sender_id, process_incomplete_jobs, process_job, process_row, ) -from app.config import QueueNames, TaskNames +from app.config import QueueNames from app.dao.invited_org_user_dao import ( delete_org_invitations_created_more_than_two_days_ago, ) @@ -28,12 +27,7 @@ from app.dao.jobs_dao import ( find_jobs_with_missing_rows, find_missing_row_for_job, ) -from app.dao.notifications_dao import ( - dao_old_letters_with_created_status, - dao_precompiled_letters_still_pending_virus_check, - letters_missing_from_sending_bucket, - notifications_not_yet_sent, -) +from app.dao.notifications_dao import notifications_not_yet_sent from app.dao.provider_details_dao import ( dao_adjust_provider_priority_back_to_resting_points, ) @@ -42,7 +36,6 @@ from app.dao.services_dao import ( dao_find_services_with_high_failure_rates, ) from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago -from app.letters.utils import generate_letter_pdf_filename from app.models import ( EMAIL_TYPE, JOB_STATUS_ERROR, diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 8665bf5a0..fca5a49cf 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,16 +1,12 @@ import json -from collections import defaultdict, namedtuple from datetime import datetime from flask import current_app -from notifications_utils.insensitive_dict import InsensitiveDict -from notifications_utils.postal_address import PostalAddress from notifications_utils.recipients import RecipientCSV -from notifications_utils.timezones import convert_utc_to_local_timezone from requests import HTTPError, RequestException, request from sqlalchemy.exc import IntegrityError, SQLAlchemyError -from app import create_random_identifier, create_uuid, encryption, notify_celery +from app import create_uuid, encryption, notify_celery from app.aws import s3 from app.celery import provider_tasks from app.config import QueueNames @@ -18,10 +14,8 @@ from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job from app.dao.notifications_dao import ( dao_get_last_notification_added_for_job_id, - dao_get_notification_history_by_reference, dao_update_notifications_by_reference, get_notification_by_id, - update_notification_status_by_reference, ) from app.dao.returned_letters_dao import insert_or_update_returned_letters from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id @@ -29,28 +23,20 @@ from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id from app.dao.templates_dao import dao_get_template_by_id from app.models import ( - DVLA_RESPONSE_STATUS_SENT, EMAIL_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_FINISHED, JOB_STATUS_IN_PROGRESS, JOB_STATUS_PENDING, KEY_TYPE_NORMAL, - LETTER_TYPE, - NOTIFICATION_CREATED, - NOTIFICATION_DELIVERED, NOTIFICATION_RETURNED_LETTER, - NOTIFICATION_SENDING, - NOTIFICATION_TECHNICAL_FAILURE, - NOTIFICATION_TEMPORARY_FAILURE, SMS_TYPE, - DailySortedLetter, ) from app.notifications.process_notifications import persist_notification from app.notifications.validators import check_service_over_daily_message_limit from app.serialised_models import SerialisedService, SerialisedTemplate from app.service.utils import service_allowed_to_send_to -from app.utils import DATETIME_FORMAT, get_reference_from_personalisation +from app.utils import DATETIME_FORMAT from app.v2.errors import TooManyRequestsError diff --git a/app/commands.py b/app/commands.py index 4817f3cdb..f88e6c0b1 100644 --- a/app/commands.py +++ b/app/commands.py @@ -20,7 +20,6 @@ from sqlalchemy.orm.exc import NoResultFound from app import db from app.aws import s3 from app.celery.tasks import process_row -from app.config import QueueNames from app.dao.annual_billing_dao import ( dao_create_or_update_annual_billing_for_year, set_default_free_allowance_for_service, diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 47db5aba7..87e77a7d0 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -10,10 +10,7 @@ from notifications_utils.recipients import ( try_validate_and_format_phone_number, validate_and_format_email_address, ) -from notifications_utils.timezones import ( - convert_local_timezone_to_utc, - convert_utc_to_local_timezone, -) +from notifications_utils.timezones import convert_local_timezone_to_utc from sqlalchemy import asc, desc, func, or_, union from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound @@ -654,49 +651,6 @@ def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline): return notifications -def dao_old_letters_with_created_status(): - yesterday_bst = convert_utc_to_local_timezone(datetime.utcnow()) - timedelta(days=1) - last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0) - - notifications = Notification.query.filter( - Notification.created_at < convert_local_timezone_to_utc(last_processing_deadline), - Notification.notification_type == LETTER_TYPE, - Notification.status == NOTIFICATION_CREATED - ).order_by( - Notification.created_at - ).all() - return notifications - - -def letters_missing_from_sending_bucket(seconds_to_subtract): - older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract) - # We expect letters to have a `created` status, updated_at timestamp and billable units greater than zero. - notifications = Notification.query.filter( - Notification.billable_units == 0, - Notification.updated_at == None, # noqa - Notification.status == NOTIFICATION_CREATED, - Notification.created_at <= older_than_date, - Notification.notification_type == LETTER_TYPE, - Notification.key_type == KEY_TYPE_NORMAL - ).order_by( - Notification.created_at - ).all() - - return notifications - - -def dao_precompiled_letters_still_pending_virus_check(): - ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400) - - notifications = Notification.query.filter( - Notification.created_at < ninety_minutes_ago, - Notification.status == NOTIFICATION_PENDING_VIRUS_CHECK - ).order_by( - Notification.created_at - ).all() - return notifications - - def _duplicate_update_warning(notification, status): current_app.logger.info( ( diff --git a/app/models.py b/app/models.py index e2c5c5bed..098857644 100644 --- a/app/models.py +++ b/app/models.py @@ -1368,8 +1368,6 @@ NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notif NOTIFICATION_STATUS_LETTER_ACCEPTED = 'accepted' NOTIFICATION_STATUS_LETTER_RECEIVED = 'received' -DVLA_RESPONSE_STATUS_SENT = 'Sent' - FIRST_CLASS = 'first' SECOND_CLASS = 'second' EUROPE = 'europe' diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index c06635dcb..f3d127baa 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -368,8 +368,6 @@ def process_letter_notification( status = NOTIFICATION_DELIVERED updated_at = datetime.utcnow() - queue = QueueNames.CREATE_LETTERS_PDF if not test_key else QueueNames.RESEARCH_MODE - notification = create_letter_notification(letter_data=letter_data, service=service, template=template, diff --git a/tests/app/celery/test_nightly_tasks.py b/tests/app/celery/test_nightly_tasks.py index 2c9cbfd25..070d23b09 100644 --- a/tests/app/celery/test_nightly_tasks.py +++ b/tests/app/celery/test_nightly_tasks.py @@ -2,12 +2,7 @@ from datetime import date, datetime, timedelta from unittest.mock import ANY, call import pytest -import pytz -from flask import current_app from freezegun import freeze_time -from notifications_utils.clients.zendesk.zendesk_client import ( - NotifySupportTicket, -) from app.celery import nightly_tasks from app.celery.nightly_tasks import ( @@ -192,58 +187,6 @@ def test_delete_inbound_sms_calls_child_task(notify_api, mocker): assert nightly_tasks.delete_inbound_sms_older_than_retention.call_count == 1 -@freeze_time('2018-01-11T23:00:00') -@pytest.mark.skip(reason="Skipping letter-related functionality for now") -def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db_session): - mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match) - letter_raise_alert_if_no_ack_file_for_zip() - - yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # Datatime format on AWS - subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent' - assert mock_file_list.call_count == 2 - assert mock_file_list.call_args_list == [ - call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'), - call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', - suffix='.ACK.txt', last_modified=yesterday), - ] - - -@freeze_time('2018-01-11T23:00:00') -@pytest.mark.skip(reason="Skipping letter-related functionality for now") -def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db_session): - mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff) - mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__') - mock_send_ticket_to_zendesk = mocker.patch( - 'app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - letter_raise_alert_if_no_ack_file_for_zip() - - assert mock_file_list.call_count == 2 - - mock_create_ticket.assert_called_once_with( - ANY, - subject="Letter acknowledge error", - message=ANY, - ticket_type='incident', - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - mock_send_ticket_to_zendesk.assert_called_once() - assert "['NOTIFY.2018-01-11175009', 'NOTIFY.2018-01-11175010']" in mock_create_ticket.call_args[1]['message'] - assert '2018-01-11/zips_sent' in mock_create_ticket.call_args[1]['message'] - - -@freeze_time('2018-01-11T23:00:00') -@pytest.mark.skip(reason="Skipping letter-related functionality for now") -def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db_session): - mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None) - letter_raise_alert_if_no_ack_file_for_zip() - - assert mock_file_list.call_count == 2 - - @freeze_time('2021-01-18T02:00') @pytest.mark.parametrize('date_provided', [None, '2021-1-17']) def test_save_daily_notification_processing_time(mocker, sample_template, date_provided): diff --git a/tests/app/celery/test_research_mode_tasks.py b/tests/app/celery/test_research_mode_tasks.py index c8fa93b48..6fa9a8d0a 100644 --- a/tests/app/celery/test_research_mode_tasks.py +++ b/tests/app/celery/test_research_mode_tasks.py @@ -1,10 +1,8 @@ import uuid -from unittest.mock import ANY, call +from unittest.mock import ANY import pytest -import requests_mock -from flask import current_app, json -from freezegun import freeze_time +from flask import json from app.celery.research_mode_tasks import ( HTTPError, @@ -15,7 +13,7 @@ from app.celery.research_mode_tasks import ( ses_notification_callback, ) from app.config import QueueNames -from tests.conftest import Matcher, set_config_values +from tests.conftest import Matcher dvla_response_file_matcher = Matcher( 'dvla_response_file', diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 181c4181c..c9b66a3d3 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -4,7 +4,6 @@ from unittest import mock from unittest.mock import ANY, call import pytest -from freezegun import freeze_time from notifications_utils.clients.zendesk.zendesk_client import ( NotifySupportTicket, ) @@ -19,15 +18,13 @@ from app.celery.scheduled_tasks import ( replay_created_notifications, run_scheduled_jobs, ) -from app.config import QueueNames, TaskNames, Test +from app.config import QueueNames, Test from app.dao.jobs_dao import dao_get_job_by_id from app.models import ( JOB_STATUS_ERROR, JOB_STATUS_FINISHED, JOB_STATUS_IN_PROGRESS, JOB_STATUS_PENDING, - NOTIFICATION_DELIVERED, - NOTIFICATION_PENDING_VIRUS_CHECK, ) from tests.app import load_example_csv from tests.app.db import create_job, create_notification, create_template @@ -275,159 +272,6 @@ def test_check_job_status_task_does_not_raise_error(sample_template): check_job_status() -@freeze_time("2019-05-30 14:00:00") -@pytest.mark.skip(reason="Skipping letter-related functionality for now") -def test_check_if_letters_still_pending_virus_check_restarts_scan_for_stuck_letters( - mocker, - sample_letter_template -): - mock_file_exists = mocker.patch('app.aws.s3.file_exists', return_value=True) - mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__') - mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task') - - create_notification( - template=sample_letter_template, - status=NOTIFICATION_PENDING_VIRUS_CHECK, - created_at=datetime.utcnow() - timedelta(seconds=5401), - reference='one' - ) - expected_filename = 'NOTIFY.ONE.D.2.C.20190530122959.PDF' - - check_if_letters_still_pending_virus_check() - - mock_file_exists.assert_called_once_with('test-letters-scan', expected_filename) - - mock_celery.assert_called_once_with( - name=TaskNames.SCAN_FILE, - kwargs={'filename': expected_filename}, - queue=QueueNames.ANTIVIRUS - ) - - assert mock_create_ticket.called is False - - -@freeze_time("2019-05-30 14:00:00") -@pytest.mark.skip(reason="Skipping letter-related functionality for now") -def test_check_if_letters_still_pending_virus_check_raises_zendesk_if_files_cant_be_found( - mocker, - sample_letter_template -): - mock_file_exists = mocker.patch('app.aws.s3.file_exists', return_value=False) - mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__') - mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task') - mock_send_ticket_to_zendesk = mocker.patch( - 'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - create_notification(template=sample_letter_template, - status=NOTIFICATION_PENDING_VIRUS_CHECK, - created_at=datetime.utcnow() - timedelta(seconds=5400)) - create_notification(template=sample_letter_template, - status=NOTIFICATION_DELIVERED, - created_at=datetime.utcnow() - timedelta(seconds=6000)) - notification_1 = create_notification(template=sample_letter_template, - status=NOTIFICATION_PENDING_VIRUS_CHECK, - created_at=datetime.utcnow() - timedelta(seconds=5401), - reference='one') - notification_2 = create_notification(template=sample_letter_template, - status=NOTIFICATION_PENDING_VIRUS_CHECK, - created_at=datetime.utcnow() - timedelta(seconds=70000), - reference='two') - - check_if_letters_still_pending_virus_check() - - assert mock_file_exists.call_count == 2 - mock_file_exists.assert_has_calls([ - call('test-letters-scan', 'NOTIFY.ONE.D.2.C.20190530122959.PDF'), - call('test-letters-scan', 'NOTIFY.TWO.D.2.C.20190529183320.PDF'), - ], any_order=True) - assert mock_celery.called is False - - mock_create_ticket.assert_called_once_with( - ANY, - subject='[test] Letters still pending virus check', - message=ANY, - ticket_type='incident', - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - assert '2 precompiled letters have been pending-virus-check' in mock_create_ticket.call_args.kwargs['message'] - assert f'{(str(notification_1.id), notification_1.reference)}' in mock_create_ticket.call_args.kwargs['message'] - assert f'{(str(notification_2.id), notification_2.reference)}' in mock_create_ticket.call_args.kwargs['message'] - mock_send_ticket_to_zendesk.assert_called_once() - - -@freeze_time("2019-05-30 14:00:00") -@pytest.mark.skip(reason="Skipping letter-related functionality for now") -def test_check_if_letters_still_in_created_during_bst(mocker, sample_letter_template): - mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error') - mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__') - mock_send_ticket_to_zendesk = mocker.patch( - 'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29)) - create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0)) - - check_if_letters_still_in_created() - - message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \ - "Follow runbook to resolve: " \ - "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created." - - mock_logger.assert_called_once_with(message) - mock_create_ticket.assert_called_with( - ANY, - message=message, - subject="[test] Letters still in 'created' status", - ticket_type='incident', - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - mock_send_ticket_to_zendesk.assert_called_once() - - -@freeze_time("2019-01-30 14:00:00") -@pytest.mark.skip(reason="Skipping letter-related functionality for now") -def test_check_if_letters_still_in_created_during_utc(mocker, sample_letter_template): - mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error') - mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__') - mock_send_ticket_to_zendesk = mocker.patch( - 'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29)) - create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0)) - create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0)) - - check_if_letters_still_in_created() - - message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \ - "Follow runbook to resolve: " \ - "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created." - - mock_logger.assert_called_once_with(message) - mock_create_ticket.assert_called_once_with( - ANY, - message=message, - subject="[test] Letters still in 'created' status", - ticket_type='incident', - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - mock_send_ticket_to_zendesk.assert_called_once() - - @pytest.mark.parametrize('offset', ( timedelta(days=1), pytest.param(timedelta(hours=23, minutes=59), marks=pytest.mark.xfail), diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 66dbc057a..a5965d84c 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -66,7 +66,6 @@ from tests.app.db import ( create_template, create_user, ) -from tests.conftest import set_config_values class AnyStringWith(str): @@ -931,7 +930,6 @@ def test_save_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample assert not retry.called - def test_save_sms_uses_sms_sender_reply_to_text(mocker, notify_db_session): service = create_service_with_defined_sms_sender(sms_sender_value='07123123123') template = create_template(service=service)