darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-15 15:28:02 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into move-folders

Browse Source
This commit is contained in:
Rebecca Law
2018-11-09 15:26:23 +00:00
parent 64f7f4c302 cd51818dfe
commit 5df9a307be
7 changed files with 99 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/celery/reporting_tasks.py
View File

@@ -15,7 +15,7 @@ from app.dao.fact_notification_status_dao import fetch_notification_status_for_d
@statsd(namespace="tasks")
def create_nightly_billing(day_start=None):
# day_start is a datetime.date() object. e.g.
# 3 days of data counting back from day_start is consolidated
# up to 10 days of data counting back from day_start is consolidated
if day_start is None:
day_start = datetime.today() - timedelta(days=1)
else:
@@ -37,13 +37,13 @@ def create_nightly_billing(day_start=None):
@statsd(namespace="tasks")
def create_nightly_notification_status(day_start=None):
# day_start is a datetime.date() object. e.g.
# 3 days of data counting back from day_start is consolidated
# 4 days of data counting back from day_start is consolidated
if day_start is None:
day_start = datetime.today() - timedelta(days=1)
else:
# When calling the task its a string in the format of "YYYY-MM-DD"
day_start = datetime.strptime(day_start, "%Y-%m-%d")
for i in range(0, 3):
for i in range(0, 4):
process_day = day_start - timedelta(days=i)
transit_data = fetch_notification_status_for_day(process_day=process_day)
@@ -51,4 +51,7 @@ def create_nightly_notification_status(day_start=None):
update_fact_notification_status(transit_data, process_day)
current_app.logger.info(
"create-nightly-notification-status task: {} rows updated for day: {}")
"create-nightly-notification-status task: {} rows updated for day: {}".format(
len(transit_data), process_day
)
)

9
app/commands.py
View File

@@ -527,6 +527,12 @@ def migrate_data_to_ft_notification_status(start_date, end_date):
while process_date < end_date:
start_time = datetime.now()
# migrate data into ft_notification_status and update if record already exists
db.session.execute(
'delete from ft_notification_status where bst_date = :process_date',
{"process_date": process_date}
)
sql = \
"""
insert into ft_notification_status (bst_date, template_id, service_id, job_id, notification_type, key_type,
@@ -546,9 +552,6 @@ def migrate_data_to_ft_notification_status(start_date, end_date):
and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
group by bst_date, template_id, service_id, job_id, notification_type, key_type, notification_status
order by bst_date
on conflict on constraint ft_notification_status_pkey do update set
notification_count = excluded.notification_count,
updated_at = now()
"""
result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
db.session.commit()

88
app/config.py
View File

@@ -175,49 +175,66 @@ class Config(object):
'schedule': timedelta(minutes=66),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-sms-notifications': {
'task': 'delete-sms-notifications',
'schedule': crontab(hour=0, minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-email-notifications': {
'task': 'delete-email-notifications',
'schedule': crontab(hour=0, minute=20),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-letter-notifications': {
'task': 'delete-letter-notifications',
'schedule': crontab(hour=0, minute=40),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',
'schedule': crontab(hour=1, minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'send-daily-performance-platform-stats': {
'task': 'send-daily-performance-platform-stats',
'schedule': crontab(hour=2, minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'switch-current-sms-provider-on-slow-delivery': {
'task': 'switch-current-sms-provider-on-slow-delivery',
'schedule': crontab(), # Every minute
'options': {'queue': QueueNames.PERIODIC}
},
'check-job-status': {
'task': 'check-job-status',
'schedule': crontab(),
'options': {'queue': QueueNames.PERIODIC}
},
'replay-created-notifications': {
'task': 'replay-created-notifications',
'schedule': crontab(minute='0, 15, 30, 45'),
'options': {'queue': QueueNames.PERIODIC}
},
# nightly tasks:
'timeout-sending-notifications': {
'task': 'timeout-sending-notifications',
'schedule': crontab(hour=3, minute=0),
'schedule': crontab(hour=0, minute=5),
'options': {'queue': QueueNames.PERIODIC}
},
'daily-stats-template-usage-by-month': {
'task': 'daily-stats-template-usage-by-month',
'schedule': crontab(hour=0, minute=10),
'options': {'queue': QueueNames.PERIODIC}
},
'create-nightly-billing': {
'task': 'create-nightly-billing',
'schedule': crontab(hour=3, minute=30),
'schedule': crontab(hour=0, minute=15),
'options': {'queue': QueueNames.PERIODIC}
},
'create-nightly-notification-status': {
'task': 'create-nightly-notification-status',
'schedule': crontab(hour=4, minute=30),
'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications'
'options': {'queue': QueueNames.PERIODIC}
},
'delete-sms-notifications': {
'task': 'delete-sms-notifications',
'schedule': crontab(hour=0, minute=45), # after 'create-nightly-notification-status'
'options': {'queue': QueueNames.PERIODIC}
},
'delete-email-notifications': {
'task': 'delete-email-notifications',
'schedule': crontab(hour=1, minute=0), # after 'create-nightly-notification-status'
'options': {'queue': QueueNames.PERIODIC}
},
'delete-letter-notifications': {
'task': 'delete-letter-notifications',
'schedule': crontab(hour=1, minute=20), # after 'create-nightly-notification-status'
'options': {'queue': QueueNames.PERIODIC}
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',
'schedule': crontab(hour=1, minute=40),
'options': {'queue': QueueNames.PERIODIC}
},
'send-daily-performance-platform-stats': {
'task': 'send-daily-performance-platform-stats',
'schedule': crontab(hour=2, minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'remove_sms_email_jobs': {
@@ -254,21 +271,6 @@ class Config(object):
'schedule': crontab(hour=23, minute=00),
'options': {'queue': QueueNames.PERIODIC}
},
'check-job-status': {
'task': 'check-job-status',
'schedule': crontab(),
'options': {'queue': QueueNames.PERIODIC}
},
'daily-stats-template-usage-by-month': {
'task': 'daily-stats-template-usage-by-month',
'schedule': crontab(hour=0, minute=5),
'options': {'queue': QueueNames.PERIODIC}
},
'replay-created-notifications': {
'task': 'replay-created-notifications',
'schedule': crontab(minute='0, 15, 30, 45'),
'options': {'queue': QueueNames.PERIODIC}
}
}
CELERY_QUEUES = []

16
app/dao/fact_notification_status_dao.py
View File

@@ -49,12 +49,9 @@ def fetch_notification_status_for_day(process_day, service_id=None):
def update_fact_notification_status(data, process_day):
table = FactNotificationStatus.__table__
'''
This uses the Postgres upsert to avoid race conditions when two threads try to insert
at the same row. The excluded object refers to values that we tried to insert but were
rejected.
http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
'''
FactNotificationStatus.query.filter(
FactNotificationStatus.bst_date == process_day.date()
).delete()
for row in data:
stmt = insert(table).values(
bst_date=process_day.date(),
@@ -66,13 +63,6 @@ def update_fact_notification_status(data, process_day):
notification_status=row.status,
notification_count=row.notification_count,
)
stmt = stmt.on_conflict_do_update(
constraint="ft_notification_status_pkey",
set_={"notification_count": stmt.excluded.notification_count,
"updated_at": datetime.utcnow()
}
)
db.session.connection().execute(stmt)
db.session.commit()

3
app/dao/templates_dao.py
View File

@@ -38,6 +38,9 @@ def dao_create_template(template):
@transactional
@version_class(Template, TemplateHistory)
def dao_update_template(template):
if template.archived:
template.folder = None
db.session.add(template)

10
tests/app/celery/test_reporting_tasks.py
View File

@@ -460,24 +460,24 @@ def test_create_nightly_notification_status(notify_db_session):
create_notification(template=first_template, status='delivered')
create_notification(template=first_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=1))
create_notification(template=first_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=2))
create_notification(template=first_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=3))
create_notification(template=first_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=first_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=5))
create_notification(template=second_template, status='temporary-failure')
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=1))
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=2))
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=3))
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=5))
create_notification(template=third_template, status='created')
create_notification(template=third_template, status='created', created_at=datetime.utcnow() - timedelta(days=1))
create_notification(template=third_template, status='created', created_at=datetime.utcnow() - timedelta(days=2))
create_notification(template=third_template, status='created', created_at=datetime.utcnow() - timedelta(days=3))
create_notification(template=third_template, status='created', created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=third_template, status='created', created_at=datetime.utcnow() - timedelta(days=5))
assert len(FactNotificationStatus.query.all()) == 0
@@ -487,6 +487,6 @@ def test_create_nightly_notification_status(notify_db_session):
FactNotificationStatus.notification_type
).all()
assert len(new_data) == 9
assert str(new_data[0].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=3), "%Y-%m-%d")
assert str(new_data[0].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=4), "%Y-%m-%d")
assert str(new_data[3].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=2), "%Y-%m-%d")
assert str(new_data[6].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=1), "%Y-%m-%d")

30
tests/app/dao/test_templates_dao.py
View File

@@ -15,6 +15,7 @@ from app.dao.templates_dao import (
)
from app.models import (
Template,
TemplateFolder,
TemplateHistory,
TemplateRedacted
)
@@ -93,6 +94,35 @@ def test_update_template(sample_service, sample_user):
assert dao_get_all_templates_for_service(sample_service.id)[0].name == 'new name'
def test_update_template_in_a_folder_to_archived(sample_service, sample_user):
template_data = {
'name': 'Sample Template',
'template_type': "sms",
'content': "Template content",
'service': sample_service,
'created_by': sample_user
}
template = Template(**template_data)
template_folder_data = {
'name': 'My Folder',
'service_id': sample_service.id,
}
template_folder = TemplateFolder(**template_folder_data)
template.folder = template_folder
dao_create_template(template)
template.archived = True
dao_update_template(template)
template_folder = TemplateFolder.query.one()
archived_template = Template.query.one()
assert template_folder
assert not archived_template.folder
def test_dao_update_template_reply_to_none_to_some(sample_service, sample_user):
letter_contact = create_letter_contact(sample_service, 'Edinburgh, ED1 1AA')