darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-13 00:32:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

Database tweak and BST bug fix

Browse Source
This commit is contained in:
venusbb
2018-03-27 10:37:56 +01:00
parent c02b112e9b
commit 2262db4f62
4 changed files with 243 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
app/commands.py
View File

@@ -388,24 +388,39 @@ def migrate_data_to_ft_billing(start_date, end_date):
print('Billing migration from date {} to {}'.format(start_date, end_date))
process_date = start_date
while (process_date <= end_date):
total_updated = 0
while process_date < end_date:
sql = \
"""
select count(*) from notification_history where notification_status!='technical-failure'
and key_type!='test'
and notification_status!='created'
and created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
"""
num_notifications = db.session.execute(sql, {"start": process_date,
"end": process_date + timedelta(days=1)}).fetchall()[0][0]
sql = \
"""
select count(*) from
(select distinct date_part('day', created_at) as utc_date, service_id, template_id, rate_multiplier,
(select distinct service_id, template_id, rate_multiplier,
sent_by from notification_history
where notification_status!='technical-failure'
and key_type!='test'
and notification_status!='created'
and created_at >= :start
and created_at < :end order by utc_date) as distinct_records
and created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
) as distinct_records
"""
predicted_records = db.session.execute(sql, {"start": process_date,
"end": process_date + timedelta(days=1)}).fetchall()[0][0]
start_time = datetime.now()
print('{}: Migrating date: {}, expecting {} rows'
.format(start_time, process_date, predicted_records))
print('ft_billing: Migrating date: {}, notifications: {}, expecting {} ft_billing rows'
.format(process_date.date(), num_notifications, predicted_records))
# migrate data into ft_billing, ignore if records already exist - do not do upsert
sql = \
@@ -443,14 +458,15 @@ def migrate_data_to_ft_billing(start_date, end_date):
1 as notifications_sent
from public.notification_history n
left join templates t on t.id = n.template_id
left join dm_datetime da on n.created_at> da.utc_daytime_start
left join dm_datetime da on n.created_at>= da.utc_daytime_start
and n.created_at < da.utc_daytime_end
left join services s on s.id = n.service_id
where n.notification_status!='technical-failure'
and n.key_type!='test'
and n.notification_status!='created'
and n.created_at >= :start
and n.created_at < :end
and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
at time zone 'UTC'
and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
) as individual_record
group by bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
sms_rate, letter_rate
@@ -463,11 +479,13 @@ def migrate_data_to_ft_billing(start_date, end_date):
result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
db.session.commit()
print('{}: --- Completed took {}ms. Migrated {} rows.'.format(datetime.now(), datetime.now() - start_time,
result.rowcount,
))
print('ft_billing: --- Completed took {}ms. Migrated {} rows.'.format(datetime.now() - start_time,
result.rowcount))
if predicted_records != result.rowcount:
print(' : --- Result mismatch by {} rows'
print(' : ^^^ Result mismatch by {} rows ^^^'
.format(predicted_records - result.rowcount))
process_date += timedelta(days=1)
total_updated += result.rowcount
print('Total inserted/updated records = {}'.format(total_updated))