darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-04 10:21:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

Update the command to migrate data for ft_billing.

Browse Source 
There is a massive performance improvement.
Another commit will need to alter the pk to include rate.
This commit is contained in:
Rebecca Law
2018-05-18 17:22:51 +01:00
parent 4b36fe0d9e
commit 64aec4a64c
1 changed files with 5 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
app/commands.py
View File

@@ -401,43 +401,13 @@ def setup_commands(application):
@statsd(namespace="tasks") @statsd(namespace="tasks")
def migrate_data_to_ft_billing(start_date, end_date): def migrate_data_to_ft_billing(start_date, end_date):
print('Billing migration from date {} to {}'.format(start_date, end_date)) current_app.logger.info('Billing migration from date {} to {}'.format(start_date, end_date))
process_date = start_date process_date = start_date
total_updated = 0 total_updated = 0
while process_date < end_date: while process_date < end_date:
start_time = datetime.utcnow()
sql = \
"""
select count(*) from notification_history where notification_status!='technical-failure'
and key_type!='test'
and notification_status!='created'
and created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
"""
num_notifications = db.session.execute(sql, {"start": process_date,
"end": process_date + timedelta(days=1)}).fetchall()[0][0]
sql = \
"""
select count(*) from
(select distinct service_id, template_id, rate_multiplier,
sent_by from notification_history
where notification_status!='technical-failure'
and key_type!='test'
and notification_status!='created'
and created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
) as distinct_records
"""
predicted_records = db.session.execute(sql, {"start": process_date,
"end": process_date + timedelta(days=1)}).fetchall()[0][0]
start_time = datetime.now()
print('ft_billing: Migrating date: {}, notifications: {}, expecting {} ft_billing rows'
.format(process_date.date(), num_notifications, predicted_records))
# migrate data into ft_billing, ignore if records already exist - do not do upsert # migrate data into ft_billing, ignore if records already exist - do not do upsert
sql = \ sql = \
""" """
@@ -449,7 +419,7 @@ def migrate_data_to_ft_billing(start_date, end_date):
from ( from (
select select
n.id, n.id,
da.bst_date, (n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id, coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id, coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id,
n.notification_type, n.notification_type,
@@ -473,9 +443,6 @@ def migrate_data_to_ft_billing(start_date, end_date):
n.billable_units, n.billable_units,
1 as notifications_sent 1 as notifications_sent
from public.notification_history n from public.notification_history n
left join templates t on t.id = n.template_id
left join dm_datetime da on n.created_at>= da.utc_daytime_start
and n.created_at < da.utc_daytime_end
left join services s on s.id = n.service_id left join services s on s.id = n.service_id
where n.notification_status!='technical-failure' where n.notification_status!='technical-failure'
and n.key_type!='test' and n.key_type!='test'
@@ -495,16 +462,13 @@ def migrate_data_to_ft_billing(start_date, end_date):
result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)}) result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
db.session.commit() db.session.commit()
print('ft_billing: --- Completed took {}ms. Migrated {} rows.'.format(datetime.now() - start_time, current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows.'.format(datetime.now() - start_time,
result.rowcount)) result.rowcount))
if predicted_records != result.rowcount:
print(' : ^^^ Result mismatch by {} rows ^^^'
.format(predicted_records - result.rowcount))
process_date += timedelta(days=1) process_date += timedelta(days=1)
total_updated += result.rowcount total_updated += result.rowcount
print('Total inserted/updated records = {}'.format(total_updated)) current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
@notify_command() @notify_command()