From 24a87d82604487b246de285c1fddb1d65ffe5677 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 17 Oct 2017 16:05:31 +0100 Subject: [PATCH 1/6] Small change to use the current datetime stamp for the Notification.created_at timestamp rather than the job.created_at start date. This will give us better more reflective data. --- app/celery/tasks.py | 6 +++--- tests/app/celery/test_tasks.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index f4218f5d9..f3244edb5 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -209,7 +209,7 @@ def send_sms(self, notification_type=SMS_TYPE, api_key_id=api_key_id, key_type=key_type, - created_at=created_at, + created_at=datetime.utcnow(), job_id=notification.get('job', None), job_row_number=notification.get('row_number', None), notification_id=notification_id @@ -254,7 +254,7 @@ def send_email(self, notification_type=EMAIL_TYPE, api_key_id=api_key_id, key_type=key_type, - created_at=created_at, + created_at=datetime.utcnow(), job_id=notification.get('job', None), job_row_number=notification.get('row_number', None), notification_id=notification_id @@ -295,7 +295,7 @@ def persist_letter( notification_type=LETTER_TYPE, api_key_id=None, key_type=KEY_TYPE_NORMAL, - created_at=created_at, + created_at=datetime.utcnow(), job_id=notification['job'], job_row_number=notification['row_number'], notification_id=notification_id, diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index c4c5d1bd1..6841f69e7 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -700,7 +700,7 @@ def test_should_send_sms_template_to_and_persist_with_job_id(sample_job, sample_ assert persisted_notification.template_id == sample_job.template.id assert persisted_notification.status == 'created' assert not persisted_notification.sent_at - assert persisted_notification.created_at <= now + assert persisted_notification.created_at >= now assert not persisted_notification.sent_by assert persisted_notification.job_row_number == 2 assert persisted_notification.api_key_id == sample_api_key.id @@ -794,7 +794,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh assert persisted_notification.to == 'my_email@my_email.com' assert persisted_notification.template_id == sample_email_template_with_placeholders.id assert persisted_notification.template_version == sample_email_template_with_placeholders.version - assert persisted_notification.created_at == now + assert persisted_notification.created_at >= now assert not persisted_notification.sent_at assert persisted_notification.status == 'created' assert not persisted_notification.sent_by @@ -831,7 +831,7 @@ def test_send_email_should_use_template_version_from_job_not_latest(sample_email assert persisted_notification.to == 'my_email@my_email.com' assert persisted_notification.template_id == sample_email_template.id assert persisted_notification.template_version == version_on_notification - assert persisted_notification.created_at == now + assert persisted_notification.created_at >= now assert not persisted_notification.sent_at assert persisted_notification.status == 'created' assert not persisted_notification.sent_by @@ -857,7 +857,7 @@ def test_should_use_email_template_subject_placeholders(sample_email_template_wi assert persisted_notification.to == 'my_email@my_email.com' assert persisted_notification.template_id == sample_email_template_with_placeholders.id assert persisted_notification.status == 'created' - assert persisted_notification.created_at == now + assert persisted_notification.created_at >= now assert not persisted_notification.sent_by assert persisted_notification.personalisation == {"name": "Jo"} assert not persisted_notification.reference @@ -883,7 +883,7 @@ def test_should_use_email_template_and_persist_without_personalisation(sample_em persisted_notification = Notification.query.one() assert persisted_notification.to == 'my_email@my_email.com' assert persisted_notification.template_id == sample_email_template.id - assert persisted_notification.created_at == now + assert persisted_notification.created_at >= now assert not persisted_notification.sent_at assert persisted_notification.status == 'created' assert not persisted_notification.sent_by @@ -1019,7 +1019,7 @@ def test_persist_letter_saves_letter_to_database(sample_letter_job, mocker): assert notification_db.template_id == sample_letter_job.template.id assert notification_db.template_version == sample_letter_job.template.version assert notification_db.status == 'created' - assert notification_db.created_at == created_at + assert notification_db.created_at >= created_at assert notification_db.notification_type == 'letter' assert notification_db.sent_at is None assert notification_db.sent_by is None From f52ec2c40697d8d1174ddb12ff82014e3bd0a16d Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 18 Oct 2017 09:50:39 +0100 Subject: [PATCH 2/6] Added an order by to the check_job_status query. The test was failing intermittently --- app/celery/scheduled_tasks.py | 2 +- tests/app/celery/test_scheduled_tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index c8154721d..2e0413fe8 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -392,7 +392,7 @@ def check_job_status(): jobs_not_complete_after_30_minutes = Job.query.filter( Job.job_status == JOB_STATUS_IN_PROGRESS, and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago) - ).all() + ).order_by(Job.processing_started).all() job_ids = [str(x.id) for x in jobs_not_complete_after_30_minutes] if job_ids: diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index ba87a94de..a61c70807 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -831,6 +831,6 @@ def test_check_job_status_task_raises_job_incomplete_error_for_multiple_jobs(moc mock_celery.assert_called_once_with( name=TaskNames.PROCESS_INCOMPLETE_JOBS, - args=([str(job_2.id), str(job.id)],), + args=([str(job.id), str(job_2.id)],), queue=QueueNames.JOBS ) From 5b2912c0f516e70a2b147d810fbe40f9b3dc04f3 Mon Sep 17 00:00:00 2001 From: venusbb Date: Wed, 18 Oct 2017 15:09:05 +0100 Subject: [PATCH 3/6] create new table annual_billing --- app/commands.py | 24 ++++++++++++++- app/dao/date_util.py | 2 +- app/models.py | 20 +++++++++++++ application.py | 2 ++ .../versions/0126_add_annual_billing.py | 29 +++++++++++++++++++ 5 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 migrations/versions/0126_add_annual_billing.py diff --git a/app/commands.py b/app/commands.py index 8686eeb0e..cb1298382 100644 --- a/app/commands.py +++ b/app/commands.py @@ -247,7 +247,7 @@ class PopulateServiceEmailReplyTo(Command): result = db.session.execute(services_to_update) db.session.commit() - print("Populated email reply to adderesses for {}".format(result.rowcount)) + print("Populated email reply to addresses for {}".format(result.rowcount)) class PopulateServiceSmsSender(Command): @@ -329,3 +329,25 @@ class PopulateServiceAndServiceHistoryFreeSmsFragmentLimit(Command): print("Populated free sms fragment limits for {} services".format(services_result.rowcount)) print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount)) + + +class PopulateAnnualBilling(Command): + def run(self): + financial_year=[2016,2017,2018] + + for fy in financial_year: + populate_data = """ + INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start, + created_at, updated_at) + SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}' + FROM services + WHERE id NOT IN( + SELECT service_id + FROM annual_billing + WHERE financial_year_start={}) + """.format(fy, datetime.utcnow(), datetime.utcnow(), fy) + + services_result1 = db.session.execute(populate_data) + db.session.commit() + + print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount)) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 8815e94cd..a0044517d 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -26,7 +26,7 @@ def get_financial_year(year): def get_april_fools(year): """ This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, - the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. + the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone. :param year: the year to calculate the April 1, 00:00 BST for :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 """ diff --git a/app/models.py b/app/models.py index 809c32eb6..bd4a51668 100644 --- a/app/models.py +++ b/app/models.py @@ -1462,3 +1462,23 @@ class NotificationEmailReplyTo(db.Model): nullable=False, primary_key=True ) + + +class AnnualBilling(db.Model): + __tablename__ = "annual_billing" + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False) + financial_year_start = db.Column(db.Integer, nullable=False, default=True) + free_sms_fragment_limit = db.Column(db.Integer, nullable=False, index=False, unique=False) + updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + + def serialize(self): + return { + 'id': str(self.id), + 'service_id': str(self.service_id), + 'free_sms_fragment_limit': str(self.free_sms_fragment_limit), + 'financial_year_start': str(self.financial_year_start), + 'created_at': self.created_at.strftime(DATETIME_FORMAT), + 'updated_at': self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None + } diff --git a/application.py b/application.py index 07fe4f0de..fb4f815c7 100644 --- a/application.py +++ b/application.py @@ -23,6 +23,8 @@ manager.add_command('populate_service_sms_sender', commands.PopulateServiceSmsSe manager.add_command('populate_service_letter_contact', commands.PopulateServiceLetterContact) manager.add_command('populate_service_and_service_history_free_sms_fragment_limit', commands.PopulateServiceAndServiceHistoryFreeSmsFragmentLimit) +manager.add_command('populate_annual_billing', + commands.PopulateAnnualBilling) @manager.command diff --git a/migrations/versions/0126_add_annual_billing.py b/migrations/versions/0126_add_annual_billing.py new file mode 100644 index 000000000..7f6dfe5b2 --- /dev/null +++ b/migrations/versions/0126_add_annual_billing.py @@ -0,0 +1,29 @@ +""" + +Revision ID: 0126_add_annual_billing +Revises: 0125_add_organisation_type +Create Date: 2017-10-18 11:42:54.261575 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0126_add_annual_billing' +down_revision = '0125_add_organisation_type' + + +def upgrade(): + op.create_table('annual_billing', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('financial_year_start', sa.Integer(), nullable=False), + sa.Column('free_sms_fragment_limit', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id')) + + +def downgrade(): + op.drop_table('annual_billing') From b2fa7cdd8327a43f628c17cfba95f7ddaaf18337 Mon Sep 17 00:00:00 2001 From: venusbb Date: Wed, 18 Oct 2017 15:09:05 +0100 Subject: [PATCH 4/6] create new table annual_billing --- app/commands.py | 24 ++++++++++++++- app/dao/date_util.py | 2 +- app/models.py | 20 +++++++++++++ application.py | 2 ++ .../versions/0126_add_annual_billing.py | 29 +++++++++++++++++++ 5 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 migrations/versions/0126_add_annual_billing.py diff --git a/app/commands.py b/app/commands.py index 8686eeb0e..a217f480a 100644 --- a/app/commands.py +++ b/app/commands.py @@ -247,7 +247,7 @@ class PopulateServiceEmailReplyTo(Command): result = db.session.execute(services_to_update) db.session.commit() - print("Populated email reply to adderesses for {}".format(result.rowcount)) + print("Populated email reply to addresses for {}".format(result.rowcount)) class PopulateServiceSmsSender(Command): @@ -329,3 +329,25 @@ class PopulateServiceAndServiceHistoryFreeSmsFragmentLimit(Command): print("Populated free sms fragment limits for {} services".format(services_result.rowcount)) print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount)) + + +class PopulateAnnualBilling(Command): + def run(self): + financial_year = [2016, 2017, 2018] + + for fy in financial_year: + populate_data = """ + INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start, + created_at, updated_at) + SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}' + FROM services + WHERE id NOT IN( + SELECT service_id + FROM annual_billing + WHERE financial_year_start={}) + """.format(fy, datetime.utcnow(), datetime.utcnow(), fy) + + services_result1 = db.session.execute(populate_data) + db.session.commit() + + print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount)) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 8815e94cd..a0044517d 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -26,7 +26,7 @@ def get_financial_year(year): def get_april_fools(year): """ This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, - the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. + the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone. :param year: the year to calculate the April 1, 00:00 BST for :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 """ diff --git a/app/models.py b/app/models.py index 809c32eb6..bd4a51668 100644 --- a/app/models.py +++ b/app/models.py @@ -1462,3 +1462,23 @@ class NotificationEmailReplyTo(db.Model): nullable=False, primary_key=True ) + + +class AnnualBilling(db.Model): + __tablename__ = "annual_billing" + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False) + financial_year_start = db.Column(db.Integer, nullable=False, default=True) + free_sms_fragment_limit = db.Column(db.Integer, nullable=False, index=False, unique=False) + updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + + def serialize(self): + return { + 'id': str(self.id), + 'service_id': str(self.service_id), + 'free_sms_fragment_limit': str(self.free_sms_fragment_limit), + 'financial_year_start': str(self.financial_year_start), + 'created_at': self.created_at.strftime(DATETIME_FORMAT), + 'updated_at': self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None + } diff --git a/application.py b/application.py index 07fe4f0de..fb4f815c7 100644 --- a/application.py +++ b/application.py @@ -23,6 +23,8 @@ manager.add_command('populate_service_sms_sender', commands.PopulateServiceSmsSe manager.add_command('populate_service_letter_contact', commands.PopulateServiceLetterContact) manager.add_command('populate_service_and_service_history_free_sms_fragment_limit', commands.PopulateServiceAndServiceHistoryFreeSmsFragmentLimit) +manager.add_command('populate_annual_billing', + commands.PopulateAnnualBilling) @manager.command diff --git a/migrations/versions/0126_add_annual_billing.py b/migrations/versions/0126_add_annual_billing.py new file mode 100644 index 000000000..7f6dfe5b2 --- /dev/null +++ b/migrations/versions/0126_add_annual_billing.py @@ -0,0 +1,29 @@ +""" + +Revision ID: 0126_add_annual_billing +Revises: 0125_add_organisation_type +Create Date: 2017-10-18 11:42:54.261575 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0126_add_annual_billing' +down_revision = '0125_add_organisation_type' + + +def upgrade(): + op.create_table('annual_billing', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('financial_year_start', sa.Integer(), nullable=False), + sa.Column('free_sms_fragment_limit', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id')) + + +def downgrade(): + op.drop_table('annual_billing') From 7e78e19500631d594c36bd3fcd274957fbba24a4 Mon Sep 17 00:00:00 2001 From: venusbb Date: Thu, 19 Oct 2017 12:00:01 +0100 Subject: [PATCH 5/6] Unique constrain and db.relationship --- app/models.py | 41 ++++++++++--------- .../versions/0126_add_annual_billing.py | 10 +++-- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/app/models.py b/app/models.py index bd4a51668..ab8978875 100644 --- a/app/models.py +++ b/app/models.py @@ -179,6 +179,26 @@ class ServicePermissionTypes(db.Model): name = db.Column(db.String(255), primary_key=True) +class AnnualBilling(db.Model): + __tablename__ = "annual_billing" + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=False) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False) + financial_year_start = db.Column(db.Integer, nullable=False, default=True, unique=False) + free_sms_fragment_limit = db.Column(db.Integer, nullable=False, index=False, unique=False) + updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + + def serialize(self): + return { + 'id': str(self.id), + 'service_id': str(self.service_id), + 'free_sms_fragment_limit': str(self.free_sms_fragment_limit), + 'financial_year_start': str(self.financial_year_start), + 'created_at': self.created_at.strftime(DATETIME_FORMAT), + 'updated_at': self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None + } + + class Service(db.Model, Versioned): __tablename__ = 'services' @@ -213,6 +233,7 @@ class Service(db.Model, Versioned): organisation_id = db.Column(UUID(as_uuid=True), db.ForeignKey('organisation.id'), index=True, nullable=True) free_sms_fragment_limit = db.Column(db.BigInteger, index=False, unique=False, nullable=True) organisation = db.relationship('Organisation') + annual_billing = db.relationship('AnnualBilling') dvla_organisation_id = db.Column( db.String, db.ForeignKey('dvla_organisation.id'), @@ -1462,23 +1483,3 @@ class NotificationEmailReplyTo(db.Model): nullable=False, primary_key=True ) - - -class AnnualBilling(db.Model): - __tablename__ = "annual_billing" - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False) - financial_year_start = db.Column(db.Integer, nullable=False, default=True) - free_sms_fragment_limit = db.Column(db.Integer, nullable=False, index=False, unique=False) - updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) - created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - - def serialize(self): - return { - 'id': str(self.id), - 'service_id': str(self.service_id), - 'free_sms_fragment_limit': str(self.free_sms_fragment_limit), - 'financial_year_start': str(self.financial_year_start), - 'created_at': self.created_at.strftime(DATETIME_FORMAT), - 'updated_at': self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None - } diff --git a/migrations/versions/0126_add_annual_billing.py b/migrations/versions/0126_add_annual_billing.py index 7f6dfe5b2..1a20a1c0b 100644 --- a/migrations/versions/0126_add_annual_billing.py +++ b/migrations/versions/0126_add_annual_billing.py @@ -2,7 +2,7 @@ Revision ID: 0126_add_annual_billing Revises: 0125_add_organisation_type -Create Date: 2017-10-18 11:42:54.261575 +Create Date: 2017-10-19 11:38:32.849573 """ from alembic import op @@ -19,11 +19,15 @@ def upgrade(): sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), sa.Column('financial_year_start', sa.Integer(), nullable=False), sa.Column('free_sms_fragment_limit', sa.Integer(), nullable=False), - sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), - sa.PrimaryKeyConstraint('id')) + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_annual_billing_service_id'), 'annual_billing', ['service_id'], unique=False) def downgrade(): + op.drop_index(op.f('ix_annual_billing_service_id'), table_name='annual_billing') op.drop_table('annual_billing') + From b0872d9466ea50dd0f4bbb684eb090c24845ff1c Mon Sep 17 00:00:00 2001 From: venusbb Date: Thu, 19 Oct 2017 14:15:34 +0100 Subject: [PATCH 6/6] add unique service_id&year constraint --- app/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models.py b/app/models.py index ab8978875..0637293fa 100644 --- a/app/models.py +++ b/app/models.py @@ -187,6 +187,7 @@ class AnnualBilling(db.Model): free_sms_fragment_limit = db.Column(db.Integer, nullable=False, index=False, unique=False) updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + UniqueConstraint('financial_year_start', 'service_id', name='ix_annual_billing_service_id') def serialize(self): return {