Merge branch 'master' into clean-up

app/__init__.py

@@ -268,10 +268,6 @@ def register_v2_blueprints(application):
 
 def init_app(app):
 
-    @app.before_request
-    def record_user_agent():
-        statsd_client.incr("user-agent.{}".format(process_user_agent(request.headers.get('User-Agent', None))))
-
     @app.before_request
     def record_request_details():
         CONCURRENT_REQUESTS.inc()
@@ -318,18 +314,6 @@ def create_random_identifier():
     return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
 
 
-def process_user_agent(user_agent_string):
-    if user_agent_string and user_agent_string.lower().startswith("notify"):
-        components = user_agent_string.split("/")
-        client_name = components[0].lower()
-        client_version = components[1].replace(".", "-")
-        return "{}.{}".format(client_name, client_version)
-    elif user_agent_string and not user_agent_string.lower().startswith("notify"):
-        return "non-notify-user-agent"
-    else:
-        return "unknown"
-
-
 def setup_sqlalchemy_events(app):
 
     TOTAL_DB_CONNECTIONS = Gauge(

app/commands.py

@@ -773,9 +773,24 @@ def get_letter_details_from_zips_sent_file(file_paths):
         rows_from_file.extend(json.loads(file_contents))
 
     notification_references = tuple(row[18:34] for row in rows_from_file)
+    get_letters_data_from_references(notification_references)
 
+
+@notify_command(name='get-notification-and-service-ids-for-letters-that-failed-to-print')
+@click.option('-f', '--file_name', required=True,
+              help="""Full path of the file to upload, file should contain letter filenames, one per line""")
+def get_notification_and_service_ids_for_letters_that_failed_to_print(file_name):
+    print("Getting service and notification ids for letter filenames list {}".format(file_name))
+    file = open(file_name)
+    references = tuple([row[7:23] for row in file])
+
+    get_letters_data_from_references(tuple(references))
+    file.close()
+
+
+def get_letters_data_from_references(notification_references):
     sql = """
-        SELECT id, service_id, reference, job_id, created_at
+        SELECT id, service_id, template_id, reference, job_id, created_at
         FROM notifications
         WHERE reference IN :notification_references
         ORDER BY service_id, job_id"""
@@ -783,7 +798,7 @@ def get_letter_details_from_zips_sent_file(file_paths):
 
     with open('zips_sent_details.csv', 'w') as csvfile:
         csv_writer = csv.writer(csvfile)
-        csv_writer.writerow(['notification_id', 'service_id', 'reference', 'job_id', 'created_at'])
+        csv_writer.writerow(['notification_id', 'service_id', 'template_id', 'reference', 'job_id', 'created_at'])
 
         for row in result:
             csv_writer.writerow(row)

app/dao/notifications_dao.py

@@ -77,7 +77,6 @@ def dao_get_last_date_template_was_used(template_id, service_id):
     return last_date
 
 
-@statsd(namespace="dao")
 @transactional
 def dao_create_notification(notification):
     if not notification.id:

app/schemas.py

@@ -63,9 +63,16 @@ class UUIDsAsStringsMixin:
     @post_dump()
     def __post_dump(self, data):
         for key, value in data.items():
+
             if isinstance(value, UUID):
                 data[key] = str(value)
 
+            if isinstance(value, list):
+                data[key] = [
+                    (str(item) if isinstance(item, UUID) else item)
+                    for item in value
+                ]
+
 
 class BaseSchema(ma.ModelSchema):
 
@@ -249,7 +256,6 @@ class ServiceSchema(BaseSchema, UUIDsAsStringsMixin):
             'inbound_number',
             'inbound_sms',
             'letter_logo_filename',
-            'rate_limit',
             'returned_letters',
             'users',
             'version',
@@ -350,9 +356,7 @@ class BaseTemplateSchema(BaseSchema):
 
 class TemplateSchema(BaseTemplateSchema, UUIDsAsStringsMixin):
 
-    created_by_id = field_for(
-        models.Template, 'created_by_id', dump_to='created_by', dump_only=True
-    )
+    created_by = field_for(models.Template, 'created_by', required=True)
     process_type = field_for(models.Template, 'process_type')
     redact_personalisation = fields.Method("redact")
 
@@ -366,9 +370,6 @@ class TemplateSchema(BaseTemplateSchema, UUIDsAsStringsMixin):
             if not subject or subject.strip() == '':
                 raise ValidationError('Invalid template subject', 'subject')
 
-    class Meta(BaseTemplateSchema.Meta):
-        exclude = BaseTemplateSchema.Meta.exclude + ('created_by',)
-
 
 class TemplateSchemaNoDetail(TemplateSchema):
     class Meta(TemplateSchema.Meta):
@@ -376,6 +377,7 @@ class TemplateSchemaNoDetail(TemplateSchema):
             'archived',
             'content',
             'created_at',
+            'created_by',
             'created_by_id',
             'hidden',
             'postage',

app/serialised_models.py

@@ -120,7 +120,9 @@ class SerialisedService(SerialisedModel):
         'active',
         'contact_link',
         'email_from',
+        'message_limit',
         'permissions',
+        'rate_limit',
         'research_mode',
         'restricted',
     }