added a second test to preserve the \r\n formatting for line endings

app/s3_client/s3_csv_client.py

@@ -28,8 +28,17 @@ def get_csv_upload(service_id, upload_id):
     return get_s3_object(*get_csv_location(service_id, upload_id))
 
 
+def remove_blank_lines(filedata):
+    # sometimes people upload files with hundreds of blank lines at the end
+    data = filedata["data"]
+    cleaned_data = "\r\n".join(line for line in data.splitlines() if line.strip())
+    filedata["data"] = cleaned_data
+    return filedata
+
+
 def s3upload(service_id, filedata):
 
+    filedata = remove_blank_lines(filedata)
     upload_id = str(uuid.uuid4())
     bucket_name, file_location, access_key, secret_key, region = get_csv_location(
         service_id, upload_id

tests/app/s3_client/test_s3_csv_client.py

@@ -1,6 +1,6 @@
 from unittest.mock import Mock
 
-from app.s3_client.s3_csv_client import set_metadata_on_csv_upload
+from app.s3_client.s3_csv_client import remove_blank_lines, set_metadata_on_csv_upload
 
 
 def test_sets_metadata(client_request, mocker):
@@ -21,3 +21,9 @@ def test_sets_metadata(client_request, mocker):
         MetadataDirective="REPLACE",
         ServerSideEncryption="AES256",
     )
+
+
+def test_removes_blank_lines():
+    filedata = { "data": "variable,phone number\r\ntest,+15555555555\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n" }
+    file_data = remove_blank_lines(filedata)
+    assert file_data == {"data": "variable,phone number\r\ntest,+15555555555"}