diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index c17e5bed8..4d8f33a07 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -28,12 +28,17 @@ def get_csv_upload(service_id, upload_id): return get_s3_object(*get_csv_location(service_id, upload_id)) -def s3upload(service_id, filedata): +def remove_blank_lines(filedata): # sometimes people upload files with hundreds of blank lines at the end data = filedata["data"] cleaned_data = "\n".join(line for line in data.splitlines() if line.strip()) filedata["data"] = cleaned_data + return filedata + +def s3upload(service_id, filedata): + + filedata = remove_blank_lines(filedata) upload_id = str(uuid.uuid4()) bucket_name, file_location, access_key, secret_key, region = get_csv_location( service_id, upload_id diff --git a/tests/app/s3_client/test_s3_csv_client.py b/tests/app/s3_client/test_s3_csv_client.py index dbf26ea47..2ea3c43db 100644 --- a/tests/app/s3_client/test_s3_csv_client.py +++ b/tests/app/s3_client/test_s3_csv_client.py @@ -1,6 +1,6 @@ from unittest.mock import Mock -from app.s3_client.s3_csv_client import set_metadata_on_csv_upload +from app.s3_client.s3_csv_client import remove_blank_lines, set_metadata_on_csv_upload def test_sets_metadata(client_request, mocker): @@ -21,3 +21,11 @@ def test_sets_metadata(client_request, mocker): MetadataDirective="REPLACE", ServerSideEncryption="AES256", ) + + +def test_removes_blank_lines(): + filedata = { + "data": "phone number\r\n15555555555\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n" + } + file_data = remove_blank_lines(filedata) + assert file_data == {"data": "phone number\n15555555555"}