From ded87706c7db0bde2aaa0f28c7b4c25919dc56da Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 12 May 2022 10:14:16 +0100 Subject: [PATCH 1/3] Upgrade xlrd to version 2 xlrd is a library for reading data and formatting information from Excel files in the historical .xls format. Version 2 of xlrd no longer supports anything other than .xls files. We were using it to also support reading .xlsm files (old Excel files with macro support). We could keep using the old version of this dependency, but hopefully this niche version of an ancient file format is obscure enough that no-one is using it, and we can drop support, keeping our dependencies up to date. --- requirements.txt | 2 +- tests/app/main/views/test_send.py | 4 ++-- .../excel 2007 with macro support.xlsm | Bin 3 files changed, 3 insertions(+), 3 deletions(-) rename tests/{spreadsheet_files => non_spreadsheet_files}/excel 2007 with macro support.xlsm (100%) diff --git a/requirements.txt b/requirements.txt index f09debcb3..697eedffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -222,7 +222,7 @@ wtforms==3.0.1 # via # -r requirements.in # flask-wtf -xlrd==1.2.0 +xlrd==2.0.1 # via pyexcel-xls xlwt==1.3.0 # via pyexcel-xls diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 3dda1a696..7d192f2a0 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -300,8 +300,8 @@ def test_set_sender_redirects_if_one_sms_sender( def test_that_test_files_exist(): - assert len(test_spreadsheet_files) == 8 - assert len(test_non_spreadsheet_files) == 6 + assert len(test_spreadsheet_files) == 7 + assert len(test_non_spreadsheet_files) == 7 def test_should_not_allow_files_to_be_uploaded_without_the_correct_permission( diff --git a/tests/spreadsheet_files/excel 2007 with macro support.xlsm b/tests/non_spreadsheet_files/excel 2007 with macro support.xlsm similarity index 100% rename from tests/spreadsheet_files/excel 2007 with macro support.xlsm rename to tests/non_spreadsheet_files/excel 2007 with macro support.xlsm From 80854ab2ccbd4b81a166a1d6800de391d92ae2f0 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 12 May 2022 15:48:09 +0100 Subject: [PATCH 2/3] Force xlsm files to open with pyexcel-xlsx `.xlsm` files are like `.xlxs` files but with macros enabled. They store data in the same XML-based format as `.xlsx` files. Pyexcel will try to use the xlrd package to parse `.xlsm` files. This used to work because xlrd used to support reading `.xlsx` files. xlrd has dropped support for `.xlsx` files in version 2 because of security concerns. This means that when pyexcel asks xlrd to parse a `.xlsm` file it causes an error. This commit adds some branching to force `.xlsm` files to be opened with pyexcel-xlsx instead, which does support `.xlsx` files. --- app/models/spreadsheet.py | 10 ++++++++++ tests/app/main/views/test_send.py | 4 ++-- .../excel 2007 with macro support.xlsm | Bin 3 files changed, 12 insertions(+), 2 deletions(-) rename tests/{non_spreadsheet_files => spreadsheet_files}/excel 2007 with macro support.xlsm (100%) diff --git a/app/models/spreadsheet.py b/app/models/spreadsheet.py index 7b66a7b0f..59ca2bdd7 100644 --- a/app/models/spreadsheet.py +++ b/app/models/spreadsheet.py @@ -3,6 +3,7 @@ from io import StringIO from os import path import pyexcel +import pyexcel_xlsx class Spreadsheet(): @@ -72,6 +73,15 @@ class Spreadsheet(): file_content = StringIO( Spreadsheet.normalise_newlines(file_content)) + if extension == 'xlsm': + file_data = pyexcel_xlsx.get_data(file_content) + instance = cls.from_rows( + # Get the first sheet from the workbook + list(file_data.values())[0], + filename, + ) + return instance + instance = cls.from_rows( pyexcel.iget_array( file_type=extension, diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 7d192f2a0..3dda1a696 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -300,8 +300,8 @@ def test_set_sender_redirects_if_one_sms_sender( def test_that_test_files_exist(): - assert len(test_spreadsheet_files) == 7 - assert len(test_non_spreadsheet_files) == 7 + assert len(test_spreadsheet_files) == 8 + assert len(test_non_spreadsheet_files) == 6 def test_should_not_allow_files_to_be_uploaded_without_the_correct_permission( diff --git a/tests/non_spreadsheet_files/excel 2007 with macro support.xlsm b/tests/spreadsheet_files/excel 2007 with macro support.xlsm similarity index 100% rename from tests/non_spreadsheet_files/excel 2007 with macro support.xlsm rename to tests/spreadsheet_files/excel 2007 with macro support.xlsm From 42de68e6a02f389596f7bf371499fc0eae79b60a Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 13 May 2022 13:24:36 +0100 Subject: [PATCH 3/3] Update other Excel-related dependencies Make sure the latest versions of all Excel-related dependencies all work nicely together. --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 697eedffe..7457048f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -105,7 +105,7 @@ lml==0.1.0 # via # pyexcel # pyexcel-io -lxml==4.6.5 +lxml==4.8.0 # via # pyexcel-ezodf # pyexcel-ods3 @@ -119,7 +119,7 @@ notifications-python-client==6.3.0 # via -r requirements.in notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@55.1.6 # via -r requirements.in -openpyxl==3.0.7 +openpyxl==3.0.9 # via pyexcel-xlsx orderedset==2.0.3 # via notifications-utils @@ -205,7 +205,7 @@ smartypants==2.0.1 # via notifications-utils statsd==3.3.0 # via notifications-utils -texttable==1.6.3 +texttable==1.6.4 # via pyexcel urllib3==1.26.5 # via