From a02bb78a199aa0bc733e7f8114955b03b21e96fb Mon Sep 17 00:00:00 2001 From: GillesFischerV Date: Mon, 6 Nov 2023 16:40:41 +0100 Subject: [PATCH 01/11] Fix CAMS message error handler - Issue#1799 --- pvlib/iotools/sodapro.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 95d6ffd866..c3f437367a 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -215,12 +215,14 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params, timeout=timeout) - # Invalid requests returns an XML error message and the HTTP staus code 200 - # as if the request was successful. Therefore, errors cannot be handled - # automatic (e.g. res.raise_for_status()) and errors are handled manually - if res.headers['Content-Type'] == 'application/xml': + # Response from CAMS follows the status and reason format of PyWPS4 + # If an error occurs on our side, we will return error 400 - bad request + # Additional information is available in the response text + # We add it here to the error displayed to facilitate users effort to fix their request + if not res.ok: errors = res.text.split('ows:ExceptionText')[1][1:-2] - raise requests.HTTPError(errors, response=res) + res.reason = "%s: <%s>"%(res.reason, errors) + res.raise_for_status() # Successful requests returns a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) From 1da05c00cf04ac9784c95e6fbdf47776bf60cc37 Mon Sep 17 00:00:00 2001 From: GillesFischerV Date: Mon, 6 Nov 2023 16:40:41 +0100 Subject: [PATCH 02/11] Fix CAMS message error handler - Issue#1799 --- pvlib/iotools/sodapro.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 95d6ffd866..f140d7442b 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -215,12 +215,14 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params, timeout=timeout) - # Invalid requests returns an XML error message and the HTTP staus code 200 - # as if the request was successful. Therefore, errors cannot be handled - # automatic (e.g. res.raise_for_status()) and errors are handled manually - if res.headers['Content-Type'] == 'application/xml': + # Response from CAMS follows the status and reason format of PyWPS4 + # If an error occurs on our side, we will return error 400 - bad request + # Additional information is available in the response text so we add it here + # to the error displayed to facilitate users effort to fix their request + if not res.ok: errors = res.text.split('ows:ExceptionText')[1][1:-2] - raise requests.HTTPError(errors, response=res) + res.reason = "%s: <%s>"%(res.reason, errors) + res.raise_for_status() # Successful requests returns a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) From 81bfc8f3763aa0d6e07bb15bbfa468041d56ece7 Mon Sep 17 00:00:00 2001 From: GillesFischerV Date: Tue, 7 Nov 2023 16:19:53 +0100 Subject: [PATCH 03/11] fix Python Flake8 Linter error --- pvlib/iotools/sodapro.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index f140d7442b..d00346f1e9 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -217,11 +217,12 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', # Response from CAMS follows the status and reason format of PyWPS4 # If an error occurs on our side, we will return error 400 - bad request - # Additional information is available in the response text so we add it here + # Additional information is available in the response text so + # we add it here # to the error displayed to facilitate users effort to fix their request if not res.ok: errors = res.text.split('ows:ExceptionText')[1][1:-2] - res.reason = "%s: <%s>"%(res.reason, errors) + res.reason = "%s: <%s>" % (res.reason, errors) res.raise_for_status() # Successful requests returns a csv data file elif res.headers['Content-Type'] == 'application/csv': From e6a49cd2b2a5ddcd5709802ddaa228fc22837ac0 Mon Sep 17 00:00:00 2001 From: GillesFischerV Date: Thu, 9 Nov 2023 16:00:04 +0100 Subject: [PATCH 04/11] Fix associated UT and add contribution in WhatsNews --- docs/sphinx/source/whatsnew/v0.10.3.rst | 4 +++- pvlib/iotools/sodapro.py | 2 +- pvlib/tests/iotools/test_sodapro.py | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.10.3.rst b/docs/sphinx/source/whatsnew/v0.10.3.rst index 8bdc2cf98f..988ac3bb9b 100644 --- a/docs/sphinx/source/whatsnew/v0.10.3.rst +++ b/docs/sphinx/source/whatsnew/v0.10.3.rst @@ -15,7 +15,8 @@ Enhancements Bug fixes ~~~~~~~~~ - +* Fixed CAMS error message handler in + :py:func:`pvlib.iotools.sodapro.get_cams` (:issue:`1799`, :pull:`1905`) Testing ~~~~~~~ @@ -32,3 +33,4 @@ Contributors * Miguel Sánchez de León Peque (:ghuser:`Peque`) * Will Hobbs (:ghuser:`williamhobbs`) * Anton Driesse (:ghuser:`adriesse`) +* Gilles Fischer (:ghuser: `GillesFischerV`) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index d00346f1e9..719b2da8ce 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -225,7 +225,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', res.reason = "%s: <%s>" % (res.reason, errors) res.raise_for_status() # Successful requests returns a csv data file - elif res.headers['Content-Type'] == 'application/csv': + else: fbuf = io.StringIO(res.content.decode('utf-8')) data, metadata = parse_cams(fbuf, integrated=integrated, label=label, map_variables=map_variables) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index ff17691a98..66a06fb3cd 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -248,7 +248,7 @@ def test_get_cams_bad_request(requests_mock): requests inputs. Also tests if the specified server url gets used""" # Subset of an xml file returned for errornous requests - mock_response_bad = """ + mock_response_bad_text = """ Failed to execute WPS process [get_mcclear]: Please, register yourself at www.soda-pro.com @@ -256,12 +256,12 @@ def test_get_cams_bad_request(requests_mock): url_cams_bad_request = 'https://pro.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=-999;date_begin=2020-01-01;date_end=2020-05-04;time_ref=TST;summarization=PT01H;username=test%2540test.com;verbose=false&Service=WPS&Request=Execute&Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 - requests_mock.get(url_cams_bad_request, text=mock_response_bad, - headers={'Content-Type': 'application/xml'}) + requests_mock.get(url_cams_bad_request, status_code=400, + text=mock_response_bad_text) # Test if HTTPError is raised if incorrect input is specified # In the below example a non-registrered email is specified - with pytest.raises(requests.HTTPError, match='Failed to execute WPS'): + with pytest.raises(requests.exceptions.HTTPError, match='Failed to execute WPS process'): _ = sodapro.get_cams( start=pd.Timestamp('2020-01-01'), end=pd.Timestamp('2020-05-04'), From 9c11d8faa9e68b65c0d3d2e64a2a59da5324fc14 Mon Sep 17 00:00:00 2001 From: GillesFischerV Date: Mon, 13 Nov 2023 15:02:59 +0100 Subject: [PATCH 05/11] Fix line code length in test_sodapro.py --- pvlib/tests/iotools/test_sodapro.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 66a06fb3cd..4729cf2c64 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -261,7 +261,8 @@ def test_get_cams_bad_request(requests_mock): # Test if HTTPError is raised if incorrect input is specified # In the below example a non-registrered email is specified - with pytest.raises(requests.exceptions.HTTPError, match='Failed to execute WPS process'): + with pytest.raises(requests.exceptions.HTTPError, + match='Failed to execute WPS process'): _ = sodapro.get_cams( start=pd.Timestamp('2020-01-01'), end=pd.Timestamp('2020-05-04'), From c544a494c9a6a243c67923aa658420b6f25224f0 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 21 Nov 2023 18:23:06 +0100 Subject: [PATCH 06/11] Fix typo in contributor ghuser --- docs/sphinx/source/whatsnew/v0.10.3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.10.3.rst b/docs/sphinx/source/whatsnew/v0.10.3.rst index eae9613910..15ccc832e4 100644 --- a/docs/sphinx/source/whatsnew/v0.10.3.rst +++ b/docs/sphinx/source/whatsnew/v0.10.3.rst @@ -33,5 +33,5 @@ Contributors * Miguel Sánchez de León Peque (:ghuser:`Peque`) * Will Hobbs (:ghuser:`williamhobbs`) * Anton Driesse (:ghuser:`adriesse`) -* Gilles Fischer (:ghuser: `GillesFischerV`) +* Gilles Fischer (:ghuser:`GillesFischerV`) * :ghuser:`matsuobasho` From 892dd0ec630beba107547cd684e96021fc297879 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 21 Nov 2023 18:58:29 +0100 Subject: [PATCH 07/11] Update geographical coverage description --- docs/sphinx/source/whatsnew/v0.10.3.rst | 4 +++- pvlib/iotools/sodapro.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.10.3.rst b/docs/sphinx/source/whatsnew/v0.10.3.rst index 15ccc832e4..5cff607089 100644 --- a/docs/sphinx/source/whatsnew/v0.10.3.rst +++ b/docs/sphinx/source/whatsnew/v0.10.3.rst @@ -16,7 +16,7 @@ Enhancements Bug fixes ~~~~~~~~~ * Fixed CAMS error message handler in - :py:func:`pvlib.iotools.sodapro.get_cams` (:issue:`1799`, :pull:`1905`) + :py:func:`pvlib.iotools.get_cams` (:issue:`1799`, :pull:`1905`) Testing ~~~~~~~ @@ -34,4 +34,6 @@ Contributors * Will Hobbs (:ghuser:`williamhobbs`) * Anton Driesse (:ghuser:`adriesse`) * Gilles Fischer (:ghuser:`GillesFischerV`) +* Adam R. Jensen (:ghusuer:`AdamRJensen`) * :ghuser:`matsuobasho` + diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 719b2da8ce..97569e32d4 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -57,8 +57,10 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', Access: free, but requires registration, see [2]_ Requests: max. 100 per day + Geographical coverage: worldwide for CAMS McClear and approximately -66° to - 66° in both latitude and longitude for CAMS Radiation. + 66° in latitude and -66° to 180° in longitude for CAMS Radiation. See [3]_ + for a map of the geographical coverage. Parameters ---------- @@ -157,6 +159,9 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', `_ .. [2] `CAMS Radiation Automatic Access (SoDa) `_ + .. [3] A. R. Jensen et al., pvlib iotools — Open-source Python functions + for seamless access to solar irradiance data. Solar Energy. 2023. Vol + 266, pp. 112092. :doi:`10.1016/j.solener.2023.112092` """ try: time_step_str = TIME_STEPS_MAP[time_step] From dd0a5724144e82dfe0f0de79cab0f38d5f1b1955 Mon Sep 17 00:00:00 2001 From: GillesFischerV Date: Wed, 22 Nov 2023 15:09:29 +0100 Subject: [PATCH 08/11] correction in maximum longitude available for CAMS Radiation --- pvlib/iotools/sodapro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 97569e32d4..dbbc63227d 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -59,7 +59,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', Requests: max. 100 per day Geographical coverage: worldwide for CAMS McClear and approximately -66° to - 66° in latitude and -66° to 180° in longitude for CAMS Radiation. See [3]_ + 66° in latitude and -66° to 200° in longitude for CAMS Radiation. See [3]_ for a map of the geographical coverage. Parameters From 80c4fa473cead26ae09e895d40b4c70996aba1c1 Mon Sep 17 00:00:00 2001 From: GillesFischerV Date: Wed, 22 Nov 2023 15:14:37 +0100 Subject: [PATCH 09/11] revert of last change --- pvlib/iotools/sodapro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index dbbc63227d..97569e32d4 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -59,7 +59,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', Requests: max. 100 per day Geographical coverage: worldwide for CAMS McClear and approximately -66° to - 66° in latitude and -66° to 200° in longitude for CAMS Radiation. See [3]_ + 66° in latitude and -66° to 180° in longitude for CAMS Radiation. See [3]_ for a map of the geographical coverage. Parameters From f1302483635ce278254b5313d7b43112010c7117 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:08:40 +0100 Subject: [PATCH 10/11] Add doc string changes from adriesse Co-authored-by: Anton Driesse --- pvlib/iotools/sodapro.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 97569e32d4..f1e7a8aeec 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -221,9 +221,8 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', timeout=timeout) # Response from CAMS follows the status and reason format of PyWPS4 - # If an error occurs on our side, we will return error 400 - bad request - # Additional information is available in the response text so - # we add it here + # If an error occurs on the server side, it will return error 400 - bad request + # Additional information is available in the response text, so it is added # to the error displayed to facilitate users effort to fix their request if not res.ok: errors = res.text.split('ows:ExceptionText')[1][1:-2] From fcf88454b2de53806bfe54dd01453337d3eecd9e Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 12 Dec 2023 19:57:24 +0100 Subject: [PATCH 11/11] Flake8 correction --- pvlib/iotools/sodapro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index f1e7a8aeec..a5c0c351c8 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -221,8 +221,8 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', timeout=timeout) # Response from CAMS follows the status and reason format of PyWPS4 - # If an error occurs on the server side, it will return error 400 - bad request - # Additional information is available in the response text, so it is added + # If an error occurs on server side, it will return error 400 - bad request + # Additional information is available in the response text, so it is added # to the error displayed to facilitate users effort to fix their request if not res.ok: errors = res.text.split('ows:ExceptionText')[1][1:-2]