From 3f0332326e969877b9ec6a2b8d059de685955064 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:10:21 -0400 Subject: [PATCH 01/19] create acis.py and add get_acis_precipitation function --- pvlib/iotools/acis.py | 105 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 pvlib/iotools/acis.py diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py new file mode 100644 index 0000000000..cdfbc26ff7 --- /dev/null +++ b/pvlib/iotools/acis.py @@ -0,0 +1,105 @@ +import requests +import pandas as pd +import numpy as np + + +def get_acis_precipitation(latitude, longitude, start, end, dataset, + url="https://data.rcc-acis.org/GridData", **kwargs): + """ + Retrieve daily gridded precipitation data from the Applied Climate + Information System (ACIS). + + The Applied Climate Information System (ACIS) was developed and is + maintained by the NOAA Regional Climate Centers (RCCs) and brings together + climate data from many sources. + + Parameters + ---------- + latitude : float + in decimal degrees, between -90 and 90, north is positive + longitude : float + in decimal degrees, between -180 and 180, east is positive + start : datetime-like + First day of the requested period + end : datetime-like + Last day of the requested period + dataset : int + A number indicating which gridded dataset to query. Options include: + + * 1: NRCC Interpolated + * 2: Multi-Sensor Precipitation Estimates + * 3: NRCC Hi-Res + * 21: PRISM + + See [1]_ for the full list of options. + + url : str, default: 'https://data.rcc-acis.org/GridData' + API endpoint URL + kwargs: + Optional parameters passed to ``requests.get``. + + Returns + ------- + data : pandas.DataFrame + Daily rainfall [mm] + metadata : dict + Coordinates for the selected grid cell + + Raises + ------ + requests.HTTPError + A message from the ACIS server if the request is rejected + + Notes + ----- + The returned precipitation values are 24-hour aggregates, but + the aggregation period may not be midnight to midnight in local time. + For example, PRISM data is aggregated from 12:00 to 12:00 UTC, + meaning PRISM data labeled May 26 reflects to the 24 hours ending at + 7:00am Eastern Standard Time on May 26. + + Examples + -------- + >>> prism, metadata = get_acis_precipitation(40.0, -80.0, '2020-01-01', + >>> '2020-12-31', dataset=21) + + References + ---------- + .. [1] `ACIS Web Services `_ + .. [2] `ACIS Gridded Data `_ + .. [3] `NRCC `_ + .. [4] `Multisensor Precipitation Estimates + `_ + .. [5] `PRISM `_ + """ + elems = [ + # other variables exist, but are not of interest for PV modeling + {"name": "pcpn", "interval": "dly", "units": "mm"}, + ] + params = { + 'loc': f"{longitude},{latitude}", + 'sdate': pd.to_datetime(start).strftime('%Y-%m-%d'), + 'edate': pd.to_datetime(end).strftime('%Y-%m-%d'), + 'grid': str(dataset), + 'elems': elems, + 'output': 'json', + 'meta': ["ll"], # "elev" should work, but it errors for some databases + } + response = requests.post(url, json=params, + headers={"Content-Type": "application/json"}, + **kwargs) + response.raise_for_status() + payload = response.json() + + if "error" in payload: + raise requests.HTTPError(payload['error'], response=response) + + metadata = payload['meta'] + metadata['latitude'] = metadata.pop('lat') + metadata['longitude'] = metadata.pop('lon') + + df = pd.DataFrame(payload['data'], columns=['date', 'precipitation']) + rainfall = df.set_index('date')['precipitation'] + rainfall = rainfall.replace(-999, np.nan) + rainfall.index = pd.to_datetime(rainfall.index) + return rainfall, metadata From 94a1e901bacdd6314d3a693cc23c27ce1563926b Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:10:35 -0400 Subject: [PATCH 02/19] add tests --- pvlib/tests/iotools/test_acis.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 pvlib/tests/iotools/test_acis.py diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py new file mode 100644 index 0000000000..dc68995622 --- /dev/null +++ b/pvlib/tests/iotools/test_acis.py @@ -0,0 +1,27 @@ +""" +tests for :mod:`pvlib.iotools.acis` +""" + +import pandas as pd +import pytest +from pvlib.iotools import get_acis_precipitation +from ..conftest import (RERUNS, RERUNS_DELAY, assert_series_equal) +from requests import HTTPError + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_acis_precipitation(): + precipitation, meta = get_acis_precipitation(40, -80, '2012-01-01', + '2012-01-05', 21) + idx = pd.date_range('2012-01-01', '2012-01-05', freq='d') + expected = pd.Series([0.6, 1.8, 1.9, 1.2, 0.0], index=idx, + name='precipitation') + assert_series_equal(precipitation, expected) + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_acis_precipitation_error(): + with pytest.raises(HTTPError, match='invalid grid'): + # 50 is not a valid dataset (or "grid", in ACIS lingo) + get_acis_precipitation(40, -80, '2012-01-01', '2012-01-05', 50) From 99fc6b59546416637ea80e991d6928e408f7845c Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:10:54 -0400 Subject: [PATCH 03/19] sphinx --- docs/sphinx/source/reference/iotools.rst | 1 + docs/sphinx/source/whatsnew/v0.9.6.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index 481f46ddb5..17ad5f630c 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -36,6 +36,7 @@ of sources and file formats relevant to solar energy modeling. iotools.get_cams iotools.read_cams iotools.parse_cams + iotools.get_acis_precipitation A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst index 729de3e457..0d1e4200f0 100644 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ b/docs/sphinx/source/whatsnew/v0.9.6.rst @@ -48,6 +48,8 @@ Enhancements * :py:func:`pvlib.iotools.get_psm3` now uses the new NSRDB 3.2.2 endpoint for hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) * The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) +* Added function to retrieve precipitation from the ACIS service from NOAA's RCCs: + :py:func:`~pvlib.iotools.get_precipitation_acis`. (:issue:`1293`, :pull:`1777`) Bug fixes ~~~~~~~~~ From 9bcac519dd0a5eab89adefc02636e92ad97393a2 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:13:20 -0400 Subject: [PATCH 04/19] tweak --- pvlib/iotools/acis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index cdfbc26ff7..115a4d1fac 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -40,7 +40,7 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, Returns ------- - data : pandas.DataFrame + data : pandas.Series Daily rainfall [mm] metadata : dict Coordinates for the selected grid cell From 14dd5982929091f0abb8b1dfe075ea479bde8f8f Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:24:39 -0400 Subject: [PATCH 05/19] better tests --- pvlib/tests/iotools/test_acis.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index dc68995622..1e3673ca31 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -8,15 +8,23 @@ from ..conftest import (RERUNS, RERUNS_DELAY, assert_series_equal) from requests import HTTPError + +@pytest.mark.parametrize('dataset,expected,lat,lon', [ + (1, [0.76, 1.78, 1.52, 0.76, 0.0], 40.0, -80.0) + (2, [0.05, 2.74, 1.43, 0.92, 0.0], 40.0083, -79.9653) + (3, [0.0, 2.79, 1.52, 1.02, 0.0], 40.0, -80.0) + (21, [0.6, 1.8, 1.9, 1.2, 0.0], 40.0, -80.9) +]) @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_acis_precipitation(): - precipitation, meta = get_acis_precipitation(40, -80, '2012-01-01', - '2012-01-05', 21) - idx = pd.date_range('2012-01-01', '2012-01-05', freq='d') - expected = pd.Series([0.6, 1.8, 1.9, 1.2, 0.0], index=idx, - name='precipitation') +def test_get_acis_precipitation(dataset, expected, lat, lon): + st = '2012-01-01' + ed = '2012-01-05' + precipitation, meta = get_acis_precipitation(40, -80, st, ed, dataset) + idx = pd.date_range(st, ed, freq='d') + expected = pd.Series(expected, index=idx, name='precipitation') assert_series_equal(precipitation, expected) + assert meta == {'latitude': lat, 'longitude': lon} @pytest.mark.remote_data From 60fb8ee664fd424ca877891f2c3bf7458e59bd78 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:25:35 -0400 Subject: [PATCH 06/19] better whatsnew --- docs/sphinx/source/whatsnew/v0.9.6.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst index 0d1e4200f0..31747ad7c8 100644 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ b/docs/sphinx/source/whatsnew/v0.9.6.rst @@ -48,8 +48,8 @@ Enhancements * :py:func:`pvlib.iotools.get_psm3` now uses the new NSRDB 3.2.2 endpoint for hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) * The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) -* Added function to retrieve precipitation from the ACIS service from NOAA's RCCs: - :py:func:`~pvlib.iotools.get_precipitation_acis`. (:issue:`1293`, :pull:`1777`) +* Added function to retrieve gridded precipitation data from the ACIS service + from NOAA's RCCs: :py:func:`~pvlib.iotools.get_precipitation_acis`. (:issue:`1293`, :pull:`1777`) Bug fixes ~~~~~~~~~ From 2719b8392832c65b53fdd155f21ab6c558b86b02 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:30:43 -0400 Subject: [PATCH 07/19] import in iotools/init --- pvlib/iotools/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 0a94e79f53..6a259afcbd 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -20,3 +20,4 @@ from pvlib.iotools.sodapro import get_cams # noqa: F401 from pvlib.iotools.sodapro import read_cams # noqa: F401 from pvlib.iotools.sodapro import parse_cams # noqa: F401 +from pvlib.iotools.acis import get_acis_precipitation # noqa: F401 From 167d49b364e963bb8ad2c43c340ea7c2d6d357f6 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 9 Jun 2023 20:51:30 -0400 Subject: [PATCH 08/19] fix all these broken things --- docs/sphinx/source/whatsnew/v0.9.6.rst | 2 +- pvlib/tests/iotools/test_acis.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst index 31747ad7c8..096d5624e9 100644 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ b/docs/sphinx/source/whatsnew/v0.9.6.rst @@ -49,7 +49,7 @@ Enhancements hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) * The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) * Added function to retrieve gridded precipitation data from the ACIS service - from NOAA's RCCs: :py:func:`~pvlib.iotools.get_precipitation_acis`. (:issue:`1293`, :pull:`1777`) + from NOAA's RCCs: :py:func:`~pvlib.iotools.get_acis_precipitation`. (:issue:`1293`, :pull:`1777`) Bug fixes ~~~~~~~~~ diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index 1e3673ca31..dcdbb57a4a 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -10,10 +10,10 @@ @pytest.mark.parametrize('dataset,expected,lat,lon', [ - (1, [0.76, 1.78, 1.52, 0.76, 0.0], 40.0, -80.0) - (2, [0.05, 2.74, 1.43, 0.92, 0.0], 40.0083, -79.9653) - (3, [0.0, 2.79, 1.52, 1.02, 0.0], 40.0, -80.0) - (21, [0.6, 1.8, 1.9, 1.2, 0.0], 40.0, -80.9) + (1, [0.76, 1.78, 1.52, 0.76, 0.0], 40.0, -80.0), + (2, [0.05, 2.74, 1.43, 0.92, 0.0], 40.0083, -79.9653), + (3, [0.0, 2.79, 1.52, 1.02, 0.0], 40.0, -80.0), + (21, [0.6, 1.8, 1.9, 1.2, 0.0], 40.0, -80.0), ]) @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) @@ -22,6 +22,8 @@ def test_get_acis_precipitation(dataset, expected, lat, lon): ed = '2012-01-05' precipitation, meta = get_acis_precipitation(40, -80, st, ed, dataset) idx = pd.date_range(st, ed, freq='d') + idx.name = 'date' + idx.freq = None expected = pd.Series(expected, index=idx, name='precipitation') assert_series_equal(precipitation, expected) assert meta == {'latitude': lat, 'longitude': lon} From 52b4d842fd9270dd991c4a3ed3f2f0db5c978ff2 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Thu, 15 Jun 2023 09:35:15 -0400 Subject: [PATCH 09/19] edits from review --- docs/sphinx/source/whatsnew/v0.9.6.rst | 2 +- pvlib/iotools/acis.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst index 096d5624e9..e132319b70 100644 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ b/docs/sphinx/source/whatsnew/v0.9.6.rst @@ -48,7 +48,7 @@ Enhancements * :py:func:`pvlib.iotools.get_psm3` now uses the new NSRDB 3.2.2 endpoint for hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) * The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) -* Added function to retrieve gridded precipitation data from the ACIS service +* Added function to retrieve daily precipitation estimates from the ACIS service from NOAA's RCCs: :py:func:`~pvlib.iotools.get_acis_precipitation`. (:issue:`1293`, :pull:`1777`) Bug fixes diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index 115a4d1fac..d8ee16021d 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -6,12 +6,14 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, url="https://data.rcc-acis.org/GridData", **kwargs): """ - Retrieve daily gridded precipitation data from the Applied Climate + Retrieve estimated daily precipitation data from the Applied Climate Information System (ACIS). The Applied Climate Information System (ACIS) was developed and is maintained by the NOAA Regional Climate Centers (RCCs) and brings together - climate data from many sources. + climate data from many sources. This function accesses precipitation + datasets covering the United States, although the exact domain + varies by dataset [1]_. Parameters ---------- @@ -31,7 +33,7 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, * 3: NRCC Hi-Res * 21: PRISM - See [1]_ for the full list of options. + See [2]_ for the full list of options. url : str, default: 'https://data.rcc-acis.org/GridData' API endpoint URL @@ -65,8 +67,8 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, References ---------- - .. [1] `ACIS Web Services `_ - .. [2] `ACIS Gridded Data `_ + .. [1] `ACIS Gridded Data `_ + .. [2] `ACIS Web Services `_ .. [3] `NRCC `_ .. [4] `Multisensor Precipitation Estimates `_ @@ -78,12 +80,17 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, ] params = { 'loc': f"{longitude},{latitude}", + # use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted 'sdate': pd.to_datetime(start).strftime('%Y-%m-%d'), 'edate': pd.to_datetime(end).strftime('%Y-%m-%d'), 'grid': str(dataset), 'elems': elems, 'output': 'json', - 'meta': ["ll"], # "elev" should work, but it errors for some databases + # [2]_ lists "ll" (lat/lon) and "elev" (elevation) as the available + # options for "meta". However, including "elev" when dataset=2 + # results in an "unknown meta elev" error. "ll" works on all + # datasets. There doesn't seem to be any other metadata available. + 'meta': ["ll"], } response = requests.post(url, json=params, headers={"Content-Type": "application/json"}, From de6333bb5ef1477c3a50a058cf7d5a0d4df07f8d Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Thu, 15 Jun 2023 09:37:51 -0400 Subject: [PATCH 10/19] correct PR number --- docs/sphinx/source/whatsnew/v0.9.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst index e132319b70..8cec556f9b 100644 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ b/docs/sphinx/source/whatsnew/v0.9.6.rst @@ -49,7 +49,7 @@ Enhancements hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) * The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) * Added function to retrieve daily precipitation estimates from the ACIS service - from NOAA's RCCs: :py:func:`~pvlib.iotools.get_acis_precipitation`. (:issue:`1293`, :pull:`1777`) + from NOAA's RCCs: :py:func:`~pvlib.iotools.get_acis_precipitation`. (:issue:`1293`, :pull:`1767`) Bug fixes ~~~~~~~~~ From 3201413154939be6fefc4f66b3d5f0e76c456b67 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 16 Jun 2023 12:32:37 -0400 Subject: [PATCH 11/19] split up code to be one function per dataset; add station functions --- docs/sphinx/source/reference/iotools.rst | 6 +- docs/sphinx/source/whatsnew/v0.9.6.rst | 7 +- pvlib/iotools/__init__.py | 6 +- pvlib/iotools/acis.py | 474 ++++++++++++++++++++--- pvlib/tests/iotools/test_acis.py | 209 +++++++++- 5 files changed, 626 insertions(+), 76 deletions(-) diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index 17ad5f630c..183f7b7bae 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -36,7 +36,11 @@ of sources and file formats relevant to solar energy modeling. iotools.get_cams iotools.read_cams iotools.parse_cams - iotools.get_acis_precipitation + iotools.get_acis_prism + iotools.get_acis_nrcc + iotools.get_acis_mpe + iotools.get_acis_station_data + iotools.get_acis_available_stations A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst index 8cec556f9b..6adbc3b8a2 100644 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ b/docs/sphinx/source/whatsnew/v0.9.6.rst @@ -48,8 +48,11 @@ Enhancements * :py:func:`pvlib.iotools.get_psm3` now uses the new NSRDB 3.2.2 endpoint for hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) * The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) -* Added function to retrieve daily precipitation estimates from the ACIS service - from NOAA's RCCs: :py:func:`~pvlib.iotools.get_acis_precipitation`. (:issue:`1293`, :pull:`1767`) +* Added functions to retrieve daily precipitation, temperature, and snowfall data + from the NOAA's ACIS service: :py:func:`~pvlib.iotools.get_acis_prism`, + :py:func:`~pvlib.iotools.get_acis_nrcc`, :py:func:`~pvlib.iotools.get_acis_mpe`, + :py:func:`~pvlib.iotools.get_acis_station_data`, and + :py:func:`~pvlib.iotools.get_acis_available_stations`. (:issue:`1293`, :pull:`1767`) Bug fixes ~~~~~~~~~ diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 6a259afcbd..cfb49e1414 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -20,4 +20,8 @@ from pvlib.iotools.sodapro import get_cams # noqa: F401 from pvlib.iotools.sodapro import read_cams # noqa: F401 from pvlib.iotools.sodapro import parse_cams # noqa: F401 -from pvlib.iotools.acis import get_acis_precipitation # noqa: F401 +from pvlib.iotools.acis import get_acis_prism # noqa: F401 +from pvlib.iotools.acis import get_acis_nrcc # noqa: F401 +from pvlib.iotools.acis import get_acis_mpe # noqa: F401 +from pvlib.iotools.acis import get_acis_station_data # noqa: F401 +from pvlib.iotools.acis import get_acis_available_stations # noqa: F401 diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index d8ee16021d..206ee93ea3 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -3,17 +3,82 @@ import numpy as np -def get_acis_precipitation(latitude, longitude, start, end, dataset, - url="https://data.rcc-acis.org/GridData", **kwargs): +VARIABLE_MAP = { + # time series names + 'pcpn': 'precipitation', + 'maxt': 'temp_air_max', + 'avgt': 'temp_air_average', + 'obst': 'temp_air_observation', + 'mint': 'temp_air_min', + 'snow': 'snowfall', + 'snwd': 'snowdepth', + + # metadata names + 'lat': 'latitude', + 'lon': 'longitude', + 'elev': 'altitude', +} + + +def _get_acis(start, end, params, map_variables, url, **kwargs): """ - Retrieve estimated daily precipitation data from the Applied Climate - Information System (ACIS). + generic helper for the public get_acis_X functions + """ + params = { + # use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted + 'sdate': pd.to_datetime(start).strftime('%Y-%m-%d'), + 'edate': pd.to_datetime(end).strftime('%Y-%m-%d'), + 'output': 'json', + **params, # endpoint-specific parameters + } + response = requests.post(url, + json=params, + headers={"Content-Type": "application/json"}, + **kwargs) + response.raise_for_status() + payload = response.json() + + # somewhat inconveniently, the ACIS API tends to return errors as "valid" + # responses instead of using proper HTTP error codes: + if "error" in payload: + raise requests.HTTPError(payload['error'], response=response) + + columns = ['date'] + [e['name'] for e in params['elems']] + df = pd.DataFrame(payload['data'], columns=columns) + df = df.set_index('date') + df.index = pd.to_datetime(df.index) + df.index.name = None + + metadata = payload['meta'] + # for StnData endpoint, unpack combination "ll" into lat, lon + ll = metadata.pop('ll', None) + if ll: + metadata['lon'], metadata['lat'] = ll + + try: + metadata['elev'] = metadata['elev'] * 0.3048 # feet to meters + except KeyError: + # some queries don't return elevation + pass + + if map_variables: + df = df.rename(columns=VARIABLE_MAP) + + for key in list(metadata.keys()): + if key in VARIABLE_MAP: + metadata[VARIABLE_MAP[key]] = metadata.pop(key) + + return df, metadata - The Applied Climate Information System (ACIS) was developed and is - maintained by the NOAA Regional Climate Centers (RCCs) and brings together - climate data from many sources. This function accesses precipitation - datasets covering the United States, although the exact domain - varies by dataset [1]_. + +def get_acis_prism(latitude, longitude, start, end, map_variables=True, + url="https://data.rcc-acis.org/GridData", **kwargs): + """ + Retrieve estimated daily precipitation and temperature data from PRISM + via the Applied Climate Information System (ACIS). + + Geographical coverage: approximately -130° to -65° in longitude and + 0° to 50° in latitude. Parameters ---------- @@ -25,16 +90,88 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, First day of the requested period end : datetime-like Last day of the requested period - dataset : int - A number indicating which gridded dataset to query. Options include: + map_variables : bool, default True + When True, rename data columns and metadata keys to pvlib variable + names where applicable. See variable :const:`VARIABLE_MAP`. + url : str, default: 'https://data.rcc-acis.org/GridData' + API endpoint URL + kwargs: + Optional parameters passed to ``requests.get``. - * 1: NRCC Interpolated - * 2: Multi-Sensor Precipitation Estimates - * 3: NRCC Hi-Res - * 21: PRISM - - See [2]_ for the full list of options. + Returns + ------- + data : pandas.DataFrame + Daily precipitation [mm] and temperature [Celsius] data + metadata : dict + Metadata of the selected grid cell + Raises + ------ + requests.HTTPError + A message from the ACIS server if the request is rejected + + Notes + ----- + PRISM data is aggregated from 12:00 to 12:00 UTC, meaning data labeled + May 26 reflects to the 24 hours ending at 7:00am Eastern Standard Time + on May 26. + + References + ---------- + .. [1] `PRISM `_ + .. [2] `ACIS Gridded Data `_ + .. [3] `ACIS Web Services `_ + + Examples + -------- + >>> df, meta = get_acis_prism(40, -80, '2020-01-01', '2020-12-31') + """ + elems = [ + {"name": "pcpn", "interval": "dly", "units": "mm"}, + {"name": "maxt", "interval": "dly", "units": "degreeC"}, + {"name": "mint", "interval": "dly", "units": "degreeC"}, + {"name": "avgt", "interval": "dly", "units": "degreeC"}, + {"name": "cdd", "interval": "dly", "units": "degreeC"}, + {"name": "hdd", "interval": "dly", "units": "degreeC"}, + {"name": "gdd", "interval": "dly", "units": "degreeC"}, + ] + params = { + 'loc': f"{longitude},{latitude}", + 'grid': "21", + 'elems': elems, + 'meta': ["ll", "elev"], + } + df, meta = _get_acis(start, end, params, map_variables, url, **kwargs) + df = df.replace(-999, np.nan) + return df, meta + + +def get_acis_nrcc(latitude, longitude, start, end, grid, map_variables=True, + url="https://data.rcc-acis.org/GridData", **kwargs): + """ + Retrieve estimated daily precipitation and temperature data from the + Northeast Regional Climate Center via the Applied Climate + Information System (ACIS). + + Geographical coverage: approximately -130° to -65° in longitude and + 0° to 50° in latitude. + + Parameters + ---------- + latitude : float + in decimal degrees, between -90 and 90, north is positive + longitude : float + in decimal degrees, between -180 and 180, east is positive + start : datetime-like + First day of the requested period + end : datetime-like + Last day of the requested period + grid : int + Options are either 1 (for "NRCC Interpolated") or 3 + (for "NRCC Hi-Resolution"). See [2]_ for details. + map_variables : bool, default True + When True, rename data columns and metadata keys to pvlib variable + names where applicable. See variable :const:`VARIABLE_MAP`. url : str, default: 'https://data.rcc-acis.org/GridData' API endpoint URL kwargs: @@ -42,10 +179,10 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, Returns ------- - data : pandas.Series - Daily rainfall [mm] + data : pandas.DataFrame + Daily precipitation [mm] and temperature [Celsius] data metadata : dict - Coordinates for the selected grid cell + Metadata of the selected grid cell Raises ------ @@ -54,59 +191,292 @@ def get_acis_precipitation(latitude, longitude, start, end, dataset, Notes ----- - The returned precipitation values are 24-hour aggregates, but + The returned values are 24-hour aggregates, but the aggregation period may not be midnight to midnight in local time. - For example, PRISM data is aggregated from 12:00 to 12:00 UTC, - meaning PRISM data labeled May 26 reflects to the 24 hours ending at - 7:00am Eastern Standard Time on May 26. + Check the ACIS and NRCC documentation for details. + + References + ---------- + .. [1] `NRCC `_ + .. [2] `ACIS Gridded Data `_ + .. [3] `ACIS Web Services `_ Examples -------- - >>> prism, metadata = get_acis_precipitation(40.0, -80.0, '2020-01-01', - >>> '2020-12-31', dataset=21) + >>> df, meta = get_acis_nrcc(40, -80, '2020-01-01', '2020-12-31', grid=1) + """ + elems = [ + {"name": "pcpn", "interval": "dly", "units": "mm"}, + {"name": "maxt", "interval": "dly", "units": "degreeC"}, + {"name": "mint", "interval": "dly", "units": "degreeC"}, + {"name": "avgt", "interval": "dly", "units": "degreeC"}, + {"name": "cdd", "interval": "dly", "units": "degreeC"}, + {"name": "hdd", "interval": "dly", "units": "degreeC"}, + {"name": "gdd", "interval": "dly", "units": "degreeC"}, + ] + params = { + 'loc': f"{longitude},{latitude}", + 'grid': grid, + 'elems': elems, + 'meta': ["ll", "elev"], + } + df, meta = _get_acis(start, end, params, map_variables, url, **kwargs) + df = df.replace(-999, np.nan) + return df, meta + + + +def get_acis_mpe(latitude, longitude, start, end, map_variables=True, + url="https://data.rcc-acis.org/GridData", **kwargs): + """ + Retrieve estimated daily Multi-sensor Precipitation Estimates + via the Applied Climate Information System (ACIS). + + This dataset covers the contiguous United States, Mexico, and parts of + Central America. + + Parameters + ---------- + latitude : float + in decimal degrees, between -90 and 90, north is positive + longitude : float + in decimal degrees, between -180 and 180, east is positive + start : datetime-like + First day of the requested period + end : datetime-like + Last day of the requested period + map_variables : bool, default True + When True, rename data columns and metadata keys to pvlib variable + names where applicable. See variable :const:`VARIABLE_MAP`. + url : str, default: 'https://data.rcc-acis.org/GridData' + API endpoint URL + kwargs: + Optional parameters passed to ``requests.get``. + + Returns + ------- + data : pandas.DataFrame + Daily precipitation [mm] data + metadata : dict + Coordinates of the selected grid cell + + Raises + ------ + requests.HTTPError + A message from the ACIS server if the request is rejected + + Notes + ----- + The returned values are 24-hour aggregates, but + the aggregation period may not be midnight to midnight in local time. + Check the ACIS and MPE documentation for details. References ---------- - .. [1] `ACIS Gridded Data `_ - .. [2] `ACIS Web Services `_ - .. [3] `NRCC `_ - .. [4] `Multisensor Precipitation Estimates + .. [1] `Multisensor Precipitation Estimates `_ - .. [5] `PRISM `_ + .. [2] `ACIS Gridded Data `_ + .. [3] `ACIS Web Services `_ + + Examples + -------- + >>> df, meta = get_acis_mpe(40, -80, '2020-01-01', '2020-12-31') """ elems = [ - # other variables exist, but are not of interest for PV modeling + # only precipitation is supported in this dataset {"name": "pcpn", "interval": "dly", "units": "mm"}, ] params = { 'loc': f"{longitude},{latitude}", - # use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted - 'sdate': pd.to_datetime(start).strftime('%Y-%m-%d'), - 'edate': pd.to_datetime(end).strftime('%Y-%m-%d'), - 'grid': str(dataset), + 'grid': "2", 'elems': elems, - 'output': 'json', - # [2]_ lists "ll" (lat/lon) and "elev" (elevation) as the available - # options for "meta". However, including "elev" when dataset=2 - # results in an "unknown meta elev" error. "ll" works on all - # datasets. There doesn't seem to be any other metadata available. - 'meta': ["ll"], + 'meta': ["ll"], # "elev" is not supported for this dataset + } + df, meta = _get_acis(start, end, params, map_variables, url, **kwargs) + df = df.replace(-999, np.nan) + return df, meta + + +def get_acis_station_data(station, start, end, trace_val=0.001, + map_variables=True, + url="https://data.rcc-acis.org/StnData", **kwargs): + """ + Retrieve weather station climate records via the Applied Climate + Information System (ACIS). + + This function can query data from stations all over the world. + The stations available in a given area can be listed using + :py:func:`get_acis_available_stations`. + + Parameters + ---------- + station : str + Identifier code for the station to query. Identifiers from many + station networks are accepted, including WBAN, COOP, FAA, WMO, GHCN, + and others. See [1]_ and [2]_ for details. + start : datetime-like + First day of the requested period + end : datetime-like + Last day of the requested period + map_variables : bool, default True + When True, rename data columns and metadata keys to pvlib variable + names where applicable. See variable :const:`VARIABLE_MAP`. + trace_val : float, default 0.001 + Value to replace "trace" values in the precipitation data + url : str, default: 'https://data.rcc-acis.org/GridData' + API endpoint URL + kwargs: + Optional parameters passed to ``requests.get``. + + Returns + ------- + data : pandas.DataFrame + Daily precipitation [mm], temperature [Celsius], and snow [cm] data + metadata : dict + station metadata + + Raises + ------ + requests.HTTPError + A message from the ACIS server if the request is rejected + + See Also + -------- + get_acis_available_stations + + References + ---------- + .. [1] `ACIS Web Services `_ + .. [2] `ACIS Metadata `_ + + Examples + -------- + >>> # Using an FAA code (Chicago O'Hare airport) + >>> df, meta = get_acis_station_data('ORD', '2020-01-01', '2020-12-31') + >>> + >>> # Look up available stations in a lat/lon rectangle, with data + >>> # available in the specified date range: + >>> stations = get_acis_available_stations([39.5, 40.5], [-80.5, -79.5], + ... '2020-01-01', '2020-01-03') + >>> stations['sids'][0] + ['369367 2', 'USC00369367 6', 'WYNP1 7'] + >>> df, meta = get_acis_station_data('369367', '2020-01-01', '2020-01-03') + """ + elems = [ + {"name": "maxt", "interval": "dly", "units": "degreeC"}, + {"name": "mint", "interval": "dly", "units": "degreeC"}, + {"name": "avgt", "interval": "dly", "units": "degreeC"}, + {"name": "obst", "interval": "dly", "units": "degreeC"}, + {"name": "pcpn", "interval": "dly", "units": "mm"}, + {"name": "snow", "interval": "dly", "units": "cm"}, + {"name": "snwd", "interval": "dly", "units": "cm"}, + {"name": "cdd", "interval": "dly", "units": "degreeC"}, + {"name": "hdd", "interval": "dly", "units": "degreeC"}, + {"name": "gdd", "interval": "dly", "units": "degreeC"}, + ] + params = { + 'sid': str(station), + 'elems': elems, + 'meta': ('name,state,sids,sid_dates,ll,elev,uid,county,' + 'climdiv,valid_daterange,tzo,network') } - response = requests.post(url, json=params, + df, metadata = _get_acis(start, end, params, map_variables, url, **kwargs) + df = df.replace("M", np.nan) + df = df.replace("T", trace_val) + df = df.astype(float) + return df, metadata + + +def get_acis_available_stations(latitude_range, longitude_range, + start=None, end=None, + url="https://data.rcc-acis.org/StnMeta", + **kwargs): + """ + List weather stations in a given area available from the + Applied Climate Information System (ACIS). + + The ``sids`` returned by this function can be used with + :py:func:`get_acis_station_data` to retrieve weather measurements + from the station. + + Parameters + ---------- + latitude_range : list + A 2-element list of [southern bound, northern bound] + in decimal degrees, between -90 and 90, north is positive + longitude_range : list + A 2-element list of [western bound, eastern bound] + in decimal degrees, between -180 and 180, east is positive + start : datetime-like, optional + If specified, return only stations that have data between ``start`` and + ``end. If not specified, all stations in the region are returned. + end : datetime-like, optional + See ``start`` + url : str, default: 'https://data.rcc-acis.org/StnMeta' + API endpoint URL + kwargs: + Optional parameters passed to ``requests.get``. + + Returns + ------- + stations : pandas.DataFrame + A dataframe of station metadata, one row per station. + The ``sids`` column contains IDs that can be used with + :py:func:`get_acis_station_data`. + + Raises + ------ + requests.HTTPError + A message from the ACIS server if the request is rejected + + See Also + -------- + get_acis_station_data + + References + ---------- + .. [1] `ACIS Web Services `_ + .. [2] `ACIS Metadata `_ + + Examples + -------- + >>> # Look up available stations in a lat/lon rectangle, with data + >>> # available in the specified date range: + >>> stations = get_acis_available_stations([39.5, 40.5], [-80.5, -79.5], + ... '2020-01-01', '2020-01-03') + >>> stations['sids'][0] + ['369367 2', 'USC00369367 6', 'WYNP1 7'] + """ + bbox = "{},{},{},{}".format( + longitude_range[0], + latitude_range[0], + longitude_range[1], + latitude_range[1], + ) + params = { + "bbox": bbox, + "meta": ("name,state,sids,sid_dates,ll,elev," + "uid,county,climdiv,tzo,network"), + } + if start is not None and end is not None: + params['elems'] = ['maxt', 'mint', 'avgt', 'obst', + 'pcpn', 'snow', 'snwd'] + params['sdate'] = pd.to_datetime(start).strftime('%Y-%m-%d') + params['edate'] = pd.to_datetime(end).strftime('%Y-%m-%d') + + response = requests.post(url, + json=params, headers={"Content-Type": "application/json"}, **kwargs) response.raise_for_status() payload = response.json() - if "error" in payload: raise requests.HTTPError(payload['error'], response=response) metadata = payload['meta'] - metadata['latitude'] = metadata.pop('lat') - metadata['longitude'] = metadata.pop('lon') - - df = pd.DataFrame(payload['data'], columns=['date', 'precipitation']) - rainfall = df.set_index('date')['precipitation'] - rainfall = rainfall.replace(-999, np.nan) - rainfall.index = pd.to_datetime(rainfall.index) - return rainfall, metadata + for station_record in metadata: + station_record['longitude'], station_record['latitude'] = \ + station_record.pop('ll') + + df = pd.DataFrame(metadata) + return df diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index dcdbb57a4a..b2a3937bf0 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -3,35 +3,204 @@ """ import pandas as pd +import numpy as np import pytest -from pvlib.iotools import get_acis_precipitation -from ..conftest import (RERUNS, RERUNS_DELAY, assert_series_equal) +from pvlib.iotools import ( + get_acis_prism, get_acis_nrcc, get_acis_mpe, + get_acis_station_data, get_acis_available_stations +) +from ..conftest import RERUNS, RERUNS_DELAY, assert_frame_equal from requests import HTTPError -@pytest.mark.parametrize('dataset,expected,lat,lon', [ - (1, [0.76, 1.78, 1.52, 0.76, 0.0], 40.0, -80.0), - (2, [0.05, 2.74, 1.43, 0.92, 0.0], 40.0083, -79.9653), - (3, [0.0, 2.79, 1.52, 1.02, 0.0], 40.0, -80.0), - (21, [0.6, 1.8, 1.9, 1.2, 0.0], 40.0, -80.0), -]) @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_acis_precipitation(dataset, expected, lat, lon): - st = '2012-01-01' - ed = '2012-01-05' - precipitation, meta = get_acis_precipitation(40, -80, st, ed, dataset) - idx = pd.date_range(st, ed, freq='d') - idx.name = 'date' - idx.freq = None - expected = pd.Series(expected, index=idx, name='precipitation') - assert_series_equal(precipitation, expected) - assert meta == {'latitude': lat, 'longitude': lon} +def test_get_acis_prism(): + # map_variables=True + df, meta = get_acis_prism(40.001, -80.001, '2020-01-01', '2020-01-02') + expected = pd.DataFrame( + [ + [0.5, 5, 0, 2.5, 0, 62, 0], + [0, 5, -3, 1, 0, 64, 0] + ], + columns=['precipitation', 'temp_air_max', 'temp_air_min', + 'temp_air_average', 'cdd', 'hdd', 'gdd'], + index=pd.to_datetime(['2020-01-01', '2020-01-02']), + ) + assert_frame_equal(df, expected) + expected_meta = {'latitude': 40, 'longitude': -80, 'altitude': 298.0944} + assert meta == expected_meta + + # map_variables=False + df, meta = get_acis_prism(40.001, -80.001, '2020-01-01', '2020-01-02', + map_variables=False) + expected.columns = ['pcpn', 'maxt', 'mint', 'avgt', 'cdd', 'hdd', 'gdd'] + assert_frame_equal(df, expected) + expected_meta = {'lat': 40, 'lon': -80, 'elev': 298.0944} + assert meta == expected_meta + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +@pytest.mark.parametrize('grid, expected', [ + (1, [[0.51, 5, 0, 2.5, 0, 62, 0]]), + (3, [[0.51, 5, -1, 2, 0, 63, 0]]) +]) +def test_get_acis_nrcc(grid, expected): + # map_variables=True + df, meta = get_acis_nrcc(40.001, -80.001, '2020-01-01', '2020-01-01', grid) + expected = pd.DataFrame( + expected, + columns=['precipitation', 'temp_air_max', 'temp_air_min', + 'temp_air_average', 'cdd', 'hdd', 'gdd'], + index=pd.to_datetime(['2020-01-01']), + ) + assert_frame_equal(df, expected) + expected_meta = {'latitude': 40., 'longitude': -80., 'altitude': 356.9208} + assert meta == pytest.approx(expected_meta) + + # map_variables=False + df, meta = get_acis_nrcc(40.001, -80.001, '2020-01-01', '2020-01-01', grid, + map_variables=False) + expected.columns = ['pcpn', 'maxt', 'mint', 'avgt', 'cdd', 'hdd', 'gdd'] + assert_frame_equal(df, expected) + expected_meta = {'lat': 40., 'lon': -80., 'elev': 356.9208} + assert meta == pytest.approx(expected_meta) @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_acis_precipitation_error(): +def test_get_acis_nrcc_error(): with pytest.raises(HTTPError, match='invalid grid'): # 50 is not a valid dataset (or "grid", in ACIS lingo) - get_acis_precipitation(40, -80, '2012-01-01', '2012-01-05', 50) + _ = get_acis_nrcc(40, -80, '2012-01-01', '2012-01-01', 50) + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_acis_mpe(): + # map_variables=True + df, meta = get_acis_mpe(40.001, -80.001, '2020-01-01', '2020-01-02') + expected = pd.DataFrame( + {'precipitation': [0.4, 0.0]}, + index=pd.to_datetime(['2020-01-01', '2020-01-02']), + ) + assert_frame_equal(df, expected) + expected_meta = {'latitude': 40.0083, 'longitude': -79.9653} + assert meta == expected_meta + + # map_variables=False + df, meta = get_acis_mpe(40.001, -80.001, '2020-01-01', '2020-01-02', + map_variables=False) + expected.columns = ['pcpn'] + assert_frame_equal(df, expected) + expected_meta = {'lat': 40.0083, 'lon': -79.9653} + assert meta == expected_meta + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_acis_station_data(): + # map_variables=True + df, meta = get_acis_station_data('ORD', '2020-01-10', '2020-01-12', + trace_val=-99) + expected = pd.DataFrame( + [[10., 2., 6., np.nan, 21.34, 0., 0., 0., 59., 0.], + [3., -4., -0.5, np.nan, 9.4, 5.3, 0., 0., 65., 0.], + [-1., -5., -3., np.nan, -99, -99, 5., 0., 68., 0.]], + columns=['temp_air_max', 'temp_air_min', 'temp_air_average', + 'temp_air_observation', 'precipitation', 'snowfall', + 'snowdepth', 'cdd', 'hdd', 'gdd'], + index=pd.to_datetime(['2020-01-10', '2020-01-11', '2020-01-12']), + ) + assert_frame_equal(df, expected) + expected_meta = { + 'uid': 48, + 'state': 'IL', + 'name': 'CHICAGO OHARE INTL AP', + 'altitude': 204.8256, + 'latitude': 41.96017, + 'longitude': -87.93164 + } + expected_meta = { + 'valid_daterange': [ + ['1958-11-01', '2023-06-15'], + ['1958-11-01', '2023-06-15'], + ['1958-11-01', '2023-06-15'], + [], + ['1958-11-01', '2023-06-15'], + ['1958-11-01', '2023-06-15'], + ['1958-11-01', '2023-06-15'], + ['1958-11-01', '2023-06-15'], + ['1958-11-01', '2023-06-15'], + ['1958-11-01', '2023-06-15'] + ], + 'name': 'CHICAGO OHARE INTL AP', + 'sids': ['94846 1', '111549 2', 'ORD 3', '72530 4', 'KORD 5', + 'USW00094846 6', 'ORD 7', 'USW00094846 32'], + 'county': '17031', + 'state': 'IL', + 'climdiv': 'IL02', + 'uid': 48, + 'tzo': -6.0, + 'sid_dates': [ + ['94846 1', '1989-01-19', '9999-12-31'], + ['94846 1', '1958-10-30', '1989-01-01'], + ['111549 2', '1989-01-19', '9999-12-31'], + ['111549 2', '1958-10-30', '1989-01-01'], + ['ORD 3', '1989-01-19', '9999-12-31'], + ['ORD 3', '1958-10-30', '1989-01-01'], + ['72530 4', '1989-01-19', '9999-12-31'], + ['72530 4', '1958-10-30', '1989-01-01'], + ['KORD 5', '1989-01-19', '9999-12-31'], + ['KORD 5', '1958-10-30', '1989-01-01'], + ['USW00094846 6', '1989-01-19', '9999-12-31'], + ['USW00094846 6', '1958-10-30', '1989-01-01'], + ['ORD 7', '1989-01-19', '9999-12-31'], + ['ORD 7', '1958-10-30', '1989-01-01'], + ['USW00094846 32', '1989-01-19', '9999-12-31'], + ['USW00094846 32', '1958-10-30', '1989-01-01']], + 'altitude': 204.8256, + 'longitude': -87.93164, + 'latitude': 41.96017 + } + assert meta == expected_meta + + # map_variables=False + df, meta = get_acis_station_data('ORD', '2020-01-10', '2020-01-12', + trace_val=-99, map_variables=False) + expected.columns = ['maxt', 'mint', 'avgt', 'obst', 'pcpn', 'snow', + 'snwd', 'cdd', 'hdd', 'gdd'] + assert_frame_equal(df, expected) + expected_meta['lat'] = expected_meta.pop('latitude') + expected_meta['lon'] = expected_meta.pop('longitude') + expected_meta['elev'] = expected_meta.pop('altitude') + assert meta == expected_meta + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_acis_available_stations(): + # use a very narrow bounding box to hopefully make this test less likely + # to fail due to new stations being added in the future + lat, lon = 39.8986, -80.1656 + stations = get_acis_available_stations([lat - 0.0001, lat + 0.0001], + [lon - 0.0001, lon + 0.0001]) + assert len(stations) == 1 + station = stations.iloc[0] + + # test the more relevant values + assert station['name'] == 'WAYNESBURG 1 E' + assert station['sids'] == ['369367 2', 'USC00369367 6', 'WYNP1 7'] + assert station['state'] == 'PA' + assert station['altitude'] == 940. + assert station['tzo'] == -5.0 + assert station['latitude'] == lat + assert station['longitude'] == lon + + # check that start/end work as filters + stations = get_acis_available_stations([lat - 0.0001, lat + 0.0001], + [lon - 0.0001, lon + 0.0001], + start='1900-01-01', + end='1900-01-02') + assert stations.empty From 77246187a61aef490e9614d8b4a45b7b0f52ca28 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 16 Jun 2023 12:47:44 -0400 Subject: [PATCH 12/19] fix tests --- pvlib/iotools/acis.py | 3 ++- pvlib/tests/iotools/test_acis.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index 206ee93ea3..5fba46aff1 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -409,7 +409,7 @@ def get_acis_available_stations(latitude_range, longitude_range, in decimal degrees, between -180 and 180, east is positive start : datetime-like, optional If specified, return only stations that have data between ``start`` and - ``end. If not specified, all stations in the region are returned. + ``end``. If not specified, all stations in the region are returned. end : datetime-like, optional See ``start`` url : str, default: 'https://data.rcc-acis.org/StnMeta' @@ -475,6 +475,7 @@ def get_acis_available_stations(latitude_range, longitude_range, metadata = payload['meta'] for station_record in metadata: + station_record['altitude'] = station_record.pop('elev') station_record['longitude'], station_record['latitude'] = \ station_record.pop('ll') diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index b2a3937bf0..ed97fbdeaf 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -44,7 +44,7 @@ def test_get_acis_prism(): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) @pytest.mark.parametrize('grid, expected', [ (1, [[0.51, 5, 0, 2.5, 0, 62, 0]]), - (3, [[0.51, 5, -1, 2, 0, 63, 0]]) + (3, [[0.51, 5, -1, 2.0, 0, 63, 0]]) ]) def test_get_acis_nrcc(grid, expected): # map_variables=True From 12cd83a76c9bee343568d4830e9f48b3340390e3 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 27 Jun 2023 09:49:15 -0400 Subject: [PATCH 13/19] Apply suggestions from code review Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com> --- pvlib/iotools/acis.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index 5fba46aff1..e596f73574 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -96,7 +96,7 @@ def get_acis_prism(latitude, longitude, start, end, map_variables=True, url : str, default: 'https://data.rcc-acis.org/GridData' API endpoint URL kwargs: - Optional parameters passed to ``requests.get``. + Optional parameters passed to ``requests.post``. Returns ------- @@ -124,7 +124,8 @@ def get_acis_prism(latitude, longitude, start, end, map_variables=True, Examples -------- - >>> df, meta = get_acis_prism(40, -80, '2020-01-01', '2020-12-31') + >>> df, meta = pvlib.iotools.get_acis_prism( + >>> latitude=40, longitude=-80, start='2020-01-01', end='2020-12-31') """ elems = [ {"name": "pcpn", "interval": "dly", "units": "mm"}, @@ -175,7 +176,7 @@ def get_acis_nrcc(latitude, longitude, start, end, grid, map_variables=True, url : str, default: 'https://data.rcc-acis.org/GridData' API endpoint URL kwargs: - Optional parameters passed to ``requests.get``. + Optional parameters passed to ``requests.post``. Returns ------- @@ -251,7 +252,7 @@ def get_acis_mpe(latitude, longitude, start, end, map_variables=True, url : str, default: 'https://data.rcc-acis.org/GridData' API endpoint URL kwargs: - Optional parameters passed to ``requests.get``. + Optional parameters passed to ``requests.post``. Returns ------- @@ -326,7 +327,7 @@ def get_acis_station_data(station, start, end, trace_val=0.001, url : str, default: 'https://data.rcc-acis.org/GridData' API endpoint URL kwargs: - Optional parameters passed to ``requests.get``. + Optional parameters passed to ``requests.post``. Returns ------- @@ -415,7 +416,7 @@ def get_acis_available_stations(latitude_range, longitude_range, url : str, default: 'https://data.rcc-acis.org/StnMeta' API endpoint URL kwargs: - Optional parameters passed to ``requests.get``. + Optional parameters passed to ``requests.post``. Returns ------- From 16fbafda009b4bedb41501fca7985ffcd3535528 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 27 Jun 2023 10:33:15 -0400 Subject: [PATCH 14/19] more from review --- pvlib/iotools/acis.py | 46 +++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index e596f73574..5ee9cb9c3e 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -50,10 +50,12 @@ def _get_acis(start, end, params, map_variables, url, **kwargs): df.index.name = None metadata = payload['meta'] - # for StnData endpoint, unpack combination "ll" into lat, lon - ll = metadata.pop('ll', None) - if ll: - metadata['lon'], metadata['lat'] = ll + + try: + # for StnData endpoint, unpack combination "ll" into lat, lon + metadata['lon'], metadata['lat'] = metadata.pop('ll') + except KeyError: + pass try: metadata['elev'] = metadata['elev'] * 0.3048 # feet to meters @@ -77,8 +79,14 @@ def get_acis_prism(latitude, longitude, start, end, map_variables=True, Retrieve estimated daily precipitation and temperature data from PRISM via the Applied Climate Information System (ACIS). - Geographical coverage: approximately -130° to -65° in longitude and - 0° to 50° in latitude. + ACIS [2]_, [3]_ aggregates and provides access to climate data + from many underlying sources. This function retrieves daily data from + the Parameter-elevation Regressions on Independent Slopes Model + (PRISM) [1]_, a gridded precipitation and temperature model + from Oregon State University. + + Geographical coverage: US, Central America, and part of South America. + Approximately 0° to 50° in latitude and -130° to -65° in longitude. Parameters ---------- @@ -124,8 +132,8 @@ def get_acis_prism(latitude, longitude, start, end, map_variables=True, Examples -------- - >>> df, meta = pvlib.iotools.get_acis_prism( - >>> latitude=40, longitude=-80, start='2020-01-01', end='2020-12-31') + >>> from pvlib.iotools import get_acis_prism + >>> df, meta = get_acis_prism(40, 80, '2020-01-01', '2020-12-31') """ elems = [ {"name": "pcpn", "interval": "dly", "units": "mm"}, @@ -154,8 +162,12 @@ def get_acis_nrcc(latitude, longitude, start, end, grid, map_variables=True, Northeast Regional Climate Center via the Applied Climate Information System (ACIS). - Geographical coverage: approximately -130° to -65° in longitude and - 0° to 50° in latitude. + ACIS [2]_, [3]_ aggregates and provides access to climate data + from many underlying sources. This function retrieves daily data from + Cornell's Northeast Regional Climate Center (NRCC) [1]_. + + Geographical coverage: US, Central America, and part of South America. + Approximately 0° to 50° in latitude and -130° to -65° in longitude. Parameters ---------- @@ -204,6 +216,7 @@ def get_acis_nrcc(latitude, longitude, start, end, grid, map_variables=True, Examples -------- + >>> from pvlib.iotools import get_acis_nrcc >>> df, meta = get_acis_nrcc(40, -80, '2020-01-01', '2020-12-31', grid=1) """ elems = [ @@ -233,6 +246,11 @@ def get_acis_mpe(latitude, longitude, start, end, map_variables=True, Retrieve estimated daily Multi-sensor Precipitation Estimates via the Applied Climate Information System (ACIS). + ACIS [2]_, [3]_ aggregates and provides access to climate data + from many underlying sources. This function retrieves daily data from + the National Weather Service's Multi-sensor Precipitation Estimates + (MPE) [1]_, a gridded precipitation model. + This dataset covers the contiguous United States, Mexico, and parts of Central America. @@ -281,6 +299,7 @@ def get_acis_mpe(latitude, longitude, start, end, map_variables=True, Examples -------- + >>> from pvlib.iotools import get_acis_mpe >>> df, meta = get_acis_mpe(40, -80, '2020-01-01', '2020-12-31') """ elems = [ @@ -305,6 +324,10 @@ def get_acis_station_data(station, start, end, trace_val=0.001, Retrieve weather station climate records via the Applied Climate Information System (ACIS). + ACIS [1]_, [2]_ aggregates and provides access to climate data + from many underlying sources. This function retrieves measurements + from ground stations belonging to various global networks. + This function can query data from stations all over the world. The stations available in a given area can be listed using :py:func:`get_acis_available_stations`. @@ -353,10 +376,12 @@ def get_acis_station_data(station, start, end, trace_val=0.001, Examples -------- >>> # Using an FAA code (Chicago O'Hare airport) + >>> from pvlib.iotools import get_acis_station_data >>> df, meta = get_acis_station_data('ORD', '2020-01-01', '2020-12-31') >>> >>> # Look up available stations in a lat/lon rectangle, with data >>> # available in the specified date range: + >>> from pvlib.iotools import get_acis_available_stations >>> stations = get_acis_available_stations([39.5, 40.5], [-80.5, -79.5], ... '2020-01-01', '2020-01-03') >>> stations['sids'][0] @@ -443,6 +468,7 @@ def get_acis_available_stations(latitude_range, longitude_range, -------- >>> # Look up available stations in a lat/lon rectangle, with data >>> # available in the specified date range: + >>> from pvlib.iotools import get_acis_available_stations >>> stations = get_acis_available_stations([39.5, 40.5], [-80.5, -79.5], ... '2020-01-01', '2020-01-03') >>> stations['sids'][0] From fa85d3add0a77bb16e67cf553a8c524864dbad8d Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 27 Jun 2023 14:24:44 -0400 Subject: [PATCH 15/19] add degree days to df descriptions --- pvlib/iotools/acis.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index 5ee9cb9c3e..f58252bf59 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -109,7 +109,8 @@ def get_acis_prism(latitude, longitude, start, end, map_variables=True, Returns ------- data : pandas.DataFrame - Daily precipitation [mm] and temperature [Celsius] data + Daily precipitation [mm], temperature [Celsius], and degree day + [Celsius-days] data metadata : dict Metadata of the selected grid cell @@ -193,7 +194,8 @@ def get_acis_nrcc(latitude, longitude, start, end, grid, map_variables=True, Returns ------- data : pandas.DataFrame - Daily precipitation [mm] and temperature [Celsius] data + Daily precipitation [mm], temperature [Celsius], and degree day + [Celsius-days] data metadata : dict Metadata of the selected grid cell @@ -355,7 +357,8 @@ def get_acis_station_data(station, start, end, trace_val=0.001, Returns ------- data : pandas.DataFrame - Daily precipitation [mm], temperature [Celsius], and snow [cm] data + Daily precipitation [mm], temperature [Celsius], snow [cm], and degree day + [Celsius-days] data data metadata : dict station metadata From 9cbe851286336cf70f3e7202cdab39f9cf2ff7f2 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 27 Jun 2023 14:35:55 -0400 Subject: [PATCH 16/19] add degree day columns to variable map --- pvlib/iotools/acis.py | 3 +++ pvlib/tests/iotools/test_acis.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index f58252bf59..4664b22f4e 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -10,6 +10,9 @@ 'avgt': 'temp_air_average', 'obst': 'temp_air_observation', 'mint': 'temp_air_min', + 'cdd': 'cooling_degree_days', + 'hdd': 'heating_degree_days', + 'gdd': 'growing_degree_days', 'snow': 'snowfall', 'snwd': 'snowdepth', diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index ed97fbdeaf..2282387ba7 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -24,7 +24,8 @@ def test_get_acis_prism(): [0, 5, -3, 1, 0, 64, 0] ], columns=['precipitation', 'temp_air_max', 'temp_air_min', - 'temp_air_average', 'cdd', 'hdd', 'gdd'], + 'temp_air_average', 'cooling_degree_days', + 'heating_degree_days', 'growing_degree_days'], index=pd.to_datetime(['2020-01-01', '2020-01-02']), ) assert_frame_equal(df, expected) @@ -52,7 +53,8 @@ def test_get_acis_nrcc(grid, expected): expected = pd.DataFrame( expected, columns=['precipitation', 'temp_air_max', 'temp_air_min', - 'temp_air_average', 'cdd', 'hdd', 'gdd'], + 'temp_air_average', 'cooling_degree_days', + 'heating_degree_days', 'growing_degree_days'], index=pd.to_datetime(['2020-01-01']), ) assert_frame_equal(df, expected) @@ -110,7 +112,8 @@ def test_get_acis_station_data(): [-1., -5., -3., np.nan, -99, -99, 5., 0., 68., 0.]], columns=['temp_air_max', 'temp_air_min', 'temp_air_average', 'temp_air_observation', 'precipitation', 'snowfall', - 'snowdepth', 'cdd', 'hdd', 'gdd'], + 'snowdepth', 'cooling_degree_days', + 'heating_degree_days', 'growing_degree_days'], index=pd.to_datetime(['2020-01-10', '2020-01-11', '2020-01-12']), ) assert_frame_equal(df, expected) From 30b37032873f802cf887e14184dd622eedd2cdfd Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 27 Jun 2023 14:36:14 -0400 Subject: [PATCH 17/19] fix valid daterange test oversight --- pvlib/tests/iotools/test_acis.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index 2282387ba7..8458e9b930 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -167,6 +167,9 @@ def test_get_acis_station_data(): 'longitude': -87.93164, 'latitude': 41.96017 } + # don't check valid dates since they get extended every day + meta.pop("valid_daterange") + expected_meta.pop("valid_daterange") assert meta == expected_meta # map_variables=False @@ -178,6 +181,7 @@ def test_get_acis_station_data(): expected_meta['lat'] = expected_meta.pop('latitude') expected_meta['lon'] = expected_meta.pop('longitude') expected_meta['elev'] = expected_meta.pop('altitude') + meta.pop("valid_daterange") assert meta == expected_meta From 3b4873928d1a3f876f8a3d776b2fb1c947de5115 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 28 Jun 2023 08:20:00 -0400 Subject: [PATCH 18/19] cm -> mm for snow variables --- pvlib/iotools/acis.py | 8 ++++---- pvlib/tests/iotools/test_acis.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index 4664b22f4e..9e8cea0104 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -360,8 +360,8 @@ def get_acis_station_data(station, start, end, trace_val=0.001, Returns ------- data : pandas.DataFrame - Daily precipitation [mm], temperature [Celsius], snow [cm], and degree day - [Celsius-days] data data + Daily precipitation [mm], temperature [Celsius], snow [mm], and + degree day [Celsius-days] data metadata : dict station metadata @@ -400,8 +400,8 @@ def get_acis_station_data(station, start, end, trace_val=0.001, {"name": "avgt", "interval": "dly", "units": "degreeC"}, {"name": "obst", "interval": "dly", "units": "degreeC"}, {"name": "pcpn", "interval": "dly", "units": "mm"}, - {"name": "snow", "interval": "dly", "units": "cm"}, - {"name": "snwd", "interval": "dly", "units": "cm"}, + {"name": "snow", "interval": "dly", "units": "mm"}, + {"name": "snwd", "interval": "dly", "units": "mm"}, {"name": "cdd", "interval": "dly", "units": "degreeC"}, {"name": "hdd", "interval": "dly", "units": "degreeC"}, {"name": "gdd", "interval": "dly", "units": "degreeC"}, diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index 8458e9b930..7b62cdf145 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -108,8 +108,8 @@ def test_get_acis_station_data(): trace_val=-99) expected = pd.DataFrame( [[10., 2., 6., np.nan, 21.34, 0., 0., 0., 59., 0.], - [3., -4., -0.5, np.nan, 9.4, 5.3, 0., 0., 65., 0.], - [-1., -5., -3., np.nan, -99, -99, 5., 0., 68., 0.]], + [3., -4., -0.5, np.nan, 9.4, 53.3, 0., 0., 65., 0.], + [-1., -5., -3., np.nan, -99, -99, 51., 0., 68., 0.]], columns=['temp_air_max', 'temp_air_min', 'temp_air_average', 'temp_air_observation', 'precipitation', 'snowfall', 'snowdepth', 'cooling_degree_days', From a03a774130efbed86e6cbde1b9fb82224fdd9926 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Thu, 29 Jun 2023 08:41:44 -0400 Subject: [PATCH 19/19] revert back to cm for snow --- pvlib/iotools/acis.py | 4 ++-- pvlib/tests/iotools/test_acis.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/acis.py b/pvlib/iotools/acis.py index 9e8cea0104..3be16cfa4c 100644 --- a/pvlib/iotools/acis.py +++ b/pvlib/iotools/acis.py @@ -400,8 +400,8 @@ def get_acis_station_data(station, start, end, trace_val=0.001, {"name": "avgt", "interval": "dly", "units": "degreeC"}, {"name": "obst", "interval": "dly", "units": "degreeC"}, {"name": "pcpn", "interval": "dly", "units": "mm"}, - {"name": "snow", "interval": "dly", "units": "mm"}, - {"name": "snwd", "interval": "dly", "units": "mm"}, + {"name": "snow", "interval": "dly", "units": "cm"}, + {"name": "snwd", "interval": "dly", "units": "cm"}, {"name": "cdd", "interval": "dly", "units": "degreeC"}, {"name": "hdd", "interval": "dly", "units": "degreeC"}, {"name": "gdd", "interval": "dly", "units": "degreeC"}, diff --git a/pvlib/tests/iotools/test_acis.py b/pvlib/tests/iotools/test_acis.py index 7b62cdf145..8458e9b930 100644 --- a/pvlib/tests/iotools/test_acis.py +++ b/pvlib/tests/iotools/test_acis.py @@ -108,8 +108,8 @@ def test_get_acis_station_data(): trace_val=-99) expected = pd.DataFrame( [[10., 2., 6., np.nan, 21.34, 0., 0., 0., 59., 0.], - [3., -4., -0.5, np.nan, 9.4, 53.3, 0., 0., 65., 0.], - [-1., -5., -3., np.nan, -99, -99, 51., 0., 68., 0.]], + [3., -4., -0.5, np.nan, 9.4, 5.3, 0., 0., 65., 0.], + [-1., -5., -3., np.nan, -99, -99, 5., 0., 68., 0.]], columns=['temp_air_max', 'temp_air_min', 'temp_air_average', 'temp_air_observation', 'precipitation', 'snowfall', 'snowdepth', 'cooling_degree_days',