ENH: Create psm3.py (#694)

* Create psm3.py

initial concept for psm3 reader

* fix well known text for geometric POINT

* need to strip all leading whitespace, and only one whitespace between,
and longitude is first, then latitude
* fix list() not []
* fix ATTRIBUTES and URL
* coerce all to float, clean up types in header, cast datevec columns to
int, create datetimeindex from datevec columns, set index, get errors
from json when returning HTTPError

Signed-off-by: Mark Mikofski <[email protected]>

* TST: add test for getting psm3

* update docs, closes #592

* stickler, default interval

* add psm3 to api

* use astype, fix typos, change param names to match API

* use pandas, better variable names, comments, set all dtypes

* stickler fixes

* api_key as param

* TST: decorate test_get_psm3 with needs_pandas_0_22

* api_key and email are required according to NREL

* other params full name and affiliation are kwargs
* set reason to pvlib something as suggested by NREL
* catch some errors, like 403 forbidden, are in content, not json, only
some errors like bad well known text WKT gemetry are in JSON
* use python-2 json decode error
* add another test to capture both bad API key and bad WKT
* add note on how to register for key, why they need email, and when it
would be used, and warning about DEMO_KEY being rate limited

Signed-off-by: Mark Mikofski <[email protected]>

* stickler: remove unused JSON import

* remove non-ASCII quotation marks around DEMO_KEY in warning

* remove "required" for api_key param in get_psm3

Co-Authored-By: mikofski <[email protected]>

* rewording for "email" param in get_psm3

Co-Authored-By: mikofski <[email protected]>

* clarify WKT formating and errors

* in raise docstring, change "returns" to "raises", and clarify that
 either r.json()['errors'] or r.content is the exception arg depending
 on the type of HTTP error, and give examples
* add warning in docstring that data are limited to NSRDB locations
* comment on WKT formatting and add TODO to consider making a
 format_WKT() function in tools.py one day
* add response kwarg to HTTPError for better user exception handling
* in test_psm3 add comments to clarify the tests for errors and add two
 more test for bad "names" and interval args

* DOC: defaults in argtype for get_psm3

Author: mikofski

docs/sphinx/source/api.rst

@@ -341,6 +341,7 @@ relevant to solar energy modeling.
    iotools.get_ecmwf_macc
    iotools.read_crn
    iotools.read_solrad
+   iotools.get_psm3
 
 A :py:class:`~pvlib.location.Location` object may be created from metadata
 in some files.

docs/sphinx/source/whatsnew/v0.6.2.rst

@@ -29,6 +29,7 @@ Enhancements
 * Add US CRN data reader to :ref:`iotools`.
 * Add SOLRAD data reader to :ref:`iotools`.
 * Add EPW data reader to :ref:`iotools`. (:issue:`591`)
+* Add PSM3 reader to :ref:`iotools`. (:issue:`592`)
 
 Bug fixes
 ~~~~~~~~~

pvlib/data/test_psm3.csv

@@ -10,3 +10,4 @@
 from pvlib.iotools.ecmwf_macc import get_ecmwf_macc  # noqa: F401
 from pvlib.iotools.crn import read_crn  # noqa: F401
 from pvlib.iotools.solrad import read_solrad  # noqa: F401
+from pvlib.iotools.psm3 import get_psm3  # noqa: F401

pvlib/iotools/__init__.py

@@ -0,0 +1,185 @@
+
+"""
+Get PSM3 TMY
+see https://developer.nrel.gov/docs/solar/nsrdb/psm3_data_download/
+"""
+
+import io
+import requests
+import pandas as pd
+# Python-2 compatible JSONDecodeError
+try:
+    from json import JSONDecodeError
+except ImportError:
+    JSONDecodeError = ValueError
+
+URL = "http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv"
+
+# 'relative_humidity', 'total_precipitable_water' are not available
+ATTRIBUTES = [
+    'air_temperature', 'dew_point', 'dhi', 'dni', 'ghi', 'surface_albedo',
+    'surface_pressure', 'wind_direction', 'wind_speed']
+PVLIB_PYTHON = 'pvlib python'
+
+
+def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60,
+             full_name=PVLIB_PYTHON, affiliation=PVLIB_PYTHON):
+    """
+    Get PSM3 data
+
+    Parameters
+    ----------
+    latitude : float or int
+        in decimal degrees, between -90 and 90, north is positive
+    longitude : float or int
+        in decimal degrees, between -180 and 180, east is positive
+    api_key : str
+        NREL Developer Network API key
+    email : str
+        NREL API uses this to automatically communicate messages back
+        to the user only if necessary
+    names : str, default 'tmy'
+        PSM3 API parameter specifing year or TMY variant to download, see notes
+        below for options
+    interval : int, default 60
+        interval size in minutes, can only be either 30 or 60
+    full_name : str, default 'pvlib python'
+        optional
+    affiliation : str, default 'pvlib python'
+        optional
+
+    Returns
+    -------
+    headers : dict
+        metadata from NREL PSM3 about the record, see notes for fields
+    data : pandas.DataFrame
+        timeseries data from NREL PSM3
+
+    Raises
+    ------
+    requests.HTTPError
+        if the request response status is not ok, then the ``'errors'`` field
+        from the JSON response or any error message in the content will be
+        raised as an exception, for example if the `api_key` was rejected or if
+        the coordinates were not found in the NSRDB
+
+    Notes
+    -----
+    The required NREL developer key, `api_key`, is available for free by
+    registering at the `NREL Developer Network <https://developer.nrel.gov/>`_.
+
+    .. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
+        result in rejected requests.
+
+    The PSM3 API `names` parameter must be a single value from the following
+    list::
+
+        ['1998', '1999', '2000', '2001', '2002', '2003', '2004', '2005',
+         '2006', '2007', '2008', '2009', '2010', '2011', '2012', '2013',
+         '2014', '2015', '2016', '2017', 'tmy', 'tmy-2016', 'tmy-2017',
+         'tdy-2017', 'tgy-2017']
+
+    The return is a tuple with two items. The first item is a header with
+    metadata from NREL PSM3 about the record containing the following fields:
+
+    * Source
+    * Location ID
+    * City
+    * State
+    * Country
+    * Latitude
+    * Longitude
+    * Time Zone
+    * Elevation
+    * Local Time Zone
+    * Dew Point Units
+    * DHI Units
+    * DNI Units
+    * GHI Units
+    * Temperature Units
+    * Pressure Units
+    * Wind Direction Units
+    * Wind Speed
+    * Surface Albedo Units
+    * Version
+
+    The second item is a dataframe with the timeseries data downloaded.
+
+    .. warning:: PSM3 is limited to data found in the NSRDB, please consult the
+        references below for locations with available data
+
+    See Also
+    --------
+    pvlib.iotools.read_tmy2, pvlib.iotools.read_tmy3
+
+    References
+    ----------
+
+    * `NREL Developer Network - Physical Solar Model (PSM) v3
+      <https://developer.nrel.gov/docs/solar/nsrdb/psm3_data_download/>`_
+    * `NREL National Solar Radiation Database (NSRDB)
+      <https://nsrdb.nrel.gov/>`_
+
+    """
+    # The well know text (WKT) representation of geometry notation is strict.
+    # A POINT object is a string with longitude first, then the latitude, with
+    # four decimals each, and exactly one space between them.
+    longitude = ('%9.4f' % longitude).strip()
+    latitude = ('%8.4f' % latitude).strip()
+    # TODO: make format_WKT(object_type, *args) in tools.py
+
+    # required query-string parameters for request to PSM3 API
+    params = {
+        'api_key': api_key,
+        'full_name': full_name,
+        'email': email,
+        'affiliation': affiliation,
+        'reason': PVLIB_PYTHON,
+        'mailing_list': 'false',
+        'wkt': 'POINT(%s %s)' % (longitude, latitude),
+        'names': names,
+        'attributes':  ','.join(ATTRIBUTES),
+        'leap_day': 'false',
+        'utc': 'false',
+        'interval': interval
+    }
+    # request CSV download from NREL PSM3
+    response = requests.get(URL, params=params)
+    if not response.ok:
+        # if the API key is rejected, then the response status will be 403
+        # Forbidden, and then the error is in the content and there is no JSON
+        try:
+            errors = response.json()['errors']
+        except JSONDecodeError:
+            errors = response.content.decode('utf-8')
+        raise requests.HTTPError(errors, response=response)
+    # the CSV is in the response content as a UTF-8 bytestring
+    # to use pandas we need to create a file buffer from the response
+    fbuf = io.StringIO(response.content.decode('utf-8'))
+    # The first 2 lines of the response are headers with metadat
+    header_fields = fbuf.readline().split(',')
+    header_fields[-1] = header_fields[-1].strip()  # strip trailing newline
+    header_values = fbuf.readline().split(',')
+    header_values[-1] = header_values[-1].strip()  # strip trailing newline
+    header = dict(zip(header_fields, header_values))
+    # the response is all strings, so set some header types to numbers
+    header['Local Time Zone'] = int(header['Local Time Zone'])
+    header['Time Zone'] = int(header['Time Zone'])
+    header['Latitude'] = float(header['Latitude'])
+    header['Longitude'] = float(header['Longitude'])
+    header['Elevation'] = int(header['Elevation'])
+    # get the column names so we can set the dtypes
+    columns = fbuf.readline().split(',')
+    columns[-1] = columns[-1].strip()  # strip trailing newline
+    dtypes = dict.fromkeys(columns, float)  # all floats except datevec
+    dtypes.update(Year=int, Month=int, Day=int, Hour=int, Minute=int)
+    data = pd.read_csv(
+        fbuf, header=None, names=columns, dtype=dtypes,
+        delimiter=',', lineterminator='\n')  # skip carriage returns \r
+    # the response 1st 5 columns are a date vector, convert to datetime
+    dtidx = pd.to_datetime(
+        data[['Year', 'Month', 'Day', 'Hour', 'Minute']])
+    # in USA all timezones are intergers
+    tz = 'Etc/GMT%+d' % -header['Time Zone']
+    data.index = pd.DatetimeIndex(dtidx).tz_localize(tz)
+    return header, data

pvlib/iotools/psm3.py

@@ -94,6 +94,10 @@ def pandas_0_22():
     return parse_version(pd.__version__) >= parse_version('0.22.0')
 
 
+needs_pandas_0_22 = pytest.mark.skipif(
+    not pandas_0_22(), reason='requires pandas 0.22 or greater')
+
+
 def has_spa_c():
     try:
         from pvlib.spa_c_files.spa_py import spa_calc

pvlib/test/conftest.py

@@ -0,0 +1,66 @@
+"""
+test iotools for PSM3
+"""
+
+import os
+from pvlib.iotools import psm3
+from conftest import needs_pandas_0_22
+import numpy as np
+import pandas as pd
+import pytest
+from requests import HTTPError
+
+BASEDIR = os.path.abspath(os.path.dirname(__file__))
+PROJDIR = os.path.dirname(BASEDIR)
+DATADIR = os.path.join(PROJDIR, 'data')
+TEST_DATA = os.path.join(DATADIR, 'test_psm3.csv')
+LATITUDE, LONGITUDE = 40.5137, -108.5449
+HEADER_FIELDS = [
+    'Source', 'Location ID', 'City', 'State', 'Country', 'Latitude',
+    'Longitude', 'Time Zone', 'Elevation', 'Local Time Zone',
+    'Dew Point Units', 'DHI Units', 'DNI Units', 'GHI Units',
+    'Temperature Units', 'Pressure Units', 'Wind Direction Units',
+    'Wind Speed', 'Surface Albedo Units', 'Version']
+PVLIB_EMAIL = '[email protected]'
+DEMO_KEY = 'DEMO_KEY'
+
+
+@needs_pandas_0_22
+def test_get_psm3():
+    """test get_psm3"""
+    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL)
+    expected = pd.read_csv(TEST_DATA)
+    # check datevec columns
+    assert np.allclose(data.Year, expected.Year)
+    assert np.allclose(data.Month, expected.Month)
+    assert np.allclose(data.Day, expected.Day)
+    assert np.allclose(data.Hour, expected.Hour)
+    assert np.allclose(data.Minute, expected.Minute)
+    # check data columns
+    assert np.allclose(data.GHI, expected.GHI)
+    assert np.allclose(data.DNI, expected.DNI)
+    assert np.allclose(data.DHI, expected.DHI)
+    assert np.allclose(data.Temperature, expected.Temperature)
+    assert np.allclose(data.Pressure, expected.Pressure)
+    assert np.allclose(data['Dew Point'], expected['Dew Point'])
+    assert np.allclose(data['Surface Albedo'], expected['Surface Albedo'])
+    assert np.allclose(data['Wind Speed'], expected['Wind Speed'])
+    assert np.allclose(data['Wind Direction'], expected['Wind Direction'])
+    # check header
+    for hf in HEADER_FIELDS:
+        assert hf in header
+    # check timezone
+    assert (data.index.tzinfo.zone == 'Etc/GMT%+d' % -header['Time Zone'])
+    # check errors
+    with pytest.raises(HTTPError):
+        # HTTP 403 forbidden because api_key is rejected
+        psm3.get_psm3(LATITUDE, LONGITUDE, api_key='BAD', email=PVLIB_EMAIL)
+    with pytest.raises(HTTPError):
+        # coordinates were not found in the NSRDB
+        psm3.get_psm3(51, -5, DEMO_KEY, PVLIB_EMAIL)
+    with pytest.raises(HTTPError):
+        # names is not one of the available options
+        psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL, names='bad')
+    with pytest.raises(HTTPError):
+        # intervals can only be 30 or 60 minutes
+        psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL, interval=15)