Use pytest remotedata (#896)

* enable pytest_remotedata for psm3

* add pytest_remotedata to test_pvgis.py

* add @pytest.mark.remote_data to other network dependent tests

* use up-to-date numpy, pandas

* install netcdf4 from pip instead of conda

* install pytest-remotedata from pip for conda 3.5

* readd @network to test_midc.py

* add pytest-remotedata to py35-min for travis

* add to what's new

* point coverage at run with --remote-data

* add --remote-data to travis run for coveralls to pickup

* Revert "point coverage at run with --remote-data"

This reverts commit e73eeb0.

* Update docs/sphinx/source/whatsnew/v0.7.2.rst

Co-Authored-By: Will Holmgren <[email protected]>

* Update docs/sphinx/source/whatsnew/v0.7.2.rst

Co-Authored-By: Will Holmgren <[email protected]>

* Revert "install netcdf4 from pip instead of conda"

This reverts commit 6fa8ecf.

* fix linelength in what's new

* add pytest-remotedata to Azure MacOS

* move --remote-data tests to conda_linux

* add remote_data decorators to tests requiring network

* only install pytest-remotedata from pip, explain

* fix tests directory name and url

* document use of --remote-data flag in Contributing>Testing

Co-authored-by: Will Holmgren <[email protected]>

Author: CameronTStark

.travis.yml

@@ -61,7 +61,7 @@ install:
     - pip install -e .
 
 script:
-    - pytest pvlib --cov=pvlib --cov-report term-missing
+    - pytest pvlib --remote-data --cov=pvlib --cov-report term-missing
 
 after_script:
   - if [[ $TASK == "coverage" ]]; then

azure-pipelines.yml

@@ -26,9 +26,8 @@ jobs:
       versionSpec: '$(python.version)'
 
   - script: |
-      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures
+      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata
       pip install -e .
-      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
     displayName: 'Test with pytest'
 
@@ -77,7 +76,7 @@ jobs:
   - script: |
       source activate test_env
       export NREL_API_KEY=$(nrelApiKey)
-      pytest pvlib --junitxml=junit/test-results.xml --cov --cov-report=xml --cov-report=html
+      pytest pvlib --remote-data --junitxml=junit/test-results.xml --cov --cov-report=xml --cov-report=html
     displayName: 'pytest'
   # - script: |
   #     source activate test_env
@@ -129,7 +128,6 @@ jobs:
     displayName: 'List installed dependencies'
   - script: |
       call activate test_env
-      set NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml
     displayName: 'pytest'
   - task: PublishTestResults@2
@@ -157,9 +155,8 @@ jobs:
       versionSpec: '$(python.version)'
 
   - script: |
-      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures
+      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata
       pip install -e .
-      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov=pvlib --cov-report=xml --cov-report=html
     displayName: 'Test with pytest'
 

ci/requirements-py35-min.yml

@@ -16,3 +16,4 @@ dependencies:
         - numpy==1.12.0
         - pandas==0.18.1
         - pytest-rerunfailures # conda version is >3.6
+        - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1

ci/requirements-py35.yml

@@ -27,3 +27,4 @@ dependencies:
         - nrel-pysam>=2.0
         - pvfactors==1.0.1
         - pytest-rerunfailures # conda version is >3.6
+        - pytest-remotedata # needs > 0.3.1

ci/requirements-py36.yml

@@ -17,6 +17,7 @@ dependencies:
     - pytest-cov
     - pytest-mock
     - pytest-rerunfailures
+    - pytest-remotedata
     - pytest-timeout
     - python=3.6
     - pytz

ci/requirements-py37.yml

@@ -18,6 +18,7 @@ dependencies:
     - pytest-mock
     - pytest-timeout
     - pytest-rerunfailures
+    - pytest-remotedata
     - python=3.7
     - pytz
     - requests

docs/sphinx/source/contributing.rst

@@ -262,7 +262,7 @@ Testing
 Developers **must** include comprehensive tests for any additions or
 modifications to pvlib. New unit test code should be placed in the
 corresponding test module in the
-`pvlib/test <https://github.com/pvlib/pvlib-python/tree/master/pvlib/test>`_
+`pvlib/tests <https://github.com/pvlib/pvlib-python/tree/master/pvlib/tests>`_
 directory.
 
 A pull request will automatically run the tests for you on a variety of
@@ -298,6 +298,14 @@ location of a test failure. As described in :ref:`code-style`, pvlib
 code does not use ``print`` or ``logging`` calls, and this also applies
 to the test suite (with rare exceptions).
 
+To include all network-dependent tests, include the ``--remote-data`` flag to
+your ``pytest`` call:
+
+``pytest pvlib --remote-data``
+
+And consider adding ``@pytest.mark.remote_data`` to any network dependent test
+you submit for a PR.
+
 pvlib-python contains 3 "layers" of code: functions, PVSystem/Location,
 and ModelChain. Contributors will need to add tests that correspond to
 the layers that they modify.

docs/sphinx/source/whatsnew/v0.7.2.rst

@@ -25,12 +25,16 @@ Bug fixes
   passing ``tz=datetime.timezone.utc`` (:pull:`879`)
 * Fix documentation homepage title to "pvlib python" based on first heading on
   the page. (:pull:`890`) (:issue:`888`)
+* Implement `pytest-remotedata <https://github.com/astropy/pytest-remotedata>`_
+  to increase test suite speed. Requires ``--remote-data`` pytest flag to
+  execute data retrieval tests over a network.
 * Fix missing
   `0.7.0 what's new <https://pvlib-python.readthedocs.io/en/stable/whatsnew.html#v0-7-0-december-18-2019>`_
   entries about changes to ``PVSystem.pvwatts_ac``. Delete unreleased
   0.6.4 what's new file. (:issue:`898`)
 * Compatibility with cftime 1.1. (:issue:`895`)
 
+
 Documentation
 ~~~~~~~~~~~~~
 * Add NumFOCUS affiliation to Sphinx documentation :pull:`862`
@@ -47,4 +51,4 @@ Contributors
 * Cameron T. Stark (:ghuser:`camerontstark`)
 * Will Holmgren (:ghuser:`wholmgren`)
 * Kevin Anderson (:ghuser:`kanderso-nrel`)
-* Karthikeyan Singaravelan (:ghuser:`tirkarthi`)
+* Karthikeyan Singaravelan (:ghuser:`tirkarthi`)

pvlib/tests/iotools/test_epw.py

@@ -1,4 +1,5 @@
 from pandas.util.testing import network
+import pytest
 
 from pvlib.iotools import epw
 from conftest import DATA_DIR
@@ -11,6 +12,7 @@ def test_read_epw():
 
 
 @network
+@pytest.mark.remote_data
 def test_read_epw_remote():
     url = 'https://energyplus.net/weather-download/europe_wmo_region_6/NLD//NLD_Amsterdam.062400_IWEC/NLD_Amsterdam.062400_IWEC.epw'
     epw.read_epw(url)

pvlib/tests/iotools/test_midc.py

@@ -65,6 +65,7 @@ def test_read_midc_var_mapping_as_arg(test_mapping):
 
 
 @network
+@pytest.mark.remote_data
 def test_read_midc_raw_data_from_nrel():
     start_ts = pd.Timestamp('20181018')
     end_ts = pd.Timestamp('20181019')

pvlib/tests/iotools/test_psm3.py

@@ -70,6 +70,7 @@ def assert_psm3_equal(header, data, expected):
     assert (data.index.tzinfo.zone == 'Etc/GMT%+d' % -header['Time Zone'])
 
 
+@pytest.mark.remote_data
 @pytest.mark.flaky(reruns=5, reruns_delay=2)
 def test_get_psm3_tmy(nrel_api_key):
     """test get_psm3 with a TMY"""
@@ -79,6 +80,7 @@ def test_get_psm3_tmy(nrel_api_key):
     assert_psm3_equal(header, data, expected)
 
 
+@pytest.mark.remote_data
 @pytest.mark.flaky(reruns=5, reruns_delay=2)
 def test_get_psm3_singleyear(nrel_api_key):
     """test get_psm3 with a single year"""
@@ -88,6 +90,7 @@ def test_get_psm3_singleyear(nrel_api_key):
     assert_psm3_equal(header, data, expected)
 
 
+@pytest.mark.remote_data
 @pytest.mark.flaky(reruns=5, reruns_delay=2)
 def test_get_psm3_check_leap_day(nrel_api_key):
     _, data_2012 = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key,
@@ -102,6 +105,7 @@ def test_get_psm3_check_leap_day(nrel_api_key):
                           (LATITUDE, LONGITUDE, nrel_api_key, 'bad', 60),
                           (LATITUDE, LONGITUDE, nrel_api_key, '2017', 15),
                           ])
+@pytest.mark.remote_data
 @pytest.mark.flaky(reruns=5, reruns_delay=2)
 def test_get_psm3_tmy_errors(
     latitude, longitude, api_key, names, interval

pvlib/tests/iotools/test_pvgis.py

@@ -67,6 +67,7 @@ def csv_meta(meta_expected):
         in meta_expected['outputs']['tmy_hourly']['variables'].items()]
 
 
+@pytest.mark.remote_data
 def test_get_pvgis_tmy(expected, month_year_expected, inputs_expected,
                        meta_expected):
     data, months_selected, inputs, meta = get_pvgis_tmy(45, 8)
@@ -92,6 +93,7 @@ def test_get_pvgis_tmy(expected, month_year_expected, inputs_expected,
     assert meta == meta_expected
 
 
+@pytest.mark.remote_data
 def test_get_pvgis_tmy_kwargs(userhorizon_expected):
     _, _, inputs, _ = get_pvgis_tmy(45, 8, usehorizon=False)
     assert inputs['meteo_data']['use_horizon'] is False
@@ -109,13 +111,15 @@ def test_get_pvgis_tmy_kwargs(userhorizon_expected):
     assert inputs['meteo_data']['year_max'] == 2016
 
 
+@pytest.mark.remote_data
 def test_get_pvgis_tmy_basic(expected, meta_expected):
     data, _, _, _ = get_pvgis_tmy(45, 8, outputformat='basic')
     # check each column of output separately
     for outvar in meta_expected['outputs']['tmy_hourly']['variables'].keys():
         assert np.allclose(data[outvar], expected[outvar])
 
 
+@pytest.mark.remote_data
 def test_get_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
                            meta_expected, csv_meta):
     data, months_selected, inputs, meta = get_pvgis_tmy(
@@ -138,6 +142,7 @@ def test_get_pvgis_tmy_csv(expected, month_year_expected, inputs_expected,
         assert meta_value in csv_meta
 
 
+@pytest.mark.remote_data
 def test_get_pvgis_tmy_epw(expected, epw_meta):
     data, _, _, meta = get_pvgis_tmy(
         45, 8, outputformat='epw')
@@ -148,6 +153,7 @@ def test_get_pvgis_tmy_epw(expected, epw_meta):
     assert meta == epw_meta
 
 
+@pytest.mark.remote_data
 def test_get_pvgis_tmy_error():
     err_msg = 'outputformat: Incorrect value.'
     with pytest.raises(requests.HTTPError, match=err_msg):

pvlib/tests/iotools/test_srml.py

@@ -14,6 +14,7 @@ def test_read_srml():
 
 
 @network
+@pytest.mark.remote_data
 def test_read_srml_remote():
     srml.read_srml('http://solardat.uoregon.edu/download/Archive/EUPO1801.txt')
 
@@ -40,6 +41,7 @@ def test_read_srml_nans_exist():
     ('http://solardat.uoregon.edu/download/Archive/EUPO1612.txt',
      2016, 12),
 ])
+@pytest.mark.remote_data
 def test_read_srml_dt_index(url, year, month):
     data = srml.read_srml(url)
     start = pd.Timestamp('{:04d}{:02d}01 00:00'.format(year, month))
@@ -63,6 +65,7 @@ def test_map_columns(column, expected):
 
 
 @network
+@pytest.mark.remote_data
 def test_read_srml_month_from_solardat():
     url = 'http://solardat.uoregon.edu/download/Archive/EUPO1801.txt'
     file_data = srml.read_srml(url)
@@ -71,6 +74,7 @@ def test_read_srml_month_from_solardat():
 
 
 @network
+@pytest.mark.remote_data
 def test_15_minute_dt_index():
     data = srml.read_srml_month_from_solardat('TW', 2019, 4, 'RQ')
     start = pd.Timestamp('20190401 00:00')
@@ -83,6 +87,7 @@ def test_15_minute_dt_index():
 
 
 @network
+@pytest.mark.remote_data
 def test_hourly_dt_index():
     data = srml.read_srml_month_from_solardat('CD', 1986, 4, 'PH')
     start = pd.Timestamp('19860401 00:00')

pvlib/tests/iotools/test_surfrad.py

@@ -1,5 +1,6 @@
 import pandas as pd
 from pandas.util.testing import network
+import pytest
 
 from pvlib.iotools import surfrad
 from conftest import DATA_DIR
@@ -10,6 +11,7 @@
 
 
 @network
+@pytest.mark.remote_data
 def test_read_surfrad_network():
     # If this test begins failing, SURFRAD's data structure or data
     # archive may have changed.

pvlib/tests/iotools/test_tmy.py

@@ -1,6 +1,7 @@
 from pandas.util.testing import network
 import numpy as np
 import pandas as pd
+import pytest
 from pvlib.iotools import tmy
 from conftest import DATA_DIR
 
@@ -17,6 +18,7 @@ def test_read_tmy3():
 
 
 @network
+@pytest.mark.remote_data
 def test_read_tmy3_remote():
     url = 'http://rredc.nrel.gov/solar/old_data/nsrdb/1991-2005/data/tmy3/703165TYA.CSV'
     tmy.read_tmy3(url)

pvlib/tests/test_forecast.py

@@ -59,6 +59,7 @@ def model(request):
 
 
 @requires_siphon
+@pytest.mark.remote_data
 def test_process_data(model):
     for how in ['liujordan', 'clearsky_scaling']:
         if model.raw_data.empty:
@@ -75,6 +76,7 @@ def test_process_data(model):
 
 
 @requires_siphon
+@pytest.mark.remote_data
 def test_bad_kwarg_get_data():
     # For more information on why you would want to pass an unknown keyword
     # argument, see Github issue #745.
@@ -85,6 +87,7 @@ def test_bad_kwarg_get_data():
 
 
 @requires_siphon
+@pytest.mark.remote_data
 def test_bad_kwarg_get_processed_data():
     # For more information on why you would want to pass an unknown keyword
     # argument, see Github issue #745.
@@ -95,6 +98,7 @@ def test_bad_kwarg_get_processed_data():
 
 
 @requires_siphon
+@pytest.mark.remote_data
 def test_how_kwarg_get_processed_data():
     amodel = NAM()
     data = amodel.get_processed_data(_latitude, _longitude, _start, _end,
@@ -103,6 +107,7 @@ def test_how_kwarg_get_processed_data():
 
 
 @requires_siphon
+@pytest.mark.remote_data
 def test_vert_level():
     amodel = NAM()
     vert_level = 5000
@@ -111,6 +116,7 @@ def test_vert_level():
 
 
 @requires_siphon
+@pytest.mark.remote_data
 def test_datetime():
     amodel = NAM()
     start = datetime.now(tz=timezone.utc)
@@ -119,6 +125,7 @@ def test_datetime():
 
 
 @requires_siphon
+@pytest.mark.remote_data
 def test_queryvariables():
     amodel = GFS()
     new_variables = ['u-component_of_wind_height_above_ground']

setup.py

@@ -42,7 +42,7 @@
                     'pytz',
                     'requests']
 TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock',
-                 'pytest-timeout', 'pytest-rerunfailures']
+                 'pytest-timeout', 'pytest-rerunfailures', 'pytest-remotedata']
 EXTRAS_REQUIRE = {
     'optional': ['ephem', 'cython', 'netcdf4', 'nrel-pysam', 'numba',
                  'pvfactors', 'scipy', 'siphon', 'tables'],