diff --git a/.gitignore b/.gitignore index b7cd3bef6f..1667960d27 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,8 @@ coverage.xml # Ignore Mac DS_store files *.DS_Store + +# airspeed velocity +env +results + diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000000..b0e27c5214 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,93 @@ +Benchmarks +========== + +pvlib includes a small number of performance benchmarking tests. These +tests are run using +[airspeed velocity](https://asv.readthedocs.io/en/stable/) (ASV). + +The basic structure of the tests and how to run them is described below. +We refer readers to the ASV documentation for more details. The AstroPy +[documentation](https://github.com/astropy/astropy-benchmarks/tree/master) +may also be helpful. + +The test configuration is described in [asv.conf.json](asv.conf.json). +The performance tests are located in the [benchmarks](benchmarks) directory. + +Comparing timings +----------------- + +Note that, unlike pytest, the asv tests require changes to be committed +to git before they can be tested. The ``run`` command takes a positional +argument to describe the range of git commits or branches to be tested. +For example, if your feature branch is named ``feature``, a useful asv +run may be (from the same directory as `asv.conf.json`): + +``` +$ asv run master..feature +``` + +This will generate timings for every commit between the two specified +revisions. If you only care about certain commits, you can run them by +their git hashes directly like this: + +``` +$ asv run e42f8d24^! +``` + +Note: depending on what shell they use, Windows users may need to use +double-carets: + +``` +$ asv run e42f8d24^^! +``` + +You can then compare the timing results of two commits: + +``` +$ asv compare 0ff98b62 e42f8d24 + +All benchmarks: + + before after ratio + [0ff98b62] [e42f8d24] + ++ 3.90±0.6ms 31.3±5ms 8.03 irradiance.Irradiance.time_aoi + 3.12±0.4ms 2.94±0.2ms 0.94 irradiance.Irradiance.time_aoi_projection + 256±9ms 267±10ms 1.05 irradiance.Irradiance.time_dirindex +``` + +The `ratio` column shows the ratio of `after / before` timings. For this +example, the `aoi` function was slowed down on purpose to demonstrate +the comparison. + +Generating an HTML report +------------------------- + +asv can generate a collection of interactive plots of benchmark timings across +a commit history. First, generate timings for a series of commits, like: + +``` +$ asv run v0.6.0..v0.8.0 +``` + +Next, generate the HTML report: + +``` +$ asv publish +``` + +Finally, start a http server to view the test results: + +``` +$ asv preview + +``` + + +Nightly benchmarking +-------------------- + +The benchmarks are run nightly for new commits to pvlib-python/master. + +- Timing results: https://pvlib-benchmarker.github.io/pvlib-benchmarks/ +- Information on the process: https://github.com/pvlib-benchmarker/pvlib-benchmarks diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 0000000000..b586ae1848 --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,186 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "pvlib-python", + + // The project's homepage + "project_url": "https://pvlib-python.readthedocs.io", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": "..", + + // The Python project's subdirectory in your repo. If missing or + // the empty string, the project is assumed to be located at the root + // of the repository. + // "repo_subdir": "", + + // Customizable commands for building, installing, and + // uninstalling the project. See asv.conf.json documentation. + // + // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + // "build_command": [ + // "python setup.py build", + // "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" + // ], + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + // "branches": ["master"], // for git + // "branches": ["default"], // for mercurial + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + //"install_timeout": 600, + + // the base URL to show a commit for the project. + "show_commit_url": "http://github.com/pvlib/pvlib-python/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["dummy"], // this gets excluded below + + // The list of conda channel names to be searched for benchmark + // dependency packages in the specified order + "conda_channels": ["defaults", "conda-forge"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list or empty string indicates to just test against the default + // (latest) version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed via + // pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + "matrix": {}, // pvlib dependencies specified in the `include` list below + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "six": null}, // don't run without six on conda + // ], + // + // "include": [ + // // additional env for python2.7 + // {"python": "2.7", "numpy": "1.8"}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""}, + // ], + + // asv takes the Cartesian product of the 'matrix' version spec, so it's + // no good for specifying sets of versions. Instead we'll use the + // 'include' machinery to specify each set of versions to benchmark. + // However, the 'pythons' key ends up generating an undesirable extra + // environment, so we 'exclude' it. + "include": [ + // minimum supported versions + { + "python": "3.6", + "numpy": "1.12.0", + "pandas": "0.22.0", + "scipy": "1.2.0", + // Note: these don't have a minimum in setup.py + "pytables": "3.6.1", + "ephem": "3.7.6.0", + }, + // latest versions available + { + "python": "3.8", + "numpy": "", + "pandas": "", + "scipy": "", + "pytables": "", + "ephem": "", + }, + ], + "exclude": [ + {"python": "dummy"} + ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + // "env_dir": "env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + // "results_dir": "results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + // "html_dir": "html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache results of the recent builds in each + // environment, making them faster to install next time. This is + // the number of builds to keep, per environment. + // "build_cache_size": 2, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // }, + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // }, +} diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmarks/benchmarks/irradiance.py b/benchmarks/benchmarks/irradiance.py new file mode 100644 index 0000000000..75cf6e965f --- /dev/null +++ b/benchmarks/benchmarks/irradiance.py @@ -0,0 +1,68 @@ +""" +ASV benchmarks for irradiance.py +""" + +import pandas as pd +from pvlib import irradiance, location + + +class Irradiance: + + def setup(self): + self.times = pd.date_range(start='20180601', freq='1min', + periods=14400) + self.days = pd.date_range(start='20180601', freq='d', periods=30) + self.location = location.Location(40, -80) + self.solar_position = self.location.get_solarposition(self.times) + self.clearsky_irradiance = self.location.get_clearsky(self.times) + self.tilt = 20 + self.azimuth = 180 + self.aoi = irradiance.aoi(self.tilt, self.azimuth, + self.solar_position.apparent_zenith, + self.solar_position.azimuth) + + def time_get_extra_radiation(self): + irradiance.get_extra_radiation(self.days) + + def time_aoi(self): + irradiance.aoi(self.tilt, self.azimuth, + self.solar_position.apparent_zenith, + self.solar_position.azimuth) + + def time_aoi_projection(self): + irradiance.aoi_projection(self.tilt, self.azimuth, + self.solar_position.apparent_zenith, + self.solar_position.azimuth) + + def time_get_ground_diffuse(self): + irradiance.get_ground_diffuse(self.tilt, self.clearsky_irradiance.ghi) + + def time_get_total_irradiance(self): + irradiance.get_total_irradiance(self.tilt, self.azimuth, + self.solar_position.apparent_zenith, + self.solar_position.azimuth, + self.clearsky_irradiance.dni, + self.clearsky_irradiance.ghi, + self.clearsky_irradiance.dhi) + + def time_disc(self): + irradiance.disc(self.clearsky_irradiance.ghi, + self.solar_position.apparent_zenith, + self.times) + + def time_dirint(self): + irradiance.dirint(self.clearsky_irradiance.ghi, + self.solar_position.apparent_zenith, + self.times) + + def time_dirindex(self): + irradiance.dirindex(self.clearsky_irradiance.ghi, + self.clearsky_irradiance.ghi, + self.clearsky_irradiance.dni, + self.solar_position.apparent_zenith, + self.times) + + def time_erbs(self): + irradiance.erbs(self.clearsky_irradiance.ghi, + self.solar_position.apparent_zenith, + self.times) diff --git a/benchmarks/benchmarks/location.py b/benchmarks/benchmarks/location.py new file mode 100644 index 0000000000..2356ca1e7e --- /dev/null +++ b/benchmarks/benchmarks/location.py @@ -0,0 +1,51 @@ +""" +ASV benchmarks for location.py +""" + +import pandas as pd +import pvlib +from pkg_resources import parse_version + + +def set_solar_position(obj): + obj.location = pvlib.location.Location(32, -110, altitude=700, + tz='Etc/GMT+7') + obj.times = pd.date_range(start='20180601', freq='3min', + periods=1440) + obj.days = pd.date_range(start='20180101', freq='d', periods=365, + tz=obj.location.tz) + obj.solar_position = obj.location.get_solarposition(obj.times) + + +class Location: + + def setup(self): + set_solar_position(self) + + # GH 502 + def time_location_get_airmass(self): + self.location.get_airmass(solar_position=self.solar_position) + + def time_location_get_solarposition(self): + self.location.get_solarposition(times=self.times) + + def time_location_get_clearsky(self): + self.location.get_clearsky(times=self.times, + solar_position=self.solar_position) + + +class Location_0_6_1: + + def setup(self): + if parse_version(pvlib.__version__) < parse_version('0.6.1'): + raise NotImplementedError + + set_solar_position(self) + + def time_location_get_sun_rise_set_transit_pyephem(self): + self.location.get_sun_rise_set_transit(times=self.days, + method='pyephem') + + def time_location_get_sun_rise_set_transit_spa(self): + self.location.get_sun_rise_set_transit(times=self.days, + method='spa') diff --git a/benchmarks/benchmarks/solarposition.py b/benchmarks/benchmarks/solarposition.py new file mode 100644 index 0000000000..594f8259cd --- /dev/null +++ b/benchmarks/benchmarks/solarposition.py @@ -0,0 +1,43 @@ +""" +ASV benchmarks for solarposition.py +""" + +import pandas as pd +import pvlib +from pvlib import solarposition + +from pkg_resources import parse_version + + +if parse_version(pvlib.__version__) >= parse_version('0.6.1'): + sun_rise_set_transit_spa = solarposition.sun_rise_set_transit_spa +else: + sun_rise_set_transit_spa = solarposition.get_sun_rise_set_transit + + +class SolarPosition: + + def setup(self): + self.times = pd.date_range(start='20180601', freq='1min', + periods=14400) + self.times_localized = self.times.tz_localize('Etc/GMT+7') + self.lat = 35.1 + self.lon = -106.6 + + # GH 512 + def time_ephemeris(self): + solarposition.ephemeris(self.times, self.lat, self.lon) + + # GH 512 + def time_ephemeris_localized(self): + solarposition.ephemeris(self.times_localized, self.lat, self.lon) + + def time_spa_python(self): + solarposition.spa_python(self.times_localized[::5], self.lat, self.lon) + + def time_sun_rise_set_transit_spa(self): + sun_rise_set_transit_spa(self.times_localized[::30], + self.lat, self.lon) + + def time_nrel_earthsun_distance(self): + solarposition.nrel_earthsun_distance(self.times_localized) diff --git a/benchmarks/benchmarks/temperature.py b/benchmarks/benchmarks/temperature.py new file mode 100644 index 0000000000..dfefbf0a6c --- /dev/null +++ b/benchmarks/benchmarks/temperature.py @@ -0,0 +1,51 @@ +""" +ASV benchmarks for irradiance.py +""" + +import pandas as pd +import pvlib +from pkg_resources import parse_version +from functools import partial + + +def set_weather_data(obj): + obj.times = pd.date_range(start='20180601', freq='1min', + periods=14400) + obj.poa = pd.Series(1000, index=obj.times) + obj.tamb = pd.Series(20, index=obj.times) + obj.windspeed = pd.Series(2, index=obj.times) + + +class SAPM: + + def setup(self): + set_weather_data(self) + if parse_version(pvlib.__version__) >= parse_version('0.7.0'): + kwargs = pvlib.temperature.TEMPERATURE_MODEL_PARAMETERS['sapm'] + kwargs = kwargs['open_rack_glass_glass'] + self.sapm_cell_wrapper = partial(pvlib.temperature.sapm_cell, + **kwargs) + else: + sapm_celltemp = pvlib.pvsystem.sapm_celltemp + + def sapm_cell_wrapper(poa_global, temp_air, wind_speed): + # just swap order; model params are provided by default + return sapm_celltemp(poa_global, wind_speed, temp_air) + self.sapm_cell_wrapper = sapm_cell_wrapper + + def time_sapm_cell(self): + # use version-appropriate wrapper + self.sapm_cell_wrapper(self.poa, self.tamb, self.windspeed) + + +class Fuentes: + + def setup(self): + if parse_version(pvlib.__version__) < parse_version('0.8.0'): + raise NotImplementedError + + set_weather_data(self) + + def time_fuentes(self): + pvlib.temperature.fuentes(self.poa, self.tamb, self.wind_speed, + noct_installed=45) diff --git a/benchmarks/benchmarks/tracking.py b/benchmarks/benchmarks/tracking.py new file mode 100644 index 0000000000..a0e5a45115 --- /dev/null +++ b/benchmarks/benchmarks/tracking.py @@ -0,0 +1,35 @@ +""" +ASV benchmarks for tracking.py +""" + +import pandas as pd +from pvlib import tracking, solarposition +import numpy as np + + +class SingleAxis: + + def setup(self): + self.times = pd.date_range(start='20180601', freq='1min', + periods=14400) + self.lat = 35.1 + self.lon = -106.6 + self.solar_position = solarposition.get_solarposition(self.times, + self.lat, + self.lon) + self.tracker = tracking.SingleAxisTracker() + + def time_singleaxis(self): + with np.errstate(invalid='ignore'): + tracking.singleaxis(self.solar_position.apparent_zenith, + self.solar_position.azimuth, + axis_tilt=0, + axis_azimuth=0, + max_angle=60, + backtrack=True, + gcr=0.45) + + def time_tracker_singleaxis(self): + with np.errstate(invalid='ignore'): + self.tracker.singleaxis(self.solar_position.apparent_zenith, + self.solar_position.azimuth) diff --git a/docs/sphinx/source/contributing.rst b/docs/sphinx/source/contributing.rst index 1945d53126..247a64f58c 100644 --- a/docs/sphinx/source/contributing.rst +++ b/docs/sphinx/source/contributing.rst @@ -482,6 +482,19 @@ PVSystem method is called through ``ModelChain.run_model``. assert isinstance(mc.dc, (pd.Series, pd.DataFrame)) +Benchmarking +~~~~~~~~~~~~ + +pvlib includes a small number of performance benchmarking tests. These +tests are run using the `airspeed velocity +`_ tool. We do not require new +performance tests for most contributions at this time. Pull request +reviewers will provide further information if a performance test is +necessary. See our `README +`_ +for instructions on running the benchmarks. + + This documentation ~~~~~~~~~~~~~~~~~~ diff --git a/docs/sphinx/source/whatsnew/v0.8.1.rst b/docs/sphinx/source/whatsnew/v0.8.1.rst new file mode 100644 index 0000000000..4bc09643ac --- /dev/null +++ b/docs/sphinx/source/whatsnew/v0.8.1.rst @@ -0,0 +1,37 @@ +.. _whatsnew_0810: + +v0.8.1 (MONTH DAY YEAR) +----------------------- + +Breaking changes +~~~~~~~~~~~~~~~~ + + +Deprecations +~~~~~~~~~~~~ + + +Enhancements +~~~~~~~~~~~~ + + +Bug fixes +~~~~~~~~~ + + +Testing +~~~~~~~ +* Add airspeed velocity performance testing configuration and a few benchmarks. + (:issue:`419`, :pull:`1049`) + +Documentation +~~~~~~~~~~~~~ + + +Requirements +~~~~~~~~~~~~ + + +Contributors +~~~~~~~~~~~~ +* Kevin Anderson (:ghuser:`kanderso-nrel`)