From 89034a1d07a5e6797b0d5b868da249171754a910 Mon Sep 17 00:00:00 2001 From: Cooper Date: Tue, 7 Mar 2023 14:12:27 -0700 Subject: [PATCH 01/22] '' --- pvlib/clearsky.py | 68 +++++++++++++++++++++++++++--------- pvlib/tests/test_clearsky.py | 37 ++++++++++++-------- 2 files changed, 74 insertions(+), 31 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index ffc7ac9b55..0a04187902 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -579,9 +579,9 @@ def _calc_stats(data, samples_per_window, sample_interval, H): """ data_mean = data.values[H].mean(axis=0) - data_mean = _to_centered_series(data_mean, data.index, samples_per_window) + data_mean = _to_centered_series(data_mean, data.index, samples_per_window, H) data_max = data.values[H].max(axis=0) - data_max = _to_centered_series(data_max, data.index, samples_per_window) + data_max = _to_centered_series(data_max, data.index, samples_per_window, H) # shift to get forward difference, .diff() is backward difference instead data_diff = data.diff().shift(-1) data_slope = data_diff / sample_interval @@ -594,27 +594,36 @@ def _slope_nstd_windowed(slopes, data, H, samples_per_window, sample_interval): with np.errstate(divide='ignore', invalid='ignore'): nstd = slopes[H[:-1, ]].std(ddof=1, axis=0) \ / data.values[H].mean(axis=0) - return _to_centered_series(nstd, data.index, samples_per_window) + return _to_centered_series(nstd, data.index, samples_per_window, H) def _max_diff_windowed(data, H, samples_per_window): raw = np.diff(data) raw = np.abs(raw[H[:-1, ]]).max(axis=0) - return _to_centered_series(raw, data.index, samples_per_window) + return _to_centered_series(raw, data.index, samples_per_window, H) def _line_length_windowed(data, H, samples_per_window, sample_interval): raw = np.sqrt(np.diff(data)**2. + sample_interval**2.) raw = np.sum(raw[H[:-1, ]], axis=0) - return _to_centered_series(raw, data.index, samples_per_window) + return _to_centered_series(raw, data.index, samples_per_window, H) -def _to_centered_series(vals, idx, samples_per_window): - vals = np.pad(vals, ((0, len(idx) - len(vals)),), mode='constant', - constant_values=np.nan) - shift = samples_per_window // 2 # align = 'center' only - return pd.Series(index=idx, data=vals).shift(shift) +def _to_centered_series(vals, idx, samples_per_window, H): + # Get center of interval using zero-indexing + center_row = samples_per_window//2 - 1 + + # Maintain tz that is stripped when idx is put in H + if idx.tz is not None: + c = pd.DatetimeIndex(idx.values[H][center_row,:],\ + tz = 'UTC').tz_convert(idx.tz) + else: + c = idx.values[H][center_row,:] + + centered = pd.Series(index = idx, dtype = 'object') + centered.loc[c] = vals + return centered def _clear_sample_index(clear_windows, samples_per_window, align, H): @@ -639,6 +648,7 @@ def _clear_sample_index(clear_windows, samples_per_window, align, H): idx = clear_windows.shift(shift) # drop rows at the end corresponding to windows past the end of data idx = idx.drop(clear_windows.index[1 - samples_per_window:]) + idx = idx[idx.isna() == False] idx = idx.astype(bool) # shift changed type to object clear_samples = np.unique(H[:, idx]) return clear_samples @@ -776,6 +786,17 @@ def detect_clearsky(measured, clearsky, times=None, window_length=10, # generate matrix of integers for creating windows with indexing H = hankel(np.arange(samples_per_window), np.arange(samples_per_window-1, len(times))) + + # Put DatetimeIndex in Hankel matrix + time_h = times.values[H] + # Identify maximum time step between consecutive Timestamps for each column + time_h_diff_max = np.max(np.diff(time_h, axis = 0)/\ + np.timedelta64(1, '60s'), axis = 0) + # Identify column indices where max time step > sample_interval + gaps = list(np.ravel(np.argwhere(time_h_diff_max > sample_interval))) + keep_columns = [i for i in range(len(times) - (samples_per_window -1)) if i not in gaps] + # Remove columns with temporal gaps + H = H[:, keep_columns] # calculate measurement statistics meas_mean, meas_max, meas_slope_nstd, meas_slope = _calc_stats( @@ -802,14 +823,26 @@ def detect_clearsky(measured, clearsky, times=None, window_length=10, line_diff = meas_line_length - clear_line_length slope_max_diff = _max_diff_windowed( meas - scaled_clear, H, samples_per_window) + # evaluate comparison criteria - c1 = np.abs(meas_mean - alpha*clear_mean) < mean_diff - c2 = np.abs(meas_max - alpha*clear_max) < max_diff - c3 = (line_diff > lower_line_length) & (line_diff < upper_line_length) - c4 = meas_slope_nstd < var_diff - c5 = slope_max_diff < slope_dev - c6 = (clear_mean != 0) & ~np.isnan(clear_mean) - clear_windows = c1 & c2 & c3 & c4 & c5 & c6 + c1 = np.abs(meas_mean - alpha*clear_mean).apply(lambda x: \ + x < mean_diff if np.isnan(x) == False else None) + c2 = np.abs(meas_max - alpha*clear_max).apply(lambda x: \ + x < max_diff if np.isnan(x) == False else None) + c3 = np.logical_and(np.abs(line_diff).apply(lambda x: lambda x: \ + x > lower_line_length if np.isnan(x) == False else None),\ + np.abs(line_diff).apply(lambda x: lambda x: \ + x < upper_line_length if np.isnan(x) == False else None)) + c4 = meas_slope_nstd.apply(lambda x: x < var_diff \ + if np.isnan(x) == False else None) + c5 = slope_max_diff.apply(lambda x: x < slope_dev \ + if np.isnan(x) == False else None) + c6 = np.logical_and(clear_mean != 0, clear_mean.isna() == False) + + # np.logical_and() maintains NaNs + clear_windows = pd.Series(index = times, + data = np.logical_and.reduce([c1,c2, c3,\ + c4, c5, c6])) # create array to return clear_samples = np.full_like(meas, False, dtype='bool') @@ -817,6 +850,7 @@ def detect_clearsky(measured, clearsky, times=None, window_length=10, idx = _clear_sample_index(clear_windows, samples_per_window, 'center', H) clear_samples[idx] = True + clear_samples[pd.Index(gaps)] = None # find a new alpha previous_alpha = alpha diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 37b95dcdf9..3921053547 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -8,7 +8,7 @@ import pytest from numpy.testing import assert_allclose -from .conftest import assert_frame_equal, assert_series_equal +from pvlib.tests.conftest import assert_frame_equal, assert_series_equal, DATA_DIR from pvlib.location import Location from pvlib import clearsky @@ -16,8 +16,11 @@ from pvlib import atmosphere from pvlib import irradiance -from .conftest import DATA_DIR - +import sys +sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib") +import clearsky +sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib\\tests") +from conftest import DATA_DIR, assert_frame_equal, assert_series_equal def test_ineichen_series(): times = pd.date_range(start='2014-06-24', end='2014-06-25', freq='3h', @@ -532,14 +535,13 @@ def detect_clearsky_data(): cs = loc.get_clearsky(expected.index, linke_turbidity=2.658197) return expected, cs - def test_detect_clearsky(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( expected['GHI'], cs['ghi'], times=cs.index, window_length=10) assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) - +# test_detect_clearsky(detect_clearsky_data()) def test_detect_clearsky_defaults(detect_clearsky_data): expected, cs = detect_clearsky_data @@ -547,7 +549,7 @@ def test_detect_clearsky_defaults(detect_clearsky_data): expected['GHI'], cs['ghi']) assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) - +# test_detect_clearsky_defaults(detect_clearsky_data()) def test_detect_clearsky_components(detect_clearsky_data): expected, cs = detect_clearsky_data @@ -558,7 +560,7 @@ def test_detect_clearsky_components(detect_clearsky_data): check_dtype=False, check_names=False) assert isinstance(components, OrderedDict) assert np.allclose(alpha, 0.9633903181941296) - +# test_detect_clearsky_components(detect_clearsky_data()) def test_detect_clearsky_iterations(detect_clearsky_data): expected, cs = detect_clearsky_data @@ -572,8 +574,9 @@ def test_detect_clearsky_iterations(detect_clearsky_data): expected['GHI'], cs['ghi']*alpha, max_iterations=20) assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) +# test_detect_clearsky_iterations(detect_clearsky_data()) - +# neither expected nor clear_samples are all True def test_detect_clearsky_kwargs(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -582,7 +585,13 @@ def test_detect_clearsky_kwargs(detect_clearsky_data): upper_line_length=1000, var_diff=10, slope_dev=1000) assert clear_samples.all() - +# expected, cs = detect_clearsky_data() +# clear_samples = clearsky.detect_clearsky( +# expected['GHI'], cs['ghi'], times=cs.index, window_length=10, +# mean_diff=1000, max_diff=1000, lower_line_length=-1000, +# upper_line_length=1000, var_diff=10, slope_dev=1000) + +# Mine returns three False values at the end whereas expected has three True values def test_detect_clearsky_window(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -591,8 +600,8 @@ def test_detect_clearsky_window(detect_clearsky_data): expected.iloc[-3:] = True assert_series_equal(expected, clear_samples, check_dtype=False, check_names=False) - - +test_detect_clearsky_window(detect_clearsky_data()) +# Passed def test_detect_clearsky_time_interval(detect_clearsky_data): expected, cs = detect_clearsky_data u = np.arange(0, len(cs), 2) @@ -603,7 +612,7 @@ def test_detect_clearsky_time_interval(detect_clearsky_data): assert_series_equal(expected2['Clear or not'], clear_samples, check_dtype=False, check_names=False) - +# Passed def test_detect_clearsky_arrays(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -612,7 +621,7 @@ def test_detect_clearsky_arrays(detect_clearsky_data): assert isinstance(clear_samples, np.ndarray) assert (clear_samples == expected['Clear or not'].values).all() - +# N/A ? def test_detect_clearsky_irregular_times(detect_clearsky_data): expected, cs = detect_clearsky_data times = cs.index.values.copy() @@ -622,7 +631,7 @@ def test_detect_clearsky_irregular_times(detect_clearsky_data): clearsky.detect_clearsky(expected['GHI'].values, cs['ghi'].values, times, 10) - +# Passed def test_detect_clearsky_missing_index(detect_clearsky_data): expected, cs = detect_clearsky_data with pytest.raises(ValueError): From 403c7987eaa295b5424f3b1d4f876ac44a2dcc4f Mon Sep 17 00:00:00 2001 From: Cooper Date: Tue, 7 Mar 2023 16:00:17 -0700 Subject: [PATCH 02/22] '' --- pvlib/tests/test_clearsky.py | 8 ++++---- pvlib/tools.py | 18 ++++-------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 3921053547..8d1bb4b079 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -16,10 +16,10 @@ from pvlib import atmosphere from pvlib import irradiance -import sys -sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib") -import clearsky -sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib\\tests") +# import sys +# sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib") +# import clearsky +# sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib\\tests") from conftest import DATA_DIR, assert_frame_equal, assert_series_equal def test_ineichen_series(): diff --git a/pvlib/tools.py b/pvlib/tools.py index 229c5dd444..65dab2429e 100644 --- a/pvlib/tools.py +++ b/pvlib/tools.py @@ -386,20 +386,10 @@ def _get_sample_intervals(times, win_length): sky detection functions """ deltas = np.diff(times.values) / np.timedelta64(1, '60s') - - # determine if we can proceed - if times.inferred_freq and len(np.unique(deltas)) == 1: - sample_interval = times[1] - times[0] - sample_interval = sample_interval.seconds / 60 # in minutes - samples_per_window = int(win_length / sample_interval) - return sample_interval, samples_per_window - else: - message = ( - 'algorithm does not yet support unequal time intervals. consider ' - 'resampling your data and checking for gaps from missing ' - 'periods, leap days, etc.' - ) - raise NotImplementedError(message) + vals, counts = np.unique(deltas, return_counts=True) + sample_interval = vals[np.argmax(counts)] + samples_per_window = int(win_length / sample_interval) + return sample_interval, samples_per_window def _degrees_to_index(degrees, coordinate): From a6573109ae7efeae3ec99d60e93cb2a6bc2ada99 Mon Sep 17 00:00:00 2001 From: Cooper Date: Thu, 16 Mar 2023 15:41:20 -0600 Subject: [PATCH 03/22] '' --- pvlib/clearsky.py | 60 +++++++++++++++++++++++------------- pvlib/tests/test_clearsky.py | 25 +++++++++------ 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 0a04187902..ef9b1254f0 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -654,7 +654,7 @@ def _clear_sample_index(clear_windows, samples_per_window, align, H): return clear_samples -def detect_clearsky(measured, clearsky, times=None, window_length=10, +def detect_clearsky(measured, clear_sky, times=None, window_length=10, mean_diff=75, max_diff=75, lower_line_length=-5, upper_line_length=10, var_diff=0.005, slope_dev=8, max_iterations=20, @@ -775,10 +775,10 @@ def detect_clearsky(measured, clearsky, times=None, window_length=10, else: meas = measured - if not isinstance(clearsky, pd.Series): - clear = pd.Series(clearsky, index=times) + if not isinstance(clear_sky, pd.Series): + clear = pd.Series(clear_sky, index=times) else: - clear = clearsky + clear = clear_sky sample_interval, samples_per_window = \ tools._get_sample_intervals(times, window_length) @@ -825,21 +825,35 @@ def detect_clearsky(measured, clearsky, times=None, window_length=10, meas - scaled_clear, H, samples_per_window) # evaluate comparison criteria - c1 = np.abs(meas_mean - alpha*clear_mean).apply(lambda x: \ - x < mean_diff if np.isnan(x) == False else None) - c2 = np.abs(meas_max - alpha*clear_max).apply(lambda x: \ - x < max_diff if np.isnan(x) == False else None) - c3 = np.logical_and(np.abs(line_diff).apply(lambda x: lambda x: \ - x > lower_line_length if np.isnan(x) == False else None),\ - np.abs(line_diff).apply(lambda x: lambda x: \ - x < upper_line_length if np.isnan(x) == False else None)) - c4 = meas_slope_nstd.apply(lambda x: x < var_diff \ - if np.isnan(x) == False else None) - c5 = slope_max_diff.apply(lambda x: x < slope_dev \ - if np.isnan(x) == False else None) - c6 = np.logical_and(clear_mean != 0, clear_mean.isna() == False) - - # np.logical_and() maintains NaNs + # Condition 1 + c1 = np.abs(meas_mean - alpha*clear_mean) + c1_where_nan = c1[c1.isna()].index + c1 = c1 < mean_diff + c1[c1_where_nan] = np.nan + # Condition 2 + c2 = np.abs(meas_max - alpha*clear_max) + c2_where_nan = c2[c2.isna()].index + c2 = c2 < max_diff + c2[c2_where_nan] = np.nan + # Condition 3a & 3b + c3_where_nan = line_diff[line_diff.isna()].index + c3a = line_diff > lower_line_length + c3b = line_diff < upper_line_length + c3 = np.logical_and(c3a, c3b) + c3[c3_where_nan] = np.nan + # Condition 4 + c4_where_nan = meas_slope_nstd[meas_slope_nstd.isna()].index + c4 = meas_slope_nstd < var_diff + c4[c4_where_nan] = np.nan + # Condition 5 + c5_where_nan = slope_max_diff[slope_max_diff.isna()].index + c5 = slope_max_diff < slope_dev + c5[c5_where_nan] = np.nan + # Condition 6 + c6 = clear_mean != 0 + c6[clear_mean[clear_mean.isna()].index] = np.nan + + # np.logical_and() maintains NaNs clear_windows = pd.Series(index = times, data = np.logical_and.reduce([c1,c2, c3,\ c4, c5, c6])) @@ -854,8 +868,10 @@ def detect_clearsky(measured, clearsky, times=None, window_length=10, # find a new alpha previous_alpha = alpha - clear_meas = meas[clear_samples] - clear_clear = clear[clear_samples] + # Must be explicit w/ '==' as clear_windows is a non-boolean array if + # it contains NaNs + clear_meas = meas[clear_windows == True] + clear_clear = clear[clear_windows == True] def rmse(alpha): return np.sqrt(np.mean((clear_meas - alpha*clear_clear)**2)) @@ -870,7 +886,7 @@ def rmse(alpha): # be polite about returning the same type as was input if ispandas: - clear_samples = pd.Series(clear_samples, index=times) + clear_samples = pd.Series(clear_windows, index=times) if return_components: components = OrderedDict() diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 8d1bb4b079..6a4e5114c9 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -535,6 +535,7 @@ def detect_clearsky_data(): cs = loc.get_clearsky(expected.index, linke_turbidity=2.658197) return expected, cs + def test_detect_clearsky(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -543,6 +544,7 @@ def test_detect_clearsky(detect_clearsky_data): check_dtype=False, check_names=False) # test_detect_clearsky(detect_clearsky_data()) + def test_detect_clearsky_defaults(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -551,6 +553,7 @@ def test_detect_clearsky_defaults(detect_clearsky_data): check_dtype=False, check_names=False) # test_detect_clearsky_defaults(detect_clearsky_data()) + def test_detect_clearsky_components(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples, components, alpha = clearsky.detect_clearsky( @@ -562,6 +565,7 @@ def test_detect_clearsky_components(detect_clearsky_data): assert np.allclose(alpha, 0.9633903181941296) # test_detect_clearsky_components(detect_clearsky_data()) + def test_detect_clearsky_iterations(detect_clearsky_data): expected, cs = detect_clearsky_data alpha = 1.0448 @@ -576,6 +580,7 @@ def test_detect_clearsky_iterations(detect_clearsky_data): check_dtype=False, check_names=False) # test_detect_clearsky_iterations(detect_clearsky_data()) + # neither expected nor clear_samples are all True def test_detect_clearsky_kwargs(detect_clearsky_data): expected, cs = detect_clearsky_data @@ -621,15 +626,17 @@ def test_detect_clearsky_arrays(detect_clearsky_data): assert isinstance(clear_samples, np.ndarray) assert (clear_samples == expected['Clear or not'].values).all() -# N/A ? -def test_detect_clearsky_irregular_times(detect_clearsky_data): - expected, cs = detect_clearsky_data - times = cs.index.values.copy() - times[0] += 10**9 - times = pd.DatetimeIndex(times) - with pytest.raises(NotImplementedError): - clearsky.detect_clearsky(expected['GHI'].values, cs['ghi'].values, - times, 10) +def test_detect_clearsky_meas_irregular_times(detect_clearsky_data): + data, cs = detect_clearsky_data + data.drop(index=data.index[15], inplace=True) + out = clearsky.detect_clearsky(data['GHI'], cs['ghi']) + expected = pd.Series(index=data.index, data=np.array([ + np.nan, np.nan, np.nan, np.nan, True, True, True, False, False, False, + np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, + np.nan, True, False, False, False, False, np.nan, np.nan, np.nan, + np.nan, np.nan], dtype=object)) + assert_series_equal(expected, out, check_dtype=False, + check_names=False) # Passed def test_detect_clearsky_missing_index(detect_clearsky_data): From a9fdfcd353cc4f7e260e86bb8442bdddc45c5f3d Mon Sep 17 00:00:00 2001 From: Cooper Date: Wed, 22 Mar 2023 10:14:08 -0600 Subject: [PATCH 04/22] '' --- pvlib/clearsky.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index ef9b1254f0..66203825e4 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -644,11 +644,17 @@ def _clear_sample_index(clear_windows, samples_per_window, align, H): # shift = - (samples_per_window // 2) # else: # shift = 0 - shift = -(samples_per_window // 2) + # Account for the row # on which the interval is centered not actually + # being in row samples_per_window // 2 + # if samples_per_window is even + if samples_per_window % 2 == 0: + shift = -(samples_per_window // 2 - 1) + else: + shift = -(samples_per_window // 2) idx = clear_windows.shift(shift) # drop rows at the end corresponding to windows past the end of data idx = idx.drop(clear_windows.index[1 - samples_per_window:]) - idx = idx[idx.isna() == False] + # idx = idx[idx.isna() == False] idx = idx.astype(bool) # shift changed type to object clear_samples = np.unique(H[:, idx]) return clear_samples From f3db8d3618d0e8a957728659bc2ea1ae8c6632c3 Mon Sep 17 00:00:00 2001 From: Cooper Date: Fri, 24 Mar 2023 14:30:33 -0600 Subject: [PATCH 05/22] 'Supporting_missing_timesteps_in_detect_clearsky()' --- pvlib/clearsky.py | 86 +++++++++++------- pvlib/data/detect_clearsky_data_missing1.csv | 64 ++++++++++++++ pvlib/data/detect_clearsky_data_missing2.csv | 64 ++++++++++++++ pvlib/data/detect_clearsky_data_missing3.csv | 50 +++++++++++ pvlib/tests/test_clearsky.py | 93 ++++++++++---------- pvlib/tests/test_tools.py | 11 ++- 6 files changed, 289 insertions(+), 79 deletions(-) create mode 100644 pvlib/data/detect_clearsky_data_missing1.csv create mode 100644 pvlib/data/detect_clearsky_data_missing2.csv create mode 100644 pvlib/data/detect_clearsky_data_missing3.csv diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 66203825e4..ed777d63ed 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -577,7 +577,7 @@ def _calc_stats(data, samples_per_window, sample_interval, H): sky irradiance in time series of GHI measurements" Renewable Energy, v90, p. 520-531, 2016. """ - + data_mean = data.values[H].mean(axis=0) data_mean = _to_centered_series(data_mean, data.index, samples_per_window, H) data_max = data.values[H].max(axis=0) @@ -611,22 +611,31 @@ def _line_length_windowed(data, H, samples_per_window, def _to_centered_series(vals, idx, samples_per_window, H): - # Get center of interval using zero-indexing - center_row = samples_per_window//2 - 1 - - # Maintain tz that is stripped when idx is put in H - if idx.tz is not None: - c = pd.DatetimeIndex(idx.values[H][center_row,:],\ - tz = 'UTC').tz_convert(idx.tz) + # Get center of interval using zero-indexing, round down to nearest + # index if there are an even number of rows + if samples_per_window % 2 == 0: + center_row = samples_per_window//2 - 1 else: + center_row = samples_per_window//2 + + try: + # Maintain tz that is stripped when idx is put in H + if idx.tz is not None: + c = pd.DatetimeIndex(idx.values[H][center_row,:], + tz = 'UTC').tz_convert(idx.tz) + else: + c = idx.values[H][center_row,:] + # If the index is a range + except AttributeError: c = idx.values[H][center_row,:] + # Assign summary values for each interval to the indices of the center row centered = pd.Series(index = idx, dtype = 'object') centered.loc[c] = vals return centered -def _clear_sample_index(clear_windows, samples_per_window, align, H): +def _clear_sample_index(clear_windows, samples_per_window, gaps, H, align): """ Returns indices of clear samples in clear windows """ @@ -644,19 +653,22 @@ def _clear_sample_index(clear_windows, samples_per_window, align, H): # shift = - (samples_per_window // 2) # else: # shift = 0 + # Account for the row # on which the interval is centered not actually - # being in row samples_per_window // 2 - # if samples_per_window is even + # being in row samples_per_window // 2 if samples_per_window is even if samples_per_window % 2 == 0: shift = -(samples_per_window // 2 - 1) else: shift = -(samples_per_window // 2) - idx = clear_windows.shift(shift) + clear_cols = clear_windows.shift(shift) # drop rows at the end corresponding to windows past the end of data - idx = idx.drop(clear_windows.index[1 - samples_per_window:]) - # idx = idx[idx.isna() == False] - idx = idx.astype(bool) # shift changed type to object - clear_samples = np.unique(H[:, idx]) + clear_cols = clear_cols.drop(clear_windows.index[1 - samples_per_window:]) + clear_cols = clear_cols.astype(bool) # shift changed type to object + # Boolean mask for column indices of intervals with temporal gaps + gap_cols = [True if c not in gaps else False for c in range(0, + len(clear_windows) - (samples_per_window - 1))] + mask = np.logical_and(clear_cols, gap_cols) + clear_samples = np.unique(H[:, mask]) return clear_samples @@ -739,8 +751,6 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, ------ ValueError If measured is not a Series and times is not provided - NotImplementedError - If timestamps are not equally spaced References ---------- @@ -764,6 +774,11 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, * option to return individual test components and clearsky scaling parameter * uses centered windows (Matlab function uses left-aligned windows) + + 2023-03-24 - This algorithm does accept data with skipped or missing + timestamps. The DatetimeIndex (either times or index of measured) + provided still must be regular, i.e. the length of intervals between + points are equal except in the case that data is missing. """ if times is None: @@ -783,6 +798,13 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, if not isinstance(clear_sky, pd.Series): clear = pd.Series(clear_sky, index=times) + # This clause is designed to address cases where measured has missing time + # steps - if this is the case, clear should be set to have the same + # missing time intervals as measured. Not doing this may cause issues with + # arrays of different lengths when evaluating comparison criteria and + # when indexing the Hankel matrix to construct clear_samples + elif len(clear_sky.index) != len(times): + clear = pd.Series(clear_sky, index=times) else: clear = clear_sky @@ -793,16 +815,14 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, H = hankel(np.arange(samples_per_window), np.arange(samples_per_window-1, len(times))) - # Put DatetimeIndex in Hankel matrix + # Identify intervals with missing indices time_h = times.values[H] - # Identify maximum time step between consecutive Timestamps for each column + # Get maximum time step (in minutes) between consecutive Timestamps + # for each column time_h_diff_max = np.max(np.diff(time_h, axis = 0)/\ np.timedelta64(1, '60s'), axis = 0) - # Identify column indices where max time step > sample_interval - gaps = list(np.ravel(np.argwhere(time_h_diff_max > sample_interval))) - keep_columns = [i for i in range(len(times) - (samples_per_window -1)) if i not in gaps] - # Remove columns with temporal gaps - H = H[:, keep_columns] + # Get column indices where max time step > sample_interval + gaps = np.ravel(np.argwhere(time_h_diff_max > sample_interval)) # calculate measurement statistics meas_mean, meas_max, meas_slope_nstd, meas_slope = _calc_stats( @@ -860,17 +880,17 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, c6[clear_mean[clear_mean.isna()].index] = np.nan # np.logical_and() maintains NaNs - clear_windows = pd.Series(index = times, - data = np.logical_and.reduce([c1,c2, c3,\ - c4, c5, c6])) + clear_windows = pd.Series(index = times, + data = np.logical_and.reduce([ + c1,c2, c3,c4,c5, c6])) # create array to return - clear_samples = np.full_like(meas, False, dtype='bool') + # dtype='bool' removed because it typecast NaNs to False values + clear_samples = np.full_like(meas, False) # find the samples contained in any window classified as clear - idx = _clear_sample_index(clear_windows, samples_per_window, 'center', - H) + idx = _clear_sample_index(clear_windows, samples_per_window, gaps, H, + 'center') clear_samples[idx] = True - clear_samples[pd.Index(gaps)] = None # find a new alpha previous_alpha = alpha @@ -892,7 +912,7 @@ def rmse(alpha): # be polite about returning the same type as was input if ispandas: - clear_samples = pd.Series(clear_windows, index=times) + clear_samples = pd.Series(clear_samples, index=times) if return_components: components = OrderedDict() diff --git a/pvlib/data/detect_clearsky_data_missing1.csv b/pvlib/data/detect_clearsky_data_missing1.csv new file mode 100644 index 0000000000..333b3b2a90 --- /dev/null +++ b/pvlib/data/detect_clearsky_data_missing1.csv @@ -0,0 +1,64 @@ +# latitude: 35.04 +# longitude: -106.62 +# elevation: 1619.0 +,CS,GHI,Clear or not +2020-01-01 11:00:00-07:00,536.302172709558,509.4870640740801,1.0 +2020-01-01 11:01:00-07:00,537.591516239233,510.7119404272713,1.0 +2020-01-01 11:02:00-07:00,538.8629773699926,511.91982850149293,1.0 +2020-01-01 11:03:00-07:00,540.1165267379411,513.110700401044,1.0 +2020-01-01 11:04:00-07:00,541.3521354397624,514.2845286677742,1.0 +2020-01-01 11:05:00-07:00,542.5697750193947,515.441286268425,1.0 +2020-01-01 11:06:00-07:00,543.7694182656351,516.5809473523533,1.0 +2020-01-01 11:07:00-07:00,544.9510360161104,517.7034842153048,1.0 +2020-01-01 11:08:00-07:00,546.1146019716558,518.808871873073,1.0 +2020-01-01 11:09:00-07:00,547.2600894760479,519.8970850022455,1.0 +2020-01-01 11:10:00-07:00,548.3874723122931,520.9680986966785,1.0 +2020-01-01 11:11:00-07:00,549.4967247102148,522.021888474704,1.0 +2020-01-01 11:12:00-07:00,550.5878213431366,523.0584302759797,1.0 +2020-01-01 11:13:00-07:00,551.6607373248604,524.0777004586174,1.0 +2020-01-01 11:14:00-07:00,552.7154482165139,525.0796758056882,1.0 +2020-01-01 11:15:00-07:00,553.7519300136697,526.0643335129862,1.0 +2020-01-01 11:16:00-07:00,554.7701591532945,527.0316511956297,1.0 +2020-01-01 11:17:00-07:00,555.7701131753,527.981607516535,1.0 +2020-01-01 11:18:00-07:00,556.7517680497913,528.9141796473017,1.0 +2020-01-01 11:19:00-07:00,557.7151022068979,529.829347096553,1.0 +2020-01-01 11:20:00-07:00,558.6600938303893,530.7270891388698,1.0 +2020-01-01 11:21:00-07:00,559.5867215409612,531.6073854639131,1.0 +2020-01-01 11:22:00-07:00,560.4949643935382,532.4702161738613,1.0 +2020-01-01 11:23:00-07:00,561.3848018765772,533.3155617827483,1.0 +2020-01-01 11:24:00-07:00,562.2562139096694,534.1434032141859,1.0 +2020-01-01 11:25:00-07:00,563.1091808493264,534.9537218068601,1.0 +2020-01-01 11:26:00-07:00,563.9436834785143,535.7464993045886,1.0 +2020-01-01 11:27:00-07:00,564.7597030125947,536.5217178619649,1.0 +2020-01-01 11:28:00-07:00,565.5572210969589,537.2793600421109,1.0 +2020-01-01 11:29:00-07:00,566.336220321224,538.0194093051628,1.0 +2020-01-01 11:31:00-07:00,567.8385900341821,539.446660532473,1.0 +2020-01-01 11:32:00-07:00,568.5619273418005,540.1338309747105,1.0 +2020-01-01 11:33:00-07:00,569.2666778570792,540.8033439642253,1.0 +2020-01-01 11:34:00-07:00,569.9528257924741,541.4551845028503,1.0 +2020-01-01 11:35:00-07:00,570.6203557904192,542.0893380008982,1.0 +2020-01-01 11:36:00-07:00,571.2692529152785,542.7057902695145,1.0 +2020-01-01 11:37:00-07:00,571.8995026579759,543.304527525077,1.0 +2020-01-01 11:38:00-07:00,572.5110909343163,543.8855363876004,1.0 +2020-01-01 11:39:00-07:00,573.1040040834109,544.4488038792404,1.0 +2020-01-01 11:40:00-07:00,573.6782292505169,544.994317787991,1.0 +2020-01-01 11:41:00-07:00,574.2337528522768,545.522065209663,1.0 +2020-01-01 11:42:00-07:00,574.7705628911892,546.0320347466297,1.0 +2020-01-01 11:43:00-07:00,575.2886474013371,546.5242150312703,1.0 +2020-01-01 11:44:00-07:00,575.7879948384899,546.9985950965654,1.0 +2020-01-01 11:45:00-07:00,576.2685940834351,547.4551643792634,1.0 +2020-01-01 11:46:00-07:00,576.7304344361013,547.8939127142962,1.0 +2020-01-01 11:47:00-07:00,577.1735056190306,548.3148303380791,1.0 +2020-01-01 11:48:00-07:00,577.5977977761287,548.7179078873222,1.0 +2020-01-01 11:49:00-07:00,578.0033014716739,549.1031363980902,1.0 +2020-01-01 11:50:00-07:00,578.3900076930416,549.4705073083895,1.0 +2020-01-01 11:51:00-07:00,578.7579080863377,549.8200126820208,1.0 +2020-01-01 11:52:00-07:00,579.1069939850056,550.1516442857553,1.0 +2020-01-01 11:53:00-07:00,579.4372578890877,550.4653949946334,1.0 +2020-01-01 11:54:00-07:00,579.7486924650192,550.7612578417682,1.0 +2020-01-01 11:55:00-07:00,580.0412908007218,551.0392262606856,1.0 +2020-01-01 11:56:00-07:00,580.3150464019636,551.2992940818655,1.0 +2020-01-01 11:57:00-07:00,580.569953194532,551.5414555348053,1.0 +2020-01-01 11:58:00-07:00,580.8060055234938,551.7657052473191,1.0 +2020-01-01 11:59:00-07:00,581.0231981526458,551.9720382450136,1.0 +2020-01-01 12:00:00-07:00,581.2215262659929,552.1604499526933,1.0 diff --git a/pvlib/data/detect_clearsky_data_missing2.csv b/pvlib/data/detect_clearsky_data_missing2.csv new file mode 100644 index 0000000000..88e9367af0 --- /dev/null +++ b/pvlib/data/detect_clearsky_data_missing2.csv @@ -0,0 +1,64 @@ +# latitude: 35.04 +# longitude: -106.62 +# elevation: 1619.0 +,CS,GHI,Clear or not +2020-01-01 11:00:00-07:00,536.302172709558,509.4870640740801,1.0 +2020-01-01 11:01:00-07:00,537.591516239233,510.7119404272713,1.0 +2020-01-01 11:02:00-07:00,538.8629773699926,511.91982850149293,1.0 +2020-01-01 11:03:00-07:00,540.1165267379411,513.110700401044,1.0 +2020-01-01 11:04:00-07:00,541.3521354397624,514.2845286677742,1.0 +2020-01-01 11:05:00-07:00,542.5697750193947,515.441286268425,1.0 +2020-01-01 11:06:00-07:00,543.7694182656351,516.5809473523533,1.0 +2020-01-01 11:07:00-07:00,544.9510360161104,517.7034842153048,1.0 +2020-01-01 11:08:00-07:00,546.1146019716558,518.808871873073,1.0 +2020-01-01 11:09:00-07:00,547.2600894760479,519.8970850022455,1.0 +2020-01-01 11:10:00-07:00,548.3874723122931,520.9680986966785,1.0 +2020-01-01 11:11:00-07:00,549.4967247102148,522.021888474704,1.0 +2020-01-01 11:12:00-07:00,550.5878213431366,523.0584302759797,1.0 +2020-01-01 11:13:00-07:00,551.6607373248604,524.0777004586174,1.0 +2020-01-01 11:14:00-07:00,552.7154482165139,525.0796758056882,1.0 +2020-01-01 11:15:00-07:00,553.7519300136697,526.0643335129862,1.0 +2020-01-01 11:16:00-07:00,554.7701591532945,527.0316511956297,1.0 +2020-01-01 11:17:00-07:00,555.7701131753,527.981607516535,1.0 +2020-01-01 11:18:00-07:00,556.7517680497913,528.9141796473017,1.0 +2020-01-01 11:19:00-07:00,557.7151022068979,529.829347096553,1.0 +2020-01-01 11:20:00-07:00,558.6600938303893,530.7270891388698,1.0 +2020-01-01 11:21:00-07:00,559.5867215409612,531.6073854639131,1.0 +2020-01-01 11:22:00-07:00,560.4949643935382,532.4702161738613,1.0 +2020-01-01 11:23:00-07:00,561.3848018765772,533.3155617827483,1.0 +2020-01-01 11:24:00-07:00,562.2562139096694,534.1434032141859,1.0 +2020-01-01 11:25:00-07:00,563.1091808493264,534.9537218068601,1.0 +2020-01-01 11:26:00-07:00,563.9436834785143,535.7464993045886,1.0 +2020-01-01 11:27:00-07:00,564.7597030125947,536.5217178619649,1.0 +2020-01-01 11:28:00-07:00,565.5572210969589,537.2793600421109,1.0 +2020-01-01 11:29:00-07:00,566.336220321224,538.0194093051628,1.0 +2020-01-01 11:31:00-07:00,567.8385900341821,539.446660532473,0.0 +2020-01-01 11:32:00-07:00,568.5619273418005,540.1338309747105,0.0 +2020-01-01 11:33:00-07:00,569.2666778570792,540.8033439642253,0.0 +2020-01-01 11:34:00-07:00,569.9528257924741,541.4551845028503,0.0 +2020-01-01 11:35:00-07:00,570.6203557904192,300.0,0.0 +2020-01-01 11:36:00-07:00,571.2692529152785,200.0,0.0 +2020-01-01 11:37:00-07:00,571.8995026579759,250.0,0.0 +2020-01-01 11:38:00-07:00,572.5110909343163,310.0,0.0 +2020-01-01 11:39:00-07:00,573.1040040834109,330.0,0.0 +2020-01-01 11:40:00-07:00,573.6782292505169,544.994317787991,1.0 +2020-01-01 11:41:00-07:00,574.2337528522768,545.522065209663,1.0 +2020-01-01 11:42:00-07:00,574.7705628911892,546.0320347466297,1.0 +2020-01-01 11:43:00-07:00,575.2886474013371,546.5242150312703,1.0 +2020-01-01 11:44:00-07:00,575.7879948384899,546.9985950965654,1.0 +2020-01-01 11:45:00-07:00,576.2685940834351,547.4551643792634,1.0 +2020-01-01 11:46:00-07:00,576.7304344361013,547.8939127142962,1.0 +2020-01-01 11:47:00-07:00,577.1735056190306,548.3148303380791,1.0 +2020-01-01 11:48:00-07:00,577.5977977761287,548.7179078873222,1.0 +2020-01-01 11:49:00-07:00,578.0033014716739,549.1031363980902,1.0 +2020-01-01 11:50:00-07:00,578.3900076930416,549.4705073083895,1.0 +2020-01-01 11:51:00-07:00,578.7579080863377,549.8200126820208,1.0 +2020-01-01 11:52:00-07:00,579.1069939850056,550.1516442857553,1.0 +2020-01-01 11:53:00-07:00,579.4372578890877,550.4653949946334,1.0 +2020-01-01 11:54:00-07:00,579.7486924650192,550.7612578417682,1.0 +2020-01-01 11:55:00-07:00,580.0412908007218,551.0392262606856,1.0 +2020-01-01 11:56:00-07:00,580.3150464019636,551.2992940818655,1.0 +2020-01-01 11:57:00-07:00,580.569953194532,551.5414555348053,1.0 +2020-01-01 11:58:00-07:00,580.8060055234938,551.7657052473191,1.0 +2020-01-01 11:59:00-07:00,581.0231981526458,551.9720382450136,1.0 +2020-01-01 12:00:00-07:00,581.2215262659929,552.1604499526933,1.0 diff --git a/pvlib/data/detect_clearsky_data_missing3.csv b/pvlib/data/detect_clearsky_data_missing3.csv new file mode 100644 index 0000000000..7b703cb6b2 --- /dev/null +++ b/pvlib/data/detect_clearsky_data_missing3.csv @@ -0,0 +1,50 @@ +# latitude: 35.04 +# longitude: -106.62 +# elevation: 1619.0 +,CS,GHI,Clear or not +2020-01-01 11:00:00-07:00,536.302172709558,509.4870640740801,1.0 +2020-01-01 11:01:00-07:00,537.591516239233,510.7119404272713,1.0 +2020-01-01 11:02:00-07:00,538.8629773699926,511.91982850149293,1.0 +2020-01-01 11:03:00-07:00,540.1165267379411,513.110700401044,1.0 +2020-01-01 11:04:00-07:00,541.3521354397624,514.2845286677742,1.0 +2020-01-01 11:05:00-07:00,542.5697750193947,515.441286268425,1.0 +2020-01-01 11:06:00-07:00,543.7694182656351,516.5809473523533,1.0 +2020-01-01 11:07:00-07:00,544.9510360161104,517.7034842153048,1.0 +2020-01-01 11:08:00-07:00,546.1146019716558,518.808871873073,1.0 +2020-01-01 11:09:00-07:00,547.2600894760479,519.8970850022455,1.0 +2020-01-01 11:10:00-07:00,548.3874723122931,520.9680986966785,1.0 +2020-01-01 11:11:00-07:00,549.4967247102148,522.021888474704,1.0 +2020-01-01 11:12:00-07:00,550.5878213431366,523.0584302759797,1.0 +2020-01-01 11:13:00-07:00,551.6607373248604,524.0777004586174,1.0 +2020-01-01 11:14:00-07:00,552.7154482165139,525.0796758056882,1.0 +2020-01-01 11:15:00-07:00,553.7519300136697,526.0643335129862,1.0 +2020-01-01 11:16:00-07:00,554.7701591532945,527.0316511956297,1.0 +2020-01-01 11:17:00-07:00,555.7701131753,527.981607516535,1.0 +2020-01-01 11:18:00-07:00,556.7517680497913,528.9141796473017,1.0 +2020-01-01 11:19:00-07:00,557.7151022068979,529.829347096553,1.0 +2020-01-01 11:20:00-07:00,558.6600938303893,530.7270891388698,1.0 +2020-01-01 11:21:00-07:00,559.5867215409612,531.6073854639131,1.0 +2020-01-01 11:22:00-07:00,560.4949643935382,532.4702161738613,1.0 +2020-01-01 11:23:00-07:00,561.3848018765772,533.3155617827483,1.0 +2020-01-01 11:24:00-07:00,562.2562139096694,534.1434032141859,1.0 +2020-01-01 11:25:00-07:00,563.1091808493264,534.9537218068601,1.0 +2020-01-01 11:26:00-07:00,563.9436834785143,535.7464993045886,1.0 +2020-01-01 11:27:00-07:00,564.7597030125947,536.5217178619649,1.0 +2020-01-01 11:28:00-07:00,565.5572210969589,537.2793600421109,1.0 +2020-01-01 11:29:00-07:00,566.336220321224,538.0194093051628,1.0 +2020-01-01 11:45:00-07:00,576.2685940834351,547.4551643792634,1.0 +2020-01-01 11:46:00-07:00,576.7304344361013,547.8939127142962,1.0 +2020-01-01 11:47:00-07:00,577.1735056190306,548.3148303380791,1.0 +2020-01-01 11:48:00-07:00,577.5977977761287,548.7179078873222,1.0 +2020-01-01 11:49:00-07:00,578.0033014716739,549.1031363980902,1.0 +2020-01-01 11:50:00-07:00,578.3900076930416,549.4705073083895,1.0 +2020-01-01 11:51:00-07:00,578.7579080863377,549.8200126820208,1.0 +2020-01-01 11:52:00-07:00,579.1069939850056,550.1516442857553,1.0 +2020-01-01 11:53:00-07:00,579.4372578890877,550.4653949946334,1.0 +2020-01-01 11:54:00-07:00,579.7486924650192,550.7612578417682,1.0 +2020-01-01 11:55:00-07:00,580.0412908007218,551.0392262606856,1.0 +2020-01-01 11:56:00-07:00,580.3150464019636,551.2992940818655,1.0 +2020-01-01 11:57:00-07:00,580.569953194532,551.5414555348053,1.0 +2020-01-01 11:58:00-07:00,580.8060055234938,551.7657052473191,1.0 +2020-01-01 11:59:00-07:00,581.0231981526458,551.9720382450136,1.0 +2020-01-01 12:00:00-07:00,581.2215262659929,552.1604499526933,1.0 diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 6a4e5114c9..d505741ea1 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -16,12 +16,6 @@ from pvlib import atmosphere from pvlib import irradiance -# import sys -# sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib") -# import clearsky -# sys.path.insert(0,"\\Users\\eccoope\\pvlib-python-forked\\pvlib\\tests") -from conftest import DATA_DIR, assert_frame_equal, assert_series_equal - def test_ineichen_series(): times = pd.date_range(start='2014-06-24', end='2014-06-25', freq='3h', tz='America/Phoenix') @@ -542,7 +536,6 @@ def test_detect_clearsky(detect_clearsky_data): expected['GHI'], cs['ghi'], times=cs.index, window_length=10) assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) -# test_detect_clearsky(detect_clearsky_data()) def test_detect_clearsky_defaults(detect_clearsky_data): @@ -551,7 +544,6 @@ def test_detect_clearsky_defaults(detect_clearsky_data): expected['GHI'], cs['ghi']) assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) -# test_detect_clearsky_defaults(detect_clearsky_data()) def test_detect_clearsky_components(detect_clearsky_data): @@ -562,8 +554,8 @@ def test_detect_clearsky_components(detect_clearsky_data): assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) assert isinstance(components, OrderedDict) - assert np.allclose(alpha, 0.9633903181941296) -# test_detect_clearsky_components(detect_clearsky_data()) + assert np.allclose(alpha, 0.9634310110595476) + # assert np.allclose(alpha, 0.9633903181941296) def test_detect_clearsky_iterations(detect_clearsky_data): @@ -578,10 +570,8 @@ def test_detect_clearsky_iterations(detect_clearsky_data): expected['GHI'], cs['ghi']*alpha, max_iterations=20) assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) -# test_detect_clearsky_iterations(detect_clearsky_data()) -# neither expected nor clear_samples are all True def test_detect_clearsky_kwargs(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -590,13 +580,7 @@ def test_detect_clearsky_kwargs(detect_clearsky_data): upper_line_length=1000, var_diff=10, slope_dev=1000) assert clear_samples.all() -# expected, cs = detect_clearsky_data() -# clear_samples = clearsky.detect_clearsky( -# expected['GHI'], cs['ghi'], times=cs.index, window_length=10, -# mean_diff=1000, max_diff=1000, lower_line_length=-1000, -# upper_line_length=1000, var_diff=10, slope_dev=1000) - -# Mine returns three False values at the end whereas expected has three True values + def test_detect_clearsky_window(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -605,8 +589,8 @@ def test_detect_clearsky_window(detect_clearsky_data): expected.iloc[-3:] = True assert_series_equal(expected, clear_samples, check_dtype=False, check_names=False) -test_detect_clearsky_window(detect_clearsky_data()) -# Passed + + def test_detect_clearsky_time_interval(detect_clearsky_data): expected, cs = detect_clearsky_data u = np.arange(0, len(cs), 2) @@ -617,7 +601,7 @@ def test_detect_clearsky_time_interval(detect_clearsky_data): assert_series_equal(expected2['Clear or not'], clear_samples, check_dtype=False, check_names=False) -# Passed + def test_detect_clearsky_arrays(detect_clearsky_data): expected, cs = detect_clearsky_data clear_samples = clearsky.detect_clearsky( @@ -626,23 +610,41 @@ def test_detect_clearsky_arrays(detect_clearsky_data): assert isinstance(clear_samples, np.ndarray) assert (clear_samples == expected['Clear or not'].values).all() -def test_detect_clearsky_meas_irregular_times(detect_clearsky_data): - data, cs = detect_clearsky_data - data.drop(index=data.index[15], inplace=True) - out = clearsky.detect_clearsky(data['GHI'], cs['ghi']) - expected = pd.Series(index=data.index, data=np.array([ - np.nan, np.nan, np.nan, np.nan, True, True, True, False, False, False, - np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, - np.nan, True, False, False, False, False, np.nan, np.nan, np.nan, - np.nan, np.nan], dtype=object)) - assert_series_equal(expected, out, check_dtype=False, - check_names=False) - -# Passed -def test_detect_clearsky_missing_index(detect_clearsky_data): - expected, cs = detect_clearsky_data - with pytest.raises(ValueError): - clearsky.detect_clearsky(expected['GHI'].values, cs['ghi'].values) + +def test_detect_clearsky_missing_index1(): + # Test for an isolated missing index + data_file = DATA_DIR / 'detect_clearsky_data_missing1.csv' + data = pd.read_csv( + data_file, index_col=0, parse_dates=True, comment='#') + meas, cs, expected = data['GHI'], data['CS'], data['Clear or not'] + clear_samples = clearsky.detect_clearsky( + meas, cs) + assert_series_equal(expected, clear_samples, check_dtype=False, + check_names=False) + + +def test_detect_clearsky_missing_index2(): + # Test for a missing index followed by an overcast period + data_file = DATA_DIR / 'detect_clearsky_data_missing2.csv' + data = pd.read_csv( + data_file, index_col=0, parse_dates=True, comment='#') + meas, cs, expected = data['GHI'], data['CS'], data['Clear or not'] + clear_samples = clearsky.detect_clearsky( + meas, cs) + assert_series_equal(expected, clear_samples, check_dtype=False, + check_names=False) + + +def test_detect_clearsky_missing_index3(): + # Test for 15 consecutive missing indices + data_file = DATA_DIR / 'detect_clearsky_data_missing3.csv' + data = pd.read_csv( + data_file, index_col=0, parse_dates=True, comment='#') + meas, cs, expected = data['GHI'], data['CS'], data['Clear or not'] + clear_samples = clearsky.detect_clearsky( + meas, cs) + assert_series_equal(expected, clear_samples, check_dtype=False, + check_names=False) @pytest.fixture @@ -666,7 +668,7 @@ def test__line_length_windowed(detect_clearsky_helper_data): expected['line_length'] = sqt + sqt.shift(-1) result = clearsky._line_length_windowed( x, H, samples_per_window, sample_interval) - assert_series_equal(result, expected['line_length']) + assert_series_equal(result, expected['line_length'], check_dtype=False) def test__max_diff_windowed(detect_clearsky_helper_data): @@ -675,7 +677,7 @@ def test__max_diff_windowed(detect_clearsky_helper_data): expected['max_diff'] = pd.Series( data=[np.nan, 3., 5., 7., 9., 11., np.nan], index=x.index) result = clearsky._max_diff_windowed(x, H, samples_per_window) - assert_series_equal(result, expected['max_diff']) + assert_series_equal(result, expected['max_diff'], check_dtype=False) def test__calc_stats(detect_clearsky_helper_data): @@ -697,10 +699,11 @@ def test__calc_stats(detect_clearsky_helper_data): result = clearsky._calc_stats( x, samples_per_window, sample_interval, H) res_mean, res_max, res_slope_nstd, res_slope = result - assert_series_equal(res_mean, expected['mean']) - assert_series_equal(res_max, expected['max']) - assert_series_equal(res_slope_nstd, expected['slope_nstd']) - assert_series_equal(res_slope, expected['slope']) + assert_series_equal(res_mean, expected['mean'], check_dtype=False) + assert_series_equal(res_max, expected['max'], check_dtype=False) + assert_series_equal(res_slope_nstd, expected['slope_nstd'], + check_dtype=False) + assert_series_equal(res_slope, expected['slope'], check_dtype=False) def test_bird(): diff --git a/pvlib/tests/test_tools.py b/pvlib/tests/test_tools.py index 4d5312088b..2903bb63c9 100644 --- a/pvlib/tests/test_tools.py +++ b/pvlib/tests/test_tools.py @@ -1,7 +1,8 @@ import pytest - from pvlib import tools import numpy as np +import pandas as pd +from pvlib.tests.conftest import DATA_DIR @pytest.mark.parametrize('keys, input_dict, expected', [ @@ -89,6 +90,14 @@ def test__golden_sect_DataFrame_nans(): _obj_test_golden_sect) assert np.allclose(x, expected, atol=1e-8, equal_nan=True) +def test_get_sample_intervals(): + data_file = DATA_DIR / 'detect_clearsky_data_missing3.csv' + data = pd.read_csv( + data_file, index_col=0, parse_dates=True, comment='#') + sample_interval, samples_per_window = tools._get_sample_intervals( + data.index, 10) + assert np.allclose(sample_interval, 1) + assert np.allclose(samples_per_window, 10) def test_degrees_to_index_1(): """Test that _degrees_to_index raises an error when something other than From 3e2ea1fce11d7c83f07f65324016b96dcfab0e74 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Fri, 24 Mar 2023 14:46:24 -0600 Subject: [PATCH 06/22] stickler changes --- pvlib/clearsky.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index ed777d63ed..2000a04630 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -577,9 +577,10 @@ def _calc_stats(data, samples_per_window, sample_interval, H): sky irradiance in time series of GHI measurements" Renewable Energy, v90, p. 520-531, 2016. """ - + data_mean = data.values[H].mean(axis=0) - data_mean = _to_centered_series(data_mean, data.index, samples_per_window, H) + data_mean = _to_centered_series(data_mean, data.index, samples_per_window, + H) data_max = data.values[H].max(axis=0) data_max = _to_centered_series(data_max, data.index, samples_per_window, H) # shift to get forward difference, .diff() is backward difference instead @@ -617,20 +618,20 @@ def _to_centered_series(vals, idx, samples_per_window, H): center_row = samples_per_window//2 - 1 else: center_row = samples_per_window//2 - + try: # Maintain tz that is stripped when idx is put in H if idx.tz is not None: - c = pd.DatetimeIndex(idx.values[H][center_row,:], - tz = 'UTC').tz_convert(idx.tz) + c = pd.DatetimeIndex(idx.values[H][center_row, :], + tz = 'UTC').tz_convert(idx.tz) else: - c = idx.values[H][center_row,:] + c = idx.values[H][center_row, :] # If the index is a range except AttributeError: - c = idx.values[H][center_row,:] + c = idx.values[H][center_row, :] # Assign summary values for each interval to the indices of the center row - centered = pd.Series(index = idx, dtype = 'object') + centered = pd.Series(index=idx, dtype='object') centered.loc[c] = vals return centered @@ -778,7 +779,7 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, 2023-03-24 - This algorithm does accept data with skipped or missing timestamps. The DatetimeIndex (either times or index of measured) provided still must be regular, i.e. the length of intervals between - points are equal except in the case that data is missing. + points are equal except in the case that data is missing. """ if times is None: @@ -814,13 +815,13 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, # generate matrix of integers for creating windows with indexing H = hankel(np.arange(samples_per_window), np.arange(samples_per_window-1, len(times))) - + # Identify intervals with missing indices time_h = times.values[H] # Get maximum time step (in minutes) between consecutive Timestamps # for each column - time_h_diff_max = np.max(np.diff(time_h, axis = 0)/\ - np.timedelta64(1, '60s'), axis = 0) + time_h_diff_max = np.max(np.diff(time_h, axis=0)/ + np.timedelta64(1, '60s'), axis = 0) # Get column indices where max time step > sample_interval gaps = np.ravel(np.argwhere(time_h_diff_max > sample_interval)) @@ -879,10 +880,10 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, c6 = clear_mean != 0 c6[clear_mean[clear_mean.isna()].index] = np.nan - # np.logical_and() maintains NaNs - clear_windows = pd.Series(index = times, - data = np.logical_and.reduce([ - c1,c2, c3,c4,c5, c6])) + # np.logical_and() maintains NaNs + clear_windows = pd.Series(index=times, + data=np.logical_and.reduce([ + c1, c2, c3, c4,c5, c6])) # create array to return # dtype='bool' removed because it typecast NaNs to False values @@ -894,10 +895,8 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, # find a new alpha previous_alpha = alpha - # Must be explicit w/ '==' as clear_windows is a non-boolean array if - # it contains NaNs - clear_meas = meas[clear_windows == True] - clear_clear = clear[clear_windows == True] + clear_meas = meas[idx] + clear_clear = clear[idx] def rmse(alpha): return np.sqrt(np.mean((clear_meas - alpha*clear_clear)**2)) From 664bfae5dc6b1bf4f292d732ebe86cd7998a2a64 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Fri, 24 Mar 2023 14:52:42 -0600 Subject: [PATCH 07/22] stickler --- pvlib/tests/test_clearsky.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index d505741ea1..f999a06047 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -8,7 +8,7 @@ import pytest from numpy.testing import assert_allclose -from pvlib.tests.conftest import assert_frame_equal, assert_series_equal, DATA_DIR +from .conftest import assert_frame_equal, assert_series_equal, DATA_DIR from pvlib.location import Location from pvlib import clearsky @@ -620,7 +620,7 @@ def test_detect_clearsky_missing_index1(): clear_samples = clearsky.detect_clearsky( meas, cs) assert_series_equal(expected, clear_samples, check_dtype=False, - check_names=False) + check_names=False) def test_detect_clearsky_missing_index2(): @@ -632,7 +632,7 @@ def test_detect_clearsky_missing_index2(): clear_samples = clearsky.detect_clearsky( meas, cs) assert_series_equal(expected, clear_samples, check_dtype=False, - check_names=False) + check_names=False) def test_detect_clearsky_missing_index3(): From 2acfd40df3e1b5539ab8a9891a8ccc801a47ed23 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Fri, 24 Mar 2023 14:53:52 -0600 Subject: [PATCH 08/22] stickler changes --- pvlib/tests/test_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pvlib/tests/test_tools.py b/pvlib/tests/test_tools.py index 2903bb63c9..e1e77b9a98 100644 --- a/pvlib/tests/test_tools.py +++ b/pvlib/tests/test_tools.py @@ -90,12 +90,13 @@ def test__golden_sect_DataFrame_nans(): _obj_test_golden_sect) assert np.allclose(x, expected, atol=1e-8, equal_nan=True) + def test_get_sample_intervals(): data_file = DATA_DIR / 'detect_clearsky_data_missing3.csv' data = pd.read_csv( data_file, index_col=0, parse_dates=True, comment='#') sample_interval, samples_per_window = tools._get_sample_intervals( - data.index, 10) + data.index, 10) assert np.allclose(sample_interval, 1) assert np.allclose(samples_per_window, 10) From d29c7de2dd4bd866d691750dd2d9a0d71eaa708a Mon Sep 17 00:00:00 2001 From: Cooper Date: Fri, 24 Mar 2023 15:06:04 -0600 Subject: [PATCH 09/22] 'Fixed_unit_test' --- pvlib/tests/test_clearsky.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index f999a06047..5009b67e03 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -554,8 +554,7 @@ def test_detect_clearsky_components(detect_clearsky_data): assert_series_equal(expected['Clear or not'], clear_samples, check_dtype=False, check_names=False) assert isinstance(components, OrderedDict) - assert np.allclose(alpha, 0.9634310110595476) - # assert np.allclose(alpha, 0.9633903181941296) + assert np.allclose(alpha, 0.9633903181941296) def test_detect_clearsky_iterations(detect_clearsky_data): From 97e53d061c79601d884221adfca3db04a25d6c1e Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Wed, 29 Mar 2023 13:19:04 -0600 Subject: [PATCH 10/22] Apply suggestions from code review from @cwhanse Co-authored-by: Cliff Hansen --- pvlib/clearsky.py | 11 +++++------ pvlib/tests/test_tools.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 2000a04630..f209986719 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -623,7 +623,7 @@ def _to_centered_series(vals, idx, samples_per_window, H): # Maintain tz that is stripped when idx is put in H if idx.tz is not None: c = pd.DatetimeIndex(idx.values[H][center_row, :], - tz = 'UTC').tz_convert(idx.tz) + tz='UTC').tz_convert(idx.tz) else: c = idx.values[H][center_row, :] # If the index is a range @@ -820,8 +820,8 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, time_h = times.values[H] # Get maximum time step (in minutes) between consecutive Timestamps # for each column - time_h_diff_max = np.max(np.diff(time_h, axis=0)/ - np.timedelta64(1, '60s'), axis = 0) + time_h_diff_max = np.max(np.diff(time_h, axis=0) / + np.timedelta64(1, '60s'), axis=0) # Get column indices where max time step > sample_interval gaps = np.ravel(np.argwhere(time_h_diff_max > sample_interval)) @@ -881,9 +881,8 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, c6[clear_mean[clear_mean.isna()].index] = np.nan # np.logical_and() maintains NaNs - clear_windows = pd.Series(index=times, - data=np.logical_and.reduce([ - c1, c2, c3, c4,c5, c6])) + clear_windows = pd.Series( + index=times, data=np.logical_and.reduce([c1, c2, c3, c4, c5, c6])) # create array to return # dtype='bool' removed because it typecast NaNs to False values diff --git a/pvlib/tests/test_tools.py b/pvlib/tests/test_tools.py index e1e77b9a98..6048030874 100644 --- a/pvlib/tests/test_tools.py +++ b/pvlib/tests/test_tools.py @@ -96,7 +96,7 @@ def test_get_sample_intervals(): data = pd.read_csv( data_file, index_col=0, parse_dates=True, comment='#') sample_interval, samples_per_window = tools._get_sample_intervals( - data.index, 10) + data.index, 10) assert np.allclose(sample_interval, 1) assert np.allclose(samples_per_window, 10) From e8e5faa2cfc90e63b0171f313c13b48577ee3d66 Mon Sep 17 00:00:00 2001 From: Cooper Date: Mon, 17 Apr 2023 12:29:10 -0600 Subject: [PATCH 11/22] 'modified_to_preserve_nans' --- pvlib/clearsky.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 2000a04630..87458303ce 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -824,6 +824,8 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, np.timedelta64(1, '60s'), axis = 0) # Get column indices where max time step > sample_interval gaps = np.ravel(np.argwhere(time_h_diff_max > sample_interval)) + # Get column indices where at least one of the values is a NaN + gaps = set().union(*[gaps, np.ravel(np.argwhere(np.isnan(meas.values[H].mean(axis=0))))]) # calculate measurement statistics meas_mean, meas_max, meas_slope_nstd, meas_slope = _calc_stats( @@ -856,34 +858,33 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, c1 = np.abs(meas_mean - alpha*clear_mean) c1_where_nan = c1[c1.isna()].index c1 = c1 < mean_diff - c1[c1_where_nan] = np.nan # Condition 2 c2 = np.abs(meas_max - alpha*clear_max) c2_where_nan = c2[c2.isna()].index c2 = c2 < max_diff - c2[c2_where_nan] = np.nan # Condition 3a & 3b c3_where_nan = line_diff[line_diff.isna()].index c3a = line_diff > lower_line_length c3b = line_diff < upper_line_length c3 = np.logical_and(c3a, c3b) - c3[c3_where_nan] = np.nan # Condition 4 c4_where_nan = meas_slope_nstd[meas_slope_nstd.isna()].index c4 = meas_slope_nstd < var_diff - c4[c4_where_nan] = np.nan # Condition 5 c5_where_nan = slope_max_diff[slope_max_diff.isna()].index c5 = slope_max_diff < slope_dev - c5[c5_where_nan] = np.nan # Condition 6 c6 = clear_mean != 0 - c6[clear_mean[clear_mean.isna()].index] = np.nan + c6_where_nan = clear_mean[clear_mean.isna()].index # np.logical_and() maintains NaNs clear_windows = pd.Series(index=times, data=np.logical_and.reduce([ c1, c2, c3, c4,c5, c6])) + windows_where_nan = pd.DatetimeIndex(set().union(*[ + c1_where_nan,c2_where_nan, c3_where_nan, + c4_where_nan, c5_where_nan, c6_where_nan])) + clear_windows[windows_where_nan] = np.nan # create array to return # dtype='bool' removed because it typecast NaNs to False values @@ -893,6 +894,10 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, 'center') clear_samples[idx] = True + # Assign NaN to datapoints that were originally NaNs + where_nan = np.argwhere(np.isnan(meas.values)) + clear_samples[where_nan] = np.nan + # find a new alpha previous_alpha = alpha clear_meas = meas[idx] From 92cb84b3e6d3b133f3b5126a6029a5a01c900653 Mon Sep 17 00:00:00 2001 From: Cooper Date: Tue, 18 Apr 2023 13:37:08 -0600 Subject: [PATCH 12/22] 'added_tests_for_data_with_nans_for_detect_clearsk'y --- pvlib/tests/test_clearsky.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 5009b67e03..c5fb04bedc 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -8,10 +8,11 @@ import pytest from numpy.testing import assert_allclose -from .conftest import assert_frame_equal, assert_series_equal, DATA_DIR +from tests.conftest import assert_frame_equal, assert_series_equal, DATA_DIR from pvlib.location import Location -from pvlib import clearsky +# from pvlib import clearsky +import clearsky from pvlib import solarposition from pvlib import atmosphere from pvlib import irradiance @@ -645,6 +646,27 @@ def test_detect_clearsky_missing_index3(): assert_series_equal(expected, clear_samples, check_dtype=False, check_names=False) +def test_detect_clearsky_nans1(): + # Test for 1 NaN value - should mark as NaN + data_file = DATA_DIR / 'detect_clearsky_data_nans1.csv' + data = pd.read_csv( + data_file, index_col=0, parse_dates=True, comment='#') + meas, cs, expected = data['GHI'], data['CS'], data['Clear or not'] + clear_samples = clearsky.detect_clearsky( + meas, cs) + assert_series_equal(expected, clear_samples, check_dtype=False, + check_names=False) + +def test_detect_clearsky_nans2(): + # Test for 1 NaN value - should mark as NaN + data_file = DATA_DIR / 'detect_clearsky_data_nans2.csv' + data = pd.read_csv( + data_file, index_col=0, parse_dates=True, comment='#') + meas, cs, expected = data['GHI'], data['CS'], data['Clear or not'] + clear_samples = clearsky.detect_clearsky( + meas, cs) + assert_series_equal(expected, clear_samples, check_dtype=False, + check_names=False) @pytest.fixture def detect_clearsky_helper_data(): From bd33436eca66d9d932ab46463a7802cf767aa938 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Tue, 18 Apr 2023 13:40:57 -0600 Subject: [PATCH 13/22] fixed relative import statements --- pvlib/tests/test_clearsky.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index c5fb04bedc..176a1c3cf0 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -8,11 +8,10 @@ import pytest from numpy.testing import assert_allclose -from tests.conftest import assert_frame_equal, assert_series_equal, DATA_DIR +from .conftest import assert_frame_equal, assert_series_equal, DATA_DIR from pvlib.location import Location -# from pvlib import clearsky -import clearsky +from pvlib import clearsky from pvlib import solarposition from pvlib import atmosphere from pvlib import irradiance From 710c056bfdc03a8ab40c151e30cdc7e37f330bd1 Mon Sep 17 00:00:00 2001 From: Cooper Date: Thu, 20 Apr 2023 16:02:04 -0600 Subject: [PATCH 14/22] updated whatsnew file and included data for test_clearsky.py --- docs/sphinx/source/whatsnew/v0.9.6.rst | 5 +- pvlib/data/detect_clearsky_data_nans1.csv | 65 +++++++++++++++++++++++ pvlib/data/detect_clearsky_data_nans2.csv | 65 +++++++++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 pvlib/data/detect_clearsky_data_nans1.csv create mode 100644 pvlib/data/detect_clearsky_data_nans2.csv diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst index f21fe80ac6..4e585ed0b4 100644 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ b/docs/sphinx/source/whatsnew/v0.9.6.rst @@ -11,7 +11,9 @@ Deprecations Enhancements ~~~~~~~~~~~~ - +* Added compatibility with missing timesteps to + :py:func`pvlib.clearsky.detect_clearsky`. (:issue:`1678`, + :pull:`1708`) Bug fixes ~~~~~~~~~ @@ -37,3 +39,4 @@ Requirements Contributors ~~~~~~~~~~~~ * Adam R. Jensen (:ghuser:`adamrjensen`) +* Emma C. Cooper (:ghuser:`eccoope`) diff --git a/pvlib/data/detect_clearsky_data_nans1.csv b/pvlib/data/detect_clearsky_data_nans1.csv new file mode 100644 index 0000000000..d4aabff89e --- /dev/null +++ b/pvlib/data/detect_clearsky_data_nans1.csv @@ -0,0 +1,65 @@ +# latitude: 35.04 +# longitude: -106.62 +# elevation: 1619.0 +,CS,GHI,Clear or not +2020-01-01 11:00:00-07:00,536.302172709558,509.4870640740801,1.0 +2020-01-01 11:01:00-07:00,537.591516239233,510.7119404272713,1.0 +2020-01-01 11:02:00-07:00,538.8629773699926,511.91982850149293,1.0 +2020-01-01 11:03:00-07:00,540.1165267379411,513.110700401044,1.0 +2020-01-01 11:04:00-07:00,541.3521354397624,514.2845286677742,1.0 +2020-01-01 11:05:00-07:00,542.5697750193947,515.441286268425,1.0 +2020-01-01 11:06:00-07:00,543.7694182656351,516.5809473523533,1.0 +2020-01-01 11:07:00-07:00,544.9510360161104,517.7034842153048,1.0 +2020-01-01 11:08:00-07:00,546.1146019716558,518.808871873073,1.0 +2020-01-01 11:09:00-07:00,547.2600894760479,519.8970850022455,1.0 +2020-01-01 11:10:00-07:00,548.3874723122931,520.9680986966785,1.0 +2020-01-01 11:11:00-07:00,549.4967247102148,522.021888474704,1.0 +2020-01-01 11:12:00-07:00,550.5878213431366,523.0584302759797,1.0 +2020-01-01 11:13:00-07:00,551.6607373248604,524.0777004586174,1.0 +2020-01-01 11:14:00-07:00,552.7154482165139,525.0796758056882,1.0 +2020-01-01 11:15:00-07:00,553.7519300136697,526.0643335129862,1.0 +2020-01-01 11:16:00-07:00,554.7701591532945,527.0316511956297,1.0 +2020-01-01 11:17:00-07:00,555.7701131753,527.981607516535,1.0 +2020-01-01 11:18:00-07:00,556.7517680497913,528.9141796473017,1.0 +2020-01-01 11:19:00-07:00,557.7151022068979,529.829347096553,1.0 +2020-01-01 11:20:00-07:00,558.6600938303893,530.7270891388698,1.0 +2020-01-01 11:21:00-07:00,559.5867215409612,531.6073854639131,1.0 +2020-01-01 11:22:00-07:00,560.4949643935382,532.4702161738613,1.0 +2020-01-01 11:23:00-07:00,561.3848018765772,533.3155617827483,1.0 +2020-01-01 11:24:00-07:00,562.2562139096694,534.1434032141859,1.0 +2020-01-01 11:25:00-07:00,563.1091808493264,534.9537218068601,1.0 +2020-01-01 11:26:00-07:00,563.9436834785143,535.7464993045886,1.0 +2020-01-01 11:27:00-07:00,564.7597030125947,536.5217178619649,1.0 +2020-01-01 11:28:00-07:00,565.5572210969589,537.2793600421109,1.0 +2020-01-01 11:29:00-07:00,566.336220321224,538.0194093051628,1.0 +2020-01-01 11:30:00-07:00,567.0966821472473,, +2020-01-01 11:31:00-07:00,567.8385900341821,539.446660532473,1.0 +2020-01-01 11:32:00-07:00,568.5619273418005,540.1338309747105,1.0 +2020-01-01 11:33:00-07:00,569.2666778570792,540.8033439642253,1.0 +2020-01-01 11:34:00-07:00,569.9528257924741,541.4551845028503,1.0 +2020-01-01 11:35:00-07:00,570.6203557904192,542.0893380008982,1.0 +2020-01-01 11:36:00-07:00,571.2692529152785,542.7057902695145,1.0 +2020-01-01 11:37:00-07:00,571.8995026579759,543.304527525077,1.0 +2020-01-01 11:38:00-07:00,572.5110909343163,543.8855363876004,1.0 +2020-01-01 11:39:00-07:00,573.1040040834109,544.4488038792404,1.0 +2020-01-01 11:40:00-07:00,573.6782292505169,544.994317787991,1.0 +2020-01-01 11:41:00-07:00,574.2337528522768,545.522065209663,1.0 +2020-01-01 11:42:00-07:00,574.7705628911892,546.0320347466297,1.0 +2020-01-01 11:43:00-07:00,575.2886474013371,546.5242150312703,1.0 +2020-01-01 11:44:00-07:00,575.7879948384899,546.9985950965654,1.0 +2020-01-01 11:45:00-07:00,576.2685940834351,547.4551643792634,1.0 +2020-01-01 11:46:00-07:00,576.7304344361013,547.8939127142962,1.0 +2020-01-01 11:47:00-07:00,577.1735056190306,548.3148303380791,1.0 +2020-01-01 11:48:00-07:00,577.5977977761287,548.7179078873222,1.0 +2020-01-01 11:49:00-07:00,578.0033014716739,549.1031363980902,1.0 +2020-01-01 11:50:00-07:00,578.3900076930416,549.4705073083895,1.0 +2020-01-01 11:51:00-07:00,578.7579080863377,549.8200126820208,1.0 +2020-01-01 11:52:00-07:00,579.1069939850056,550.1516442857553,1.0 +2020-01-01 11:53:00-07:00,579.4372578890877,550.4653949946334,1.0 +2020-01-01 11:54:00-07:00,579.7486924650192,550.7612578417682,1.0 +2020-01-01 11:55:00-07:00,580.0412908007218,551.0392262606856,1.0 +2020-01-01 11:56:00-07:00,580.3150464019636,551.2992940818655,1.0 +2020-01-01 11:57:00-07:00,580.569953194532,551.5414555348053,1.0 +2020-01-01 11:58:00-07:00,580.8060055234938,551.7657052473191,1.0 +2020-01-01 11:59:00-07:00,581.0231981526458,551.9720382450136,1.0 +2020-01-01 12:00:00-07:00,581.2215262659929,552.1604499526933,1.0 diff --git a/pvlib/data/detect_clearsky_data_nans2.csv b/pvlib/data/detect_clearsky_data_nans2.csv new file mode 100644 index 0000000000..5f91388b73 --- /dev/null +++ b/pvlib/data/detect_clearsky_data_nans2.csv @@ -0,0 +1,65 @@ +# latitude: 35.04 +# longitude: -106.62 +# elevation: 1619.0 +,CS,GHI,Clear or not +2020-01-01 11:00:00-07:00,536.302172709558,509.4870640740801,1.0 +2020-01-01 11:01:00-07:00,537.591516239233,510.7119404272713,1.0 +2020-01-01 11:02:00-07:00,538.8629773699926,511.91982850149293,1.0 +2020-01-01 11:03:00-07:00,540.1165267379411,513.110700401044,1.0 +2020-01-01 11:04:00-07:00,541.3521354397624,514.2845286677742,1.0 +2020-01-01 11:05:00-07:00,542.5697750193947,515.441286268425,1.0 +2020-01-01 11:06:00-07:00,543.7694182656351,516.5809473523533,1.0 +2020-01-01 11:07:00-07:00,544.9510360161104,517.7034842153048,1.0 +2020-01-01 11:08:00-07:00,546.1146019716558,518.808871873073,1.0 +2020-01-01 11:09:00-07:00,547.2600894760479,519.8970850022455,1.0 +2020-01-01 11:10:00-07:00,548.3874723122931,520.9680986966785,1.0 +2020-01-01 11:11:00-07:00,549.4967247102148,522.021888474704,1.0 +2020-01-01 11:12:00-07:00,550.5878213431366,523.0584302759797,1.0 +2020-01-01 11:13:00-07:00,551.6607373248604,524.0777004586174,1.0 +2020-01-01 11:14:00-07:00,552.7154482165139,525.0796758056882,1.0 +2020-01-01 11:15:00-07:00,553.7519300136697,526.0643335129862,1.0 +2020-01-01 11:16:00-07:00,554.7701591532945,527.0316511956297,1.0 +2020-01-01 11:17:00-07:00,555.7701131753,527.981607516535,1.0 +2020-01-01 11:18:00-07:00,556.7517680497913,528.9141796473017,1.0 +2020-01-01 11:19:00-07:00,557.7151022068979,529.829347096553,1.0 +2020-01-01 11:20:00-07:00,558.6600938303893,530.7270891388698,1.0 +2020-01-01 11:21:00-07:00,559.5867215409612,531.6073854639131,1.0 +2020-01-01 11:22:00-07:00,560.4949643935382,532.4702161738613,1.0 +2020-01-01 11:23:00-07:00,561.3848018765772,533.3155617827483,1.0 +2020-01-01 11:24:00-07:00,562.2562139096694,534.1434032141859,1.0 +2020-01-01 11:25:00-07:00,563.1091808493264,534.9537218068601,1.0 +2020-01-01 11:26:00-07:00,563.9436834785143,535.7464993045886,1.0 +2020-01-01 11:27:00-07:00,564.7597030125947,536.5217178619649,1.0 +2020-01-01 11:28:00-07:00,565.5572210969589,, +2020-01-01 11:29:00-07:00,566.336220321224,, +2020-01-01 11:30:00-07:00,567.0966821472473,538.7418480398849,0.0 +2020-01-01 11:31:00-07:00,567.8385900341821,539.446660532473,0.0 +2020-01-01 11:32:00-07:00,568.5619273418005,540.1338309747105,0.0 +2020-01-01 11:33:00-07:00,569.2666778570792,540.8033439642253,0.0 +2020-01-01 11:34:00-07:00,569.9528257924741,300.0,0.0 +2020-01-01 11:35:00-07:00,570.6203557904192,200.0,0.0 +2020-01-01 11:36:00-07:00,571.2692529152785,250.0,0.0 +2020-01-01 11:37:00-07:00,571.8995026579759,310.0,0.0 +2020-01-01 11:38:00-07:00,572.5110909343163,330.0,0.0 +2020-01-01 11:39:00-07:00,573.1040040834109,544.4488038792404,1.0 +2020-01-01 11:40:00-07:00,573.6782292505169,544.994317787991,1.0 +2020-01-01 11:41:00-07:00,574.2337528522768,545.522065209663,1.0 +2020-01-01 11:42:00-07:00,574.7705628911892,546.0320347466297,1.0 +2020-01-01 11:43:00-07:00,575.2886474013371,546.5242150312703,1.0 +2020-01-01 11:44:00-07:00,575.7879948384899,546.9985950965654,1.0 +2020-01-01 11:45:00-07:00,576.2685940834351,547.4551643792634,1.0 +2020-01-01 11:46:00-07:00,576.7304344361013,547.8939127142962,1.0 +2020-01-01 11:47:00-07:00,577.1735056190306,548.3148303380791,1.0 +2020-01-01 11:48:00-07:00,577.5977977761287,548.7179078873222,1.0 +2020-01-01 11:49:00-07:00,578.0033014716739,549.1031363980902,1.0 +2020-01-01 11:50:00-07:00,578.3900076930416,549.4705073083895,1.0 +2020-01-01 11:51:00-07:00,578.7579080863377,549.8200126820208,1.0 +2020-01-01 11:52:00-07:00,579.1069939850056,550.1516442857553,1.0 +2020-01-01 11:53:00-07:00,579.4372578890877,550.4653949946334,1.0 +2020-01-01 11:54:00-07:00,579.7486924650192,550.7612578417682,1.0 +2020-01-01 11:55:00-07:00,580.0412908007218,551.0392262606856,1.0 +2020-01-01 11:56:00-07:00,580.3150464019636,551.2992940818655,1.0 +2020-01-01 11:57:00-07:00,580.569953194532,551.5414555348053,1.0 +2020-01-01 11:58:00-07:00,580.8060055234938,551.7657052473191,1.0 +2020-01-01 11:59:00-07:00,581.0231981526458,551.9720382450136,1.0 +2020-01-01 12:00:00-07:00,581.2215262659929,552.1604499526933,1.0 From 868e45b42b88a4889f8a02e46483dd492f770638 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Thu, 27 Apr 2023 14:47:14 -0600 Subject: [PATCH 15/22] Stickler corrections --- pvlib/clearsky.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 0c42f2fe4c..785c477a8b 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -825,7 +825,9 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, # Get column indices where max time step > sample_interval gaps = np.ravel(np.argwhere(time_h_diff_max > sample_interval)) # Get column indices where at least one of the values is a NaN - gaps = set().union(*[gaps, np.ravel(np.argwhere(np.isnan(meas.values[H].mean(axis=0))))]) + gaps = set().union(*[ + gaps, np.ravel(np.argwhere(np.isnan(meas\ + .values[H].mean(axis=0))))]) # calculate measurement statistics meas_mean, meas_max, meas_slope_nstd, meas_slope = _calc_stats( @@ -881,8 +883,8 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, clear_windows = pd.Series( index=times, data=np.logical_and.reduce([c1, c2, c3, c4, c5, c6])) windows_where_nan = pd.DatetimeIndex(set().union(*[ - c1_where_nan,c2_where_nan, c3_where_nan, - c4_where_nan, c5_where_nan, c6_where_nan])) + c1_where_nan,c2_where_nan, c3_where_nan, c4_where_nan, c5_where_nan, + c6_where_nan])) clear_windows[windows_where_nan] = np.nan # create array to return From f5e1fbcedd9fa7073e00241ffb1741bf3792cfa1 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Thu, 27 Apr 2023 14:48:34 -0600 Subject: [PATCH 16/22] Stickler --- pvlib/tests/test_clearsky.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 176a1c3cf0..819754f7d2 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -654,8 +654,9 @@ def test_detect_clearsky_nans1(): clear_samples = clearsky.detect_clearsky( meas, cs) assert_series_equal(expected, clear_samples, check_dtype=False, - check_names=False) + check_names=False) + def test_detect_clearsky_nans2(): # Test for 1 NaN value - should mark as NaN data_file = DATA_DIR / 'detect_clearsky_data_nans2.csv' @@ -665,7 +666,7 @@ def test_detect_clearsky_nans2(): clear_samples = clearsky.detect_clearsky( meas, cs) assert_series_equal(expected, clear_samples, check_dtype=False, - check_names=False) + check_names=False) @pytest.fixture def detect_clearsky_helper_data(): From 9385702fe9b66706d97b7782354ec4e5e75de0d0 Mon Sep 17 00:00:00 2001 From: Cooper Date: Wed, 21 Jun 2023 11:25:33 -0600 Subject: [PATCH 17/22] Added tests for if/else clause in detect_clearsky() --- pvlib/clearsky.py | 5 +++-- pvlib/tests/test_clearsky.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 785c477a8b..896697f9c3 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -13,7 +13,8 @@ from scipy.linalg import hankel import h5py -from pvlib import atmosphere, tools +# from pvlib +import atmosphere, tools from pvlib.tools import _degrees_to_index @@ -700,7 +701,7 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, ---------- measured : array or Series Time series of measured GHI. [W/m2] - clearsky : array or Series + clear_sky : array or Series Time series of the expected clearsky GHI. [W/m2] times : DatetimeIndex or None, default None. Times of measured and clearsky values. If None the index of measured diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 819754f7d2..19abba43d1 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -667,6 +667,32 @@ def test_detect_clearsky_nans2(): meas, cs) assert_series_equal(expected, clear_samples, check_dtype=False, check_names=False) + +def test_detect_clearsky_diff_index_lengths(detect_clearsky_data): + ''' + Intended to test the following if/else clauses + + if not isinstance(clear_sky, pd.Series): + clear = pd.Series(clear_sky, index=times) + # This clause is designed to address cases where measured has missing time + # steps - if this is the case, clear should be set to have the same + # missing time intervals as measured. Not doing this may cause issues with + # arrays of different lengths when evaluating comparison criteria and + # when indexing the Hankel matrix to construct clear_samples + elif len(clear_sky.index) != len(times): + clear = pd.Series(clear_sky, index=times) + else: + clear = clear_sky + ''' + expected, cs = detect_clearsky_data + expected.drop(index=expected.index[10], inplace=True) + clear_samples = clearsky.detect_clearsky( + expected['GHI'], cs['ghi'], times=expected.index, + window_length=10) + new_expected = np.array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., + 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., + 1., 1., 1., 0., 0., 0., 0.]) + assert (clear_samples.values == new_expected).all() @pytest.fixture def detect_clearsky_helper_data(): From 9d5d0cb6b9c664832f6eefb7a12cf2eaa58dadfa Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Wed, 21 Jun 2023 11:30:01 -0600 Subject: [PATCH 18/22] Update clearsky.py Changed clear_sky back to clearsky --- pvlib/clearsky.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 896697f9c3..773dfff6b4 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -13,8 +13,7 @@ from scipy.linalg import hankel import h5py -# from pvlib -import atmosphere, tools +from pvlib import atmosphere, tools from pvlib.tools import _degrees_to_index @@ -674,7 +673,7 @@ def _clear_sample_index(clear_windows, samples_per_window, gaps, H, align): return clear_samples -def detect_clearsky(measured, clear_sky, times=None, window_length=10, +def detect_clearsky(measured, clearsky, times=None, window_length=10, mean_diff=75, max_diff=75, lower_line_length=-5, upper_line_length=10, var_diff=0.005, slope_dev=8, max_iterations=20, @@ -701,7 +700,7 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, ---------- measured : array or Series Time series of measured GHI. [W/m2] - clear_sky : array or Series + clearsky : array or Series Time series of the expected clearsky GHI. [W/m2] times : DatetimeIndex or None, default None. Times of measured and clearsky values. If None the index of measured @@ -798,17 +797,17 @@ def detect_clearsky(measured, clear_sky, times=None, window_length=10, else: meas = measured - if not isinstance(clear_sky, pd.Series): - clear = pd.Series(clear_sky, index=times) + if not isinstance(clearsky, pd.Series): + clear = pd.Series(clearsky, index=times) # This clause is designed to address cases where measured has missing time # steps - if this is the case, clear should be set to have the same # missing time intervals as measured. Not doing this may cause issues with # arrays of different lengths when evaluating comparison criteria and # when indexing the Hankel matrix to construct clear_samples - elif len(clear_sky.index) != len(times): - clear = pd.Series(clear_sky, index=times) + elif len(clearsky.index) != len(times): + clear = pd.Series(clearsky, index=times) else: - clear = clear_sky + clear = clearsky sample_interval, samples_per_window = \ tools._get_sample_intervals(times, window_length) From cc36315fe33867cb3be953766491d0a278e448e2 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Mon, 10 Jul 2023 14:19:01 -0600 Subject: [PATCH 19/22] Delete v0.9.6.rst --- docs/sphinx/source/whatsnew/v0.9.6.rst | 106 ------------------------- 1 file changed, 106 deletions(-) delete mode 100644 docs/sphinx/source/whatsnew/v0.9.6.rst diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst deleted file mode 100644 index 897c456b79..0000000000 --- a/docs/sphinx/source/whatsnew/v0.9.6.rst +++ /dev/null @@ -1,106 +0,0 @@ -.. _whatsnew_0960: - - -v0.9.6 (Anticipated June 2023) ------------------------------- - - -Breaking Changes -~~~~~~~~~~~~~~~~ -* Modified the ``surface_azimuth`` parameter in :py:func:`pvlib.iotools.get_pvgis_hourly` to conform to the - pvlib azimuth convention (counterclockwise from north). Previously 0 degrees represented south. - (:issue:`1724`, :pull:`1739`) -* For consistency with the rest of pvlib, the ``pw`` parameters are renamed to - ``precipitable_water`` in :py:func:`pvlib.spectrum.spectral_factor_firstsolar`. - (:pull:`1768`) -* For consistency with the rest of pvlib, the ``tilt`` parameter is renamed - to ``surface_tilt`` in :py:func:`pvlib.soiling.hsu`. (:issue:`1717`, :pull:`1738`) - -Deprecations -~~~~~~~~~~~~ -* Functions for calculating spectral modifiers have been moved to :py:mod:`pvlib.spectrum`: - :py:func:`pvlib.atmosphere.first_solar_spectral_correction` is deprecated and - replaced by :py:func:`~pvlib.spectrum.spectral_factor_firstsolar`, and - :py:func:`pvlib.pvsystem.sapm_spectral_loss` is deprecated and replaced by - :py:func:`~pvlib.spectrum.spectral_factor_sapm`. (:pull:`1628`) -* Removed the ``get_ecmwf_macc`` and ``read_ecmwf_macc`` iotools functions as the - MACC dataset has been `removed by ECMWF `_ - (data period 2003-2012). Instead, ECMWF recommends to use CAMS global - reanalysis (EAC4) from the Atmosphere Data Store (ADS). See also :py:func:`pvlib.iotools.get_cams`. - (:issue:`1691`, :pull:`1654`) - -* The ``recolumn`` parameter in :py:func:`pvlib.iotools.read_tmy3`, which maps - TMY3 column names to nonstandard alternatives, is now deprecated. - We encourage using ``map_variables`` (which produces standard pvlib names) instead. - (:issue:`1517`, :pull:`1623`) - -Enhancements -~~~~~~~~~~~~ -* Added a new irradiance decomposition model :py:func:`pvlib.irradiance.louche`. (:pull:`1705`) -* Add optional encoding parameter to :py:func:`pvlib.iotools.read_tmy3`. - (:issue:`1732`, :pull:`1737`) -* Added function to retrieve horizon data from PVGIS - :py:func:`pvlib.iotools.get_pvgis_horizon`. (:issue:`1290`, :pull:`1395`) -* Added ``map_variables`` argument to the :py:func:`pvlib.iotools.read_tmy3` in - order to offer the option of mapping column names to standard pvlib names. - (:issue:`1517`, :pull:`1623`) -* Update the URL used in the :py:func:`pvlib.iotools.get_cams` function. The new URL supports load-balancing - and redirects to the fastest server. (:issue:`1688`, :pull:`1740`) -* :py:func:`pvlib.iotools.get_psm3` now has a ``url`` parameter to give the user - the option of controlling what NSRDB endpoint is used. (:pull:`1736`) -* :py:func:`pvlib.iotools.get_psm3` now uses the new NSRDB 3.2.2 endpoint for - hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) -* The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) -* Added compatibility with missing timesteps to - :py:func`pvlib.clearsky.detect_clearsky`. (:issue:`1678`, - :pull:`1708`) - -Bug fixes -~~~~~~~~~ -* `data` can no longer be left unspecified in - :py:meth:`pvlib.modelchain.ModelChain.run_model_from_effective_irradiance`. (:issue:`1713`, :pull:`1720`) -* ``d2mutau`` and ``NsVbi`` were hardcoded in :py:func:`pvlib.pvsystem.max_power_point` instead of - passing through the arguments to the function. (:pull:`1733`) -* :py:func:`pvlib.iam.physical` no longer returns NaN when ``n=1`` and ``aoi>90``. - This bug was introduced in v0.9.5. (:issue:`1706`, :pull:`1707`) - - -Testing -~~~~~~~ -* Migrated to mamba-org/setup-micromamba. (:issue:`1746`, :pull:`1758`) - -Documentation -~~~~~~~~~~~~~ -* Updated the description of the interval parameter in - :py:func:`pvlib.iotools.get_psm3`. (:issue:`1702`, :pull:`1712`) -* Fixed outdated nbviewer links. (:issue:`1721`, :pull:`1726`) - -Benchmarking -~~~~~~~~~~~~~ - - -Requirements -~~~~~~~~~~~~ - - -Contributors -~~~~~~~~~~~~ -* Lakshya Garg (:ghuser:`Lakshyadevelops`) -* Adam R. Jensen (:ghuser:`adamrjensen`) -* Ben Pierce (:ghuser:`bgpierc`) -* Joseph Palakapilly (:ghuser:`JPalakapillyKWH`) -* Cliff Hansen (:ghuser:`cwhanse`) -* Anton Driesse (:ghuser:`adriesse`) -* Will Holmgren (:ghuser:`wholmgren`) -* Mark Mikofski (:ghuser:`mikofski`) -* Karel De Brabandere (:ghuser:`kdebrab`) -* Josh Stein (:ghuser:`jsstein`) -* Kevin Anderson (:ghuser:`kandersolar`) -* Siddharth Kaul (:ghuser:`k10blogger`) -* Kshitiz Gupta (:ghuser:`kshitiz305`) -* Stefan de Lange (:ghuser:`langestefan`) -* Andy Lam (:ghuser:`@andylam598`) -* :ghuser:`ooprathamm` -* Kevin Anderson (:ghuser:`kandersolar`) -* Devon Watt (:ghuser:`d-watt`) -* Emma C. Cooper (:ghuser:`eccoope`) From 3c29c99f2985a1030a0a92aadbb0052f35ce20ac Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Mon, 24 Jul 2023 08:48:29 -0600 Subject: [PATCH 20/22] Update tools.py Import DATA_DIR --- pvlib/tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pvlib/tools.py b/pvlib/tools.py index 9d2775703b..7e1208fd7d 100644 --- a/pvlib/tools.py +++ b/pvlib/tools.py @@ -7,6 +7,7 @@ import pandas as pd import pytz import warnings +from .conftest import DATA_DIR def cosd(angle): From 0021e1d7670552c6db6e9657f9856194be1786f5 Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:10:15 -0600 Subject: [PATCH 21/22] Update tools.py Move import statement to test_tools.py pt1 --- pvlib/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pvlib/tools.py b/pvlib/tools.py index 7e1208fd7d..9d2775703b 100644 --- a/pvlib/tools.py +++ b/pvlib/tools.py @@ -7,7 +7,6 @@ import pandas as pd import pytz import warnings -from .conftest import DATA_DIR def cosd(angle): From ecf2bbf0ebfec5a16a45555c261696982d4c902b Mon Sep 17 00:00:00 2001 From: eccoope <125493409+eccoope@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:10:48 -0600 Subject: [PATCH 22/22] Update test_tools.py Move import statement to test_tools.py pt2 --- pvlib/tests/test_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/test_tools.py b/pvlib/tests/test_tools.py index 3e44a3626f..f695cea61c 100644 --- a/pvlib/tests/test_tools.py +++ b/pvlib/tests/test_tools.py @@ -2,7 +2,7 @@ from pvlib import tools import numpy as np import pandas as pd - +from .conftest import DATA_DIR @pytest.mark.parametrize('keys, input_dict, expected', [ (['a', 'b'], {'a': 1, 'b': 2, 'c': 3}, {'a': 1, 'b': 2}),