Skip to content

add functionality to load Solcast API data to iotools #1875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 34 commits into from
Dec 20, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
beba7ba
prototype (#1)
lorenzo-solcast Sep 22, 2023
e731ee0
dynamic period
lorenzo-solcast Sep 25, 2023
1f162cf
docstring
lorenzo-solcast Sep 25, 2023
12c1fe3
feedback
lorenzo-solcast Sep 25, 2023
30278c9
linting
lorenzo-solcast Sep 26, 2023
9dfc3d0
Update pvlib/iotools/solcast.py
lorenzo-solcast Oct 11, 2023
85f8f73
Update pvlib/iotools/solcast.py
lorenzo-solcast Oct 11, 2023
90983fe
Update pvlib/iotools/solcast.py
lorenzo-solcast Oct 11, 2023
499f51a
midpoint docstring
lorenzo-solcast Oct 11, 2023
521e742
Merge remote-tracking branch 'origin/main'
lorenzo-solcast Oct 11, 2023
d79a764
flak8 formatting
lorenzo-solcast Oct 19, 2023
7cb28fe
Merge branch 'pvlib:main' into main
lorenzo-solcast Dec 7, 2023
6792ecb
PR 1875 (#2)
lorenzo-solcast Dec 7, 2023
c0fd312
kandersolar feedback (#3)
lorenzo-solcast Dec 13, 2023
be27d85
Review (#4)
lorenzo-solcast Dec 13, 2023
a33b4b2
comment on pandas version
lorenzo-solcast Dec 14, 2023
b41ab76
Merge branch 'main' into main
AdamRJensen Dec 19, 2023
f2baf74
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
25448ce
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
a9dac8f
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
ec21bec
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
abdee5f
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
ba28943
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
f2b7323
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
c9788ad
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
52ab6d7
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
d45ddec
Adams's feedback
lorenzo-solcast Dec 19, 2023
9be7792
Update pvlib/iotools/solcast.py
lorenzo-solcast Dec 19, 2023
6fc890b
Adams's feedback
lorenzo-solcast Dec 19, 2023
dad2be7
Merge remote-tracking branch 'origin/main'
lorenzo-solcast Dec 19, 2023
aa5005d
Last minor changes
AdamRJensen Dec 19, 2023
593d8ac
added test for _get_solcast
lorenzo-solcast Dec 19, 2023
26942dc
feat: add additional test coverage (#5)
hugh-solcast Dec 20, 2023
147a2b1
linting
lorenzo-solcast Dec 19, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/sphinx/source/reference/iotools.rst
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ of sources and file formats relevant to solar energy modeling.
iotools.get_acis_station_data
iotools.get_acis_available_stations
iotools.read_panond
iotools.get_solcast_tmy
iotools.get_solcast_historic
iotools.get_solcast_forecast
iotools.get_solcast_live


A :py:class:`~pvlib.location.Location` object may be created from metadata
Expand Down
4 changes: 4 additions & 0 deletions pvlib/iotools/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,7 @@
from pvlib.iotools.acis import get_acis_mpe # noqa: F401
from pvlib.iotools.acis import get_acis_station_data # noqa: F401
from pvlib.iotools.acis import get_acis_available_stations # noqa: F401
from pvlib.iotools.solcast import get_solcast_forecast # noqa: F401
from pvlib.iotools.solcast import get_solcast_live # noqa: F401
from pvlib.iotools.solcast import get_solcast_historic # noqa: F401
from pvlib.iotools.solcast import get_solcast_tmy # noqa: F401
374 changes: 374 additions & 0 deletions pvlib/iotools/solcast.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,374 @@
""" Functions to access data from the Solcast API.
"""

import requests
import pandas as pd
from dataclasses import dataclass


BASE_URL = "https://api.solcast.com.au/data"

@dataclass
class ParameterMap:
solcast_name: str
pvlib_name: str
conversion: callable=lambda x: x

# define the conventions between Solcast and PVLib nomenclature and units
VARIABLE_MAP = [
ParameterMap("air_temp", "temp_air"), # air_temp -> temp_air (deg C)
ParameterMap("surface_pressure", "pressure", lambda x: x*100), # surface_pressure (hPa) -> pressure (Pa)
ParameterMap("dewpoint_temp", "temp_dew"), # dewpoint_temp -> temp_dew (deg C)
ParameterMap("gti", "poa_global"), # gti (W/m^2) -> poa_global (W/m^2)
ParameterMap("wind_speed_10m", "wind_speed"), # wind_speed_10m (m/s) -> wind_speed (m/s)
ParameterMap("wind_direction_10m", "wind_direction"), # wind_direction_10m (deg) -> wind_direction (deg) (Convention?)
ParameterMap(
"azimuth", "solar_azimuth", lambda x: abs(x) if x <= 0 else 360 - x
), # azimuth -> solar_azimuth (degrees) (different convention)
ParameterMap("precipitable_water", "precipitable_water", lambda x: x*10), # precipitable_water (kg/m2) -> precipitable_water (cm)
ParameterMap("zenith", "solar_zenith") # zenith -> solar_zenith
]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AdamRJensen This is an interesting approach that we might consider adopting in some centralized way for iotools in general.

No action needed in this PR, of course.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is pretty cool



def get_solcast_tmy(
latitude, longitude, api_key, map_variables=True, **kwargs
):
"""Get the irradiance and weather for a Typical Meteorological Year (TMY) at a requested location.

Derived from satellite (clouds and irradiance over non-polar continental areas) and
numerical weather models (other data). The TMY is calculated with data from 2007 to 2023.

Parameters
----------
latitude : float
in decimal degrees, between -90 and 90, north is positive
longitude : float
in decimal degrees, between -180 and 180, east is positive
api_key : str
To access Solcast data you will need an API key: https://toolkit.solcast.com.au/register.
map_variables: bool, default: True
When true, renames columns of the DataFrame to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Time is made the index with the "period mid" convention from Solcast's "period end".
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this only happen when map_variables is True?

I think it would be preferable to always have the same time index convention, so I would move this comment to the output section.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, it is applied only if map_variables is True. The idea being that the raw data is returned if set to False, but we can do that by default if preferred?

kwargs:
Optional parameters passed to the API. See https://docs.solcast.com.au/ for full list of parameters.

Returns
-------
df : pandas.DataFrame
containing the values for the parameters requested

Examples
--------
get_solcast_tmy(
latitude=-33.856784,
longitude=151.215297,
api_key="your-key"
)

you can pass any of the parameters listed in the API docs, like time_zone:

get_solcast_tmy(
latitude=-33.856784,
longitude=151.215297,
time_zone=10,
api_key="your-key"
)

"""

params = dict(
latitude=latitude,
longitude=longitude,
format="json",
**kwargs
)

return _get_solcast(
endpoint="tmy/radiation_and_weather",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be an idea to expose the endpoint as a parameter? For example, in case you have different versions, e.g., tmy/radiation_and_weather/v2/?

This is of course easy to add later, but figured I'd raise the point now for discussion

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that is a valid point, and relevant for our SDK too. There are no use-cases for this functionality yet, but I agree that down the line we may need expose this to the user.

params=params,
api_key=api_key,
map_variables=map_variables
)


def get_solcast_historic(
latitude,
longitude,
start,
api_key,
end=None,
duration=None,
map_variables=True,
**kwargs
):
"""Get historical irradiance and weather estimated actuals
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"""Get historical irradiance and weather estimated actuals
"""Get historical irradiance and weather estimated actuals.

I'm unfamiliar with the term "actuals" but maybe that is standard? As a non-native, I can't really tell what meaning is trying to be conveyed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we refer to "live" and "historical" data as estimated actuals. These would be sometimes be called "measurements" I guess, but we avoid that term as we are really just estimating these values from satellite observations and not direct measurements of weather/power. Safe to assume that anyone using the Solcast API understands this?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm I think that's a very Solcast specific terminology. To be more in line with the other iotools I suggest removing mentions of "actuals" and just calling them "estimates"


for up to 31 days of data at a time for a requested location,
derived from satellite (clouds and irradiance
over non-polar continental areas) and numerical weather models (other data).
Data is available from 2007-01-01T00:00Z up to real time estimated actuals.

Parameters
----------
latitude : float
in decimal degrees, between -90 and 90, north is positive
longitude : float
in decimal degrees, between -180 and 180, east is positive
start : datetime-like
First day of the requested period
end : optional, datetime-like
Last day of the requested period
duration : optional, default is None
Must include one of end_date and duration. ISO_8601 compliant duration for the historic data.
Must be within 31 days of the start_date.
map_variables: bool, default: True
When true, renames columns of the DataFrame to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Time is made the index with the "period mid" convention from Solcast's "period end".
api_key : str
To access Solcast data you will need an API key: https://toolkit.solcast.com.au/register.
kwargs:
Optional parameters passed to the GET request

See https://docs.solcast.com.au/ for full list of parameters.

Returns
-------
df : pandas.DataFrame
containing the values for the parameters requested

Examples
--------
get_solcast_historic(
latitude=-33.856784,
longitude=151.215297,
start='2007-01-01T00:00Z',
duration='P1D',
api_key="your-key"
)

you can pass any of the parameters listed in the API docs, for example using the end parameter instead

get_solcast_historic(
latitude=-33.856784,
longitude=151.215297,
start='2007-01-01T00:00Z',
end='2007-01-02T00:00Z',
api_key="your-key"
)
"""

params = dict(
latitude=latitude,
longitude=longitude,
start=start,
end=end,
duration=duration,
api_key=api_key,
format="json",
**kwargs
)

return _get_solcast(
endpoint="historic/radiation_and_weather",
params=params,
api_key=api_key,
map_variables=map_variables
)

def get_solcast_forecast(
latitude, longitude, api_key, map_variables=True, **kwargs
):
"""Get irradiance and weather forecasts from the present time up to 14 days ahead

Parameters
----------
latitude : float
in decimal degrees, between -90 and 90, north is positive
longitude : float
in decimal degrees, between -180 and 180, east is positive
api_key : str
To access Solcast data you will need an API key: https://toolkit.solcast.com.au/register.
map_variables: bool, default: True
When true, renames columns of the DataFrame to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Time is made the index with the "period mid" convention from Solcast's "period end".
kwargs:
Optional parameters passed to the GET request

See https://docs.solcast.com.au/ for full list of parameters.

Returns
-------
df : pandas.DataFrame
containing the values for the parameters requested

Examples
--------
get_solcast_forecast(
latitude=-33.856784,
longitude=151.215297,
api_key="your-key"
)

you can pass any of the parameters listed in the API docs, like asking for specific variables
get_solcast_forecast(
latitude=-33.856784,
longitude=151.215297,
output_parameters=['dni', 'clearsky_dni', 'snow_soiling_rooftop'],
api_key="your-key"
)
"""

params = dict(
latitude=latitude,
longitude=longitude,
format="json",
**kwargs
)

return _get_solcast(
endpoint="forecast/radiation_and_weather",
params=params,
api_key=api_key,
map_variables=map_variables
)

def get_solcast_live(
latitude, longitude, api_key, map_variables=True, **kwargs
):
"""Get irradiance and weather estimated actuals for near real-time and past 7 days

Parameters
----------
latitude : float
in decimal degrees, between -90 and 90, north is positive
longitude : float
in decimal degrees, between -180 and 180, east is positive
api_key : str
To access Solcast data you will need an API key: https://toolkit.solcast.com.au/register.
map_variables: bool, default: True
When true, renames columns of the DataFrame to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Time is made the index with the "period mid" convention from Solcast's "period end".
kwargs:
Optional parameters passed to the GET request

See https://docs.solcast.com.au/ for full list of parameters.

Returns
-------
df : pandas.DataFrame
containing the values for the parameters requested

Examples
--------
get_solcast_live(
latitude=-33.856784,
longitude=151.215297,
api_key="your-key"
)

you can pass any of the parameters listed in the API docs, like

get_solcast_live(
latitude=-33.856784,
longitude=151.215297,
terrain_shading=True,
output_parameters=['ghi', 'clearsky_ghi', 'snow_soiling_rooftop'],
api_key="your-key"
)

use map_variables=False to avoid converting the data to PVLib's conventions

get_solcast_live(
latitude=-33.856784,
longitude=151.215297,
map_variables=False,
api_key="your-key"
)
"""

params = dict(
latitude=latitude,
longitude=longitude,
format="json",
**kwargs
)

return _get_solcast(
endpoint="live/radiation_and_weather",
params=params,
api_key=api_key,
map_variables=map_variables
)

def solcast2pvlib(df):
"""Formats the data from Solcast to PVLib's conventions.

Parameters
----------
df : pandas.DataFrame
contains the data as returned from the Solcast API

Returns
-------
a pandas.DataFrame with the data cast to PVLib's conventions
"""
# move from period_end to period_middle as per pvlib convention
df["period_mid"] = pd.to_datetime(df.period_end) - pd.to_timedelta(df.period.values) / 2
df = df.set_index("period_mid").drop(columns=["period_end", "period"])

# rename and convert variables
for variable in VARIABLE_MAP:
if variable.solcast_name in df.columns:
df.rename(columns={variable.solcast_name: variable.pvlib_name}, inplace=True)
df[variable.pvlib_name] = df[variable.pvlib_name].apply(variable.conversion)
return df

def _get_solcast(
endpoint,
params,
api_key,
map_variables
):
"""retrieves weather, irradiance and power data from the Solcast API

Parameters
----------
endpoint : str
one of Solcast API endpoint:
- live/radiation_and_weather
- forecast/radiation_and_weather
- historic/radiation_and_weather
- tmy/radiation_and_weather
params : dict
parameters to be passed to the API
api_key : str
To access Solcast data you will need an API key: https://toolkit.solcast.com.au/register.
map_variables: bool, default: True
When true, renames columns of the DataFrame to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Time is made the index with the "period mid" convention from Solcast's "period end".

Returns
-------
A pandas.DataFrame with the data if the request is successful, an error message otherwise
"""

response = requests.get(
url= '/'.join([BASE_URL, endpoint]),
params=params,
headers={"Authorization": f"Bearer {api_key}"}
)

if response.status_code == 200:
j = response.json()
df = pd.DataFrame.from_dict(j[list(j.keys())[0]])
if map_variables:
return solcast2pvlib(df)
else:
return df
else:
raise Exception(response.json())
Loading