diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 0b9a8e20e4..1ea0844d71 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -294,9 +294,37 @@ Functions tracking.singleaxis +.. _iotools: + +IO Tools +======== + +Functions for reading and writing data from a variety of file formats +relevant to solar energy modeling. + +.. autosummary:: + :toctree: generated/ + + iotools.read_tmy2 + iotools.read_tmy3 + +A :py:class:`~pvlib.location.Location` object may be created from metadata +in some files. + +.. autosummary:: + :toctree: generated/ + + location.Location.from_tmy + + TMY === +.. warning:: + + The :py:mod:`pvlib.tmy` module is deprecated; it will be removed + in pvlib 0.7. Please see the :ref:`pvlib.iotools ` package. + Methods and functions for reading data from TMY files. .. autosummary:: diff --git a/docs/sphinx/source/clearsky.rst b/docs/sphinx/source/clearsky.rst index 300481a287..c47212f42b 100644 --- a/docs/sphinx/source/clearsky.rst +++ b/docs/sphinx/source/clearsky.rst @@ -40,10 +40,12 @@ We'll need these imports for the examples below. In [1]: import pvlib - In [1]: from pvlib import clearsky, atmosphere, tmy, solarposition + In [1]: from pvlib import clearsky, atmosphere, solarposition In [1]: from pvlib.location import Location + In [1]: from pvlib.iotools import read_tmy3 + .. _location: @@ -214,7 +216,7 @@ wavelengths [Bir80]_, and is implemented in In [1]: tmy_file = os.path.join(pvlib_data, '703165TY.csv') # TMY file - In [1]: tmy_data, tmy_header = tmy.readtmy3(tmy_file, coerce_year=1999) # read TMY data + In [1]: tmy_data, tmy_header = read_tmy3(tmy_file, coerce_year=1999) # read TMY data In [1]: tl_historic = clearsky.lookup_linke_turbidity(time=tmy_data.index, ...: latitude=tmy_header['latitude'], longitude=tmy_header['longitude']) diff --git a/docs/sphinx/source/whatsnew/v0.6.1.rst b/docs/sphinx/source/whatsnew/v0.6.1.rst index 37f8ee312a..f8e76e242d 100644 --- a/docs/sphinx/source/whatsnew/v0.6.1.rst +++ b/docs/sphinx/source/whatsnew/v0.6.1.rst @@ -12,6 +12,10 @@ date will require Python 3. (:issue:`501`) API Changes ~~~~~~~~~~~ +* Created the ``pvlib.iotools`` subpackage. (:issue:`29`, :issue:`261`) +* Deprecated ``tmy``, ``tmy.readtmy2`` and ``tmy.readtmy3``; + they will be removed in v0.7. Use the new :py:func:`pvlib.iotools.read_tmy2` + and :py:func:`pvlib.iotools.read_tmy3` instead. (:issue:`261`) Enhancements diff --git a/pvlib/__init__.py b/pvlib/__init__.py index 0606737ee5..238d481615 100644 --- a/pvlib/__init__.py +++ b/pvlib/__init__.py @@ -6,9 +6,12 @@ from pvlib import irradiance from pvlib import location from pvlib import solarposition -from pvlib import tmy +from pvlib import iotools from pvlib import tracking from pvlib import pvsystem from pvlib import spa from pvlib import modelchain from pvlib import singlediode + +# for backwards compatibility for pvlib.tmy module +from pvlib import tmy diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py new file mode 100644 index 0000000000..e68be0fe1b --- /dev/null +++ b/pvlib/iotools/__init__.py @@ -0,0 +1,2 @@ +from pvlib.iotools.tmy import read_tmy2 # noqa: F401 +from pvlib.iotools.tmy import read_tmy3 # noqa: F401 diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py new file mode 100644 index 0000000000..9c5ba9e9ca --- /dev/null +++ b/pvlib/iotools/tmy.py @@ -0,0 +1,521 @@ +""" +Import functions for TMY2 and TMY3 data files. +""" + +import datetime +import io +import re + +try: + # python 2 compatibility + from urllib2 import urlopen, Request +except ImportError: + from urllib.request import urlopen, Request + +import dateutil +import pandas as pd + + +def read_tmy3(filename=None, coerce_year=None, recolumn=True): + ''' + Read a TMY3 file in to a pandas dataframe. + + Note that values contained in the metadata dictionary are unchanged + from the TMY3 file (i.e. units are retained). In the case of any + discrepencies between this documentation and the TMY3 User's Manual + [1], the TMY3 User's Manual takes precedence. + + The TMY3 files were updated in Jan. 2015. This function requires the + use of the updated files. + + Parameters + ---------- + filename : None or string, default None + If None, attempts to use a Tkinter file browser. A string can be + a relative file path, absolute file path, or url. + + coerce_year : None or int, default None + If supplied, the year of the data will be set to this value. + + recolumn : bool, default True + If True, apply standard names to TMY3 columns. Typically this + results in stripping the units from the column name. + + Returns + ------- + Tuple of the form (data, metadata). + + data : DataFrame + A pandas dataframe with the columns described in the table + below. For more detailed descriptions of each component, please + consult the TMY3 User's Manual ([1]), especially tables 1-1 + through 1-6. + + metadata : dict + The site metadata available in the file. + + Notes + ----- + + The returned structures have the following fields. + + =============== ====== =================== + key format description + =============== ====== =================== + altitude Float site elevation + latitude Float site latitudeitude + longitude Float site longitudeitude + Name String site name + State String state + TZ Float UTC offset + USAF Int USAF identifier + =============== ====== =================== + + ============================= ====================================================================================================================================================== + TMYData field description + ============================= ====================================================================================================================================================== + TMYData.Index A pandas datetime index. NOTE, the index is currently timezone unaware, and times are set to local standard time (daylight savings is not included) + TMYData.ETR Extraterrestrial horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + TMYData.ETRN Extraterrestrial normal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + TMYData.GHI Direct and diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + TMYData.GHISource See [1], Table 1-4 + TMYData.GHIUncertainty Uncertainty based on random and bias error estimates see [2] + TMYData.DNI Amount of direct normal radiation (modeled) recv'd during 60 mintues prior to timestamp, Wh/m^2 + TMYData.DNISource See [1], Table 1-4 + TMYData.DNIUncertainty Uncertainty based on random and bias error estimates see [2] + TMYData.DHI Amount of diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + TMYData.DHISource See [1], Table 1-4 + TMYData.DHIUncertainty Uncertainty based on random and bias error estimates see [2] + TMYData.GHillum Avg. total horizontal illuminance recv'd during the 60 minutes prior to timestamp, lx + TMYData.GHillumSource See [1], Table 1-4 + TMYData.GHillumUncertainty Uncertainty based on random and bias error estimates see [2] + TMYData.DNillum Avg. direct normal illuminance recv'd during the 60 minutes prior to timestamp, lx + TMYData.DNillumSource See [1], Table 1-4 + TMYData.DNillumUncertainty Uncertainty based on random and bias error estimates see [2] + TMYData.DHillum Avg. horizontal diffuse illuminance recv'd during the 60 minutes prior to timestamp, lx + TMYData.DHillumSource See [1], Table 1-4 + TMYData.DHillumUncertainty Uncertainty based on random and bias error estimates see [2] + TMYData.Zenithlum Avg. luminance at the sky's zenith during the 60 minutes prior to timestamp, cd/m^2 + TMYData.ZenithlumSource See [1], Table 1-4 + TMYData.ZenithlumUncertainty Uncertainty based on random and bias error estimates see [1] section 2.10 + TMYData.TotCld Amount of sky dome covered by clouds or obscuring phenonema at time stamp, tenths of sky + TMYData.TotCldSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.TotCldUnertainty See [1], Table 1-6 + TMYData.OpqCld Amount of sky dome covered by clouds or obscuring phenonema that prevent observing the sky at time stamp, tenths of sky + TMYData.OpqCldSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.OpqCldUncertainty See [1], Table 1-6 + TMYData.DryBulb Dry bulb temperature at the time indicated, deg C + TMYData.DryBulbSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.DryBulbUncertainty See [1], Table 1-6 + TMYData.DewPoint Dew-point temperature at the time indicated, deg C + TMYData.DewPointSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.DewPointUncertainty See [1], Table 1-6 + TMYData.RHum Relatitudeive humidity at the time indicated, percent + TMYData.RHumSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.RHumUncertainty See [1], Table 1-6 + TMYData.Pressure Station pressure at the time indicated, 1 mbar + TMYData.PressureSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.PressureUncertainty See [1], Table 1-6 + TMYData.Wdir Wind direction at time indicated, degrees from north (360 = north; 0 = undefined,calm) + TMYData.WdirSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.WdirUncertainty See [1], Table 1-6 + TMYData.Wspd Wind speed at the time indicated, meter/second + TMYData.WspdSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.WspdUncertainty See [1], Table 1-6 + TMYData.Hvis Distance to discernable remote objects at time indicated (7777=unlimited), meter + TMYData.HvisSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.HvisUncertainty See [1], Table 1-6 + TMYData.CeilHgt Height of cloud base above local terrain (7777=unlimited), meter + TMYData.CeilHgtSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.CeilHgtUncertainty See [1], Table 1-6 + TMYData.Pwat Total precipitable water contained in a column of unit cross section from earth to top of atmosphere, cm + TMYData.PwatSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.PwatUncertainty See [1], Table 1-6 + TMYData.AOD The broadband aerosol optical depth per unit of air mass due to extinction by aerosol component of atmosphere, unitless + TMYData.AODSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.AODUncertainty See [1], Table 1-6 + TMYData.Alb The ratio of reflected solar irradiance to global horizontal irradiance, unitless + TMYData.AlbSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.AlbUncertainty See [1], Table 1-6 + TMYData.Lprecipdepth The amount of liquid precipitation observed at indicated time for the period indicated in the liquid precipitation quantity field, millimeter + TMYData.Lprecipquantity The period of accumulatitudeion for the liquid precipitation depth field, hour + TMYData.LprecipSource See [1], Table 1-5, 8760x1 cell array of strings + TMYData.LprecipUncertainty See [1], Table 1-6 + TMYData.PresWth Present weather code, see [2]. + TMYData.PresWthSource Present weather code source, see [2]. + TMYData.PresWthUncertainty Present weather code uncertainty, see [2]. + ============================= ====================================================================================================================================================== + + References + ---------- + + [1] Wilcox, S and Marion, W. "Users Manual for TMY3 Data Sets". + NREL/TP-581-43156, Revised May 2008. + + [2] Wilcox, S. (2007). National Solar Radiation Database 1991 2005 + Update: Users Manual. 472 pp.; NREL Report No. TP-581-41364. + ''' + + if filename is None: + try: + filename = _interactive_load() + except ImportError: + raise ImportError('Interactive load failed. Tkinter not supported ' + 'on this system. Try installing X-Quartz and ' + 'reloading') + + head = ['USAF', 'Name', 'State', 'TZ', 'latitude', 'longitude', 'altitude'] + + if filename.startswith('http'): + request = Request(filename, headers={'User-Agent': ( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 ' + 'Safari/537.36')}) + response = urlopen(request) + csvdata = io.StringIO(response.read().decode(errors='ignore')) + else: + # assume it's accessible via the file system + csvdata = open(filename, 'r') + + # read in file metadata, advance buffer to second line + firstline = csvdata.readline() + if 'Request Rejected' in firstline: + raise IOError('Remote server rejected TMY file request') + + meta = dict(zip(head, firstline.rstrip('\n').split(","))) + + # convert metadata strings to numeric types + meta['altitude'] = float(meta['altitude']) + meta['latitude'] = float(meta['latitude']) + meta['longitude'] = float(meta['longitude']) + meta['TZ'] = float(meta['TZ']) + meta['USAF'] = int(meta['USAF']) + + # use pandas to read the csv file/stringio buffer + # header is actually the second line in file, but tell pandas to look for + # header information on the 1st line (0 indexing) because we've already + # advanced past the true first line with the readline call above. + data = pd.read_csv( + csvdata, header=0, + parse_dates={'datetime': ['Date (MM/DD/YYYY)', 'Time (HH:MM)']}, + date_parser=lambda *x: _parsedate(*x, year=coerce_year), + index_col='datetime') + + if recolumn: + data = _recolumn(data) # rename to standard column names + + data = data.tz_localize(int(meta['TZ'] * 3600)) + + return data, meta + + +def _interactive_load(): + import Tkinter + from tkFileDialog import askopenfilename + Tkinter.Tk().withdraw() # Start interactive file input + return askopenfilename() + + +def _parsedate(ymd, hour, year=None): + # stupidly complicated due to TMY3's usage of hour 24 + # and dateutil's inability to handle that. + offset_hour = int(hour[:2]) - 1 + offset_datetime = '{} {}:00'.format(ymd, offset_hour) + offset_date = dateutil.parser.parse(offset_datetime) + true_date = offset_date + dateutil.relativedelta.relativedelta(hours=1) + if year is not None: + true_date = true_date.replace(year=year) + return true_date + + +def _recolumn(tmy3_dataframe): + """ + Rename the columns of the TMY3 DataFrame. + + Parameters + ---------- + tmy3_dataframe : DataFrame + inplace : bool + passed to DataFrame.rename() + + Returns + ------- + Recolumned DataFrame. + """ + # paste in the header as one long line + raw_columns = 'ETR (W/m^2),ETRN (W/m^2),GHI (W/m^2),GHI source,GHI uncert (%),DNI (W/m^2),DNI source,DNI uncert (%),DHI (W/m^2),DHI source,DHI uncert (%),GH illum (lx),GH illum source,Global illum uncert (%),DN illum (lx),DN illum source,DN illum uncert (%),DH illum (lx),DH illum source,DH illum uncert (%),Zenith lum (cd/m^2),Zenith lum source,Zenith lum uncert (%),TotCld (tenths),TotCld source,TotCld uncert (code),OpqCld (tenths),OpqCld source,OpqCld uncert (code),Dry-bulb (C),Dry-bulb source,Dry-bulb uncert (code),Dew-point (C),Dew-point source,Dew-point uncert (code),RHum (%),RHum source,RHum uncert (code),Pressure (mbar),Pressure source,Pressure uncert (code),Wdir (degrees),Wdir source,Wdir uncert (code),Wspd (m/s),Wspd source,Wspd uncert (code),Hvis (m),Hvis source,Hvis uncert (code),CeilHgt (m),CeilHgt source,CeilHgt uncert (code),Pwat (cm),Pwat source,Pwat uncert (code),AOD (unitless),AOD source,AOD uncert (code),Alb (unitless),Alb source,Alb uncert (code),Lprecip depth (mm),Lprecip quantity (hr),Lprecip source,Lprecip uncert (code),PresWth (METAR code),PresWth source,PresWth uncert (code)' # noqa: E501 + + new_columns = [ + 'ETR', 'ETRN', 'GHI', 'GHISource', 'GHIUncertainty', + 'DNI', 'DNISource', 'DNIUncertainty', 'DHI', 'DHISource', + 'DHIUncertainty', 'GHillum', 'GHillumSource', 'GHillumUncertainty', + 'DNillum', 'DNillumSource', 'DNillumUncertainty', 'DHillum', + 'DHillumSource', 'DHillumUncertainty', 'Zenithlum', + 'ZenithlumSource', 'ZenithlumUncertainty', 'TotCld', 'TotCldSource', + 'TotCldUnertainty', 'OpqCld', 'OpqCldSource', 'OpqCldUncertainty', + 'DryBulb', 'DryBulbSource', 'DryBulbUncertainty', 'DewPoint', + 'DewPointSource', 'DewPointUncertainty', 'RHum', 'RHumSource', + 'RHumUncertainty', 'Pressure', 'PressureSource', + 'PressureUncertainty', 'Wdir', 'WdirSource', 'WdirUncertainty', + 'Wspd', 'WspdSource', 'WspdUncertainty', 'Hvis', 'HvisSource', + 'HvisUncertainty', 'CeilHgt', 'CeilHgtSource', 'CeilHgtUncertainty', + 'Pwat', 'PwatSource', 'PwatUncertainty', 'AOD', 'AODSource', + 'AODUncertainty', 'Alb', 'AlbSource', 'AlbUncertainty', + 'Lprecipdepth', 'Lprecipquantity', 'LprecipSource', + 'LprecipUncertainty', 'PresWth', 'PresWthSource', + 'PresWthUncertainty'] + + mapping = dict(zip(raw_columns.split(','), new_columns)) + + return tmy3_dataframe.rename(columns=mapping) + + +def read_tmy2(filename): + ''' + Read a TMY2 file in to a DataFrame. + + Note that values contained in the DataFrame are unchanged from the + TMY2 file (i.e. units are retained). Time/Date and location data + imported from the TMY2 file have been modified to a "friendlier" + form conforming to modern conventions (e.g. N latitude is postive, E + longitude is positive, the "24th" hour of any day is technically the + "0th" hour of the next day). In the case of any discrepencies + between this documentation and the TMY2 User's Manual [1], the TMY2 + User's Manual takes precedence. + + Parameters + ---------- + filename : None or string + If None, attempts to use a Tkinter file browser. A string can be + a relative file path, absolute file path, or url. + + Returns + ------- + Tuple of the form (data, metadata). + + data : DataFrame + A dataframe with the columns described in the table below. For a + more detailed descriptions of each component, please consult the + TMY2 User's Manual ([1]), especially tables 3-1 through 3-6, and + Appendix B. + + metadata : dict + The site metadata available in the file. + + Notes + ----- + + The returned structures have the following fields. + + ============= ================================== + key description + ============= ================================== + WBAN Site identifier code (WBAN number) + City Station name + State Station state 2 letter designator + TZ Hours from Greenwich + latitude Latitude in decimal degrees + longitude Longitude in decimal degrees + altitude Site elevation in meters + ============= ================================== + + ============================ ========================================================================================================================================================================== + TMYData field description + ============================ ========================================================================================================================================================================== + index Pandas timeseries object containing timestamps + year + month + day + hour + ETR Extraterrestrial horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + ETRN Extraterrestrial normal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + GHI Direct and diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + GHISource See [1], Table 3-3 + GHIUncertainty See [1], Table 3-4 + DNI Amount of direct normal radiation (modeled) recv'd during 60 mintues prior to timestamp, Wh/m^2 + DNISource See [1], Table 3-3 + DNIUncertainty See [1], Table 3-4 + DHI Amount of diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 + DHISource See [1], Table 3-3 + DHIUncertainty See [1], Table 3-4 + GHillum Avg. total horizontal illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux (e.g. value of 50 = 5000 lux) + GHillumSource See [1], Table 3-3 + GHillumUncertainty See [1], Table 3-4 + DNillum Avg. direct normal illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux + DNillumSource See [1], Table 3-3 + DNillumUncertainty See [1], Table 3-4 + DHillum Avg. horizontal diffuse illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux + DHillumSource See [1], Table 3-3 + DHillumUncertainty See [1], Table 3-4 + Zenithlum Avg. luminance at the sky's zenith during the 60 minutes prior to timestamp, units of 10 Cd/m^2 (e.g. value of 700 = 7,000 Cd/m^2) + ZenithlumSource See [1], Table 3-3 + ZenithlumUncertainty See [1], Table 3-4 + TotCld Amount of sky dome covered by clouds or obscuring phenonema at time stamp, tenths of sky + TotCldSource See [1], Table 3-5, 8760x1 cell array of strings + TotCldUnertainty See [1], Table 3-6 + OpqCld Amount of sky dome covered by clouds or obscuring phenonema that prevent observing the sky at time stamp, tenths of sky + OpqCldSource See [1], Table 3-5, 8760x1 cell array of strings + OpqCldUncertainty See [1], Table 3-6 + DryBulb Dry bulb temperature at the time indicated, in tenths of degree C (e.g. 352 = 35.2 C). + DryBulbSource See [1], Table 3-5, 8760x1 cell array of strings + DryBulbUncertainty See [1], Table 3-6 + DewPoint Dew-point temperature at the time indicated, in tenths of degree C (e.g. 76 = 7.6 C). + DewPointSource See [1], Table 3-5, 8760x1 cell array of strings + DewPointUncertainty See [1], Table 3-6 + RHum Relative humidity at the time indicated, percent + RHumSource See [1], Table 3-5, 8760x1 cell array of strings + RHumUncertainty See [1], Table 3-6 + Pressure Station pressure at the time indicated, 1 mbar + PressureSource See [1], Table 3-5, 8760x1 cell array of strings + PressureUncertainty See [1], Table 3-6 + Wdir Wind direction at time indicated, degrees from east of north (360 = 0 = north; 90 = East; 0 = undefined,calm) + WdirSource See [1], Table 3-5, 8760x1 cell array of strings + WdirUncertainty See [1], Table 3-6 + Wspd Wind speed at the time indicated, in tenths of meters/second (e.g. 212 = 21.2 m/s) + WspdSource See [1], Table 3-5, 8760x1 cell array of strings + WspdUncertainty See [1], Table 3-6 + Hvis Distance to discernable remote objects at time indicated (7777=unlimited, 9999=missing data), in tenths of kilometers (e.g. 341 = 34.1 km). + HvisSource See [1], Table 3-5, 8760x1 cell array of strings + HvisUncertainty See [1], Table 3-6 + CeilHgt Height of cloud base above local terrain (7777=unlimited, 88888=cirroform, 99999=missing data), in meters + CeilHgtSource See [1], Table 3-5, 8760x1 cell array of strings + CeilHgtUncertainty See [1], Table 3-6 + Pwat Total precipitable water contained in a column of unit cross section from Earth to top of atmosphere, in millimeters + PwatSource See [1], Table 3-5, 8760x1 cell array of strings + PwatUncertainty See [1], Table 3-6 + AOD The broadband aerosol optical depth (broadband turbidity) in thousandths on the day indicated (e.g. 114 = 0.114) + AODSource See [1], Table 3-5, 8760x1 cell array of strings + AODUncertainty See [1], Table 3-6 + SnowDepth Snow depth in centimeters on the day indicated, (999 = missing data). + SnowDepthSource See [1], Table 3-5, 8760x1 cell array of strings + SnowDepthUncertainty See [1], Table 3-6 + LastSnowfall Number of days since last snowfall (maximum value of 88, where 88 = 88 or greater days; 99 = missing data) + LastSnowfallSource See [1], Table 3-5, 8760x1 cell array of strings + LastSnowfallUncertainty See [1], Table 3-6 + PresentWeather See [1], Appendix B, an 8760x1 cell array of strings. Each string contains 10 numeric values. The string can be parsed to determine each of 10 observed weather metrics. + ============================ ========================================================================================================================================================================== + + References + ---------- + + [1] Marion, W and Urban, K. "Wilcox, S and Marion, W. "User's Manual + for TMY2s". NREL 1995. + ''' + + if filename is None: + try: + filename = _interactive_load() + except ImportError: + raise ImportError('Interactive load failed. Tkinter not supported ' + 'on this system. Try installing X-Quartz and ' + 'reloading') + + # paste in the column info as one long line + string = '%2d%2d%2d%2d%4d%4d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%2d%1s%1d%2d%1s%1d%4d%1s%1d%4d%1s%1d%3d%1s%1d%4d%1s%1d%3d%1s%1d%3d%1s%1d%4d%1s%1d%5d%1s%1d%10d%3d%1s%1d%3d%1s%1d%3d%1s%1d%2d%1s%1d' # noqa: E501 + columns = 'year,month,day,hour,ETR,ETRN,GHI,GHISource,GHIUncertainty,DNI,DNISource,DNIUncertainty,DHI,DHISource,DHIUncertainty,GHillum,GHillumSource,GHillumUncertainty,DNillum,DNillumSource,DNillumUncertainty,DHillum,DHillumSource,DHillumUncertainty,Zenithlum,ZenithlumSource,ZenithlumUncertainty,TotCld,TotCldSource,TotCldUnertainty,OpqCld,OpqCldSource,OpqCldUncertainty,DryBulb,DryBulbSource,DryBulbUncertainty,DewPoint,DewPointSource,DewPointUncertainty,RHum,RHumSource,RHumUncertainty,Pressure,PressureSource,PressureUncertainty,Wdir,WdirSource,WdirUncertainty,Wspd,WspdSource,WspdUncertainty,Hvis,HvisSource,HvisUncertainty,CeilHgt,CeilHgtSource,CeilHgtUncertainty,PresentWeather,Pwat,PwatSource,PwatUncertainty,AOD,AODSource,AODUncertainty,SnowDepth,SnowDepthSource,SnowDepthUncertainty,LastSnowfall,LastSnowfallSource,LastSnowfallUncertaint' # noqa: E501 + hdr_columns = 'WBAN,City,State,TZ,latitude,longitude,altitude' + + tmy2, tmy2_meta = _read_tmy2(string, columns, hdr_columns, filename) + + return tmy2, tmy2_meta + + +def _parsemeta_tmy2(columns, line): + """Retrieves metadata from the top line of the tmy2 file. + + Parameters + ---------- + columns : string + String of column headings in the header + + line : string + Header string containing DataFrame + + Returns + ------- + meta : Dict of metadata contained in the header string + """ + # Remove duplicated spaces, and read in each element + rawmeta = " ".join(line.split()).split(" ") + meta = rawmeta[:3] # take the first string entries + meta.append(int(rawmeta[3])) + # Convert to decimal notation with S negative + longitude = ( + float(rawmeta[5]) + float(rawmeta[6])/60) * (2*(rawmeta[4] == 'N') - 1) + # Convert to decimal notation with W negative + latitude = ( + float(rawmeta[8]) + float(rawmeta[9])/60) * (2*(rawmeta[7] == 'E') - 1) + meta.append(longitude) + meta.append(latitude) + meta.append(float(rawmeta[10])) + + # Creates a dictionary of metadata + meta_dict = dict(zip(columns.split(','), meta)) + return meta_dict + + +def _read_tmy2(string, columns, hdr_columns, fname): + head = 1 + date = [] + with open(fname) as infile: + fline = 0 + for line in infile: + # Skip the header + if head != 0: + meta = _parsemeta_tmy2(hdr_columns, line) + head -= 1 + continue + # Reset the cursor and array for each line + cursor = 1 + part = [] + for marker in string.split('%'): + # Skip the first line of markers + if marker == '': + continue + + # Read the next increment from the marker list + increment = int(re.findall('\d+', marker)[0]) + next_cursor = cursor + increment + + # Extract the value from the line in the file + val = (line[cursor:next_cursor]) + # increment the cursor by the length of the read value + cursor = next_cursor + + # Determine the datatype from the marker string + if marker[-1] == 'd': + try: + val = float(val) + except ValueError: + raise ValueError('WARNING: In {} Read value is not an ' + 'integer " {} " '.format(fname, val)) + elif marker[-1] == 's': + try: + val = str(val) + except ValueError: + raise ValueError('WARNING: In {} Read value is not a ' + 'string " {} " '.format(fname, val)) + else: + raise Exception('WARNING: In {} Improper column DataFrame ' + '" %{} " '.format(__name__, marker)) + + part.append(val) + + if fline == 0: + axes = [part] + year = part[0] + 1900 + fline = 1 + else: + axes.append(part) + + # Create datetime objects from read data + date.append(datetime.datetime(year=int(year), + month=int(part[1]), + day=int(part[2]), + hour=(int(part[3]) - 1))) + + data = pd.DataFrame( + axes, index=date, + columns=columns.split(',')).tz_localize(int(meta['TZ'] * 3600)) + + return data, meta diff --git a/pvlib/test/test_location.py b/pvlib/test/test_location.py index fd4a48d359..b37a6cecc0 100644 --- a/pvlib/test/test_location.py +++ b/pvlib/test/test_location.py @@ -232,8 +232,8 @@ def test_get_clearsky_valueerror(): def test_from_tmy_3(): from test_tmy import tmy3_testfile - from pvlib.tmy import readtmy3 - data, meta = readtmy3(tmy3_testfile) + from pvlib.iotools import read_tmy3 + data, meta = read_tmy3(tmy3_testfile) loc = Location.from_tmy(meta, data) assert loc.name is not None assert loc.altitude != 0 @@ -243,8 +243,8 @@ def test_from_tmy_3(): def test_from_tmy_2(): from test_tmy import tmy2_testfile - from pvlib.tmy import readtmy2 - data, meta = readtmy2(tmy2_testfile) + from pvlib.iotools import read_tmy2 + data, meta = read_tmy2(tmy2_testfile) loc = Location.from_tmy(meta, data) assert loc.name is not None assert loc.altitude != 0 diff --git a/pvlib/test/test_pvsystem.py b/pvlib/test/test_pvsystem.py index 42f9505a65..c0e89c5095 100644 --- a/pvlib/test/test_pvsystem.py +++ b/pvlib/test/test_pvsystem.py @@ -11,7 +11,6 @@ from pandas.util.testing import assert_series_equal, assert_frame_equal from numpy.testing import assert_allclose -from pvlib import tmy from pvlib import pvsystem from pvlib import clearsky from pvlib import irradiance @@ -23,9 +22,10 @@ def test_systemdef_tmy3(): - pvlib_abspath = os.path.dirname(os.path.abspath(inspect.getfile(tmy))) + from pvlib.iotools import tmy + pvlib_abspath = os.path.dirname(os.path.abspath(inspect.getfile(pvsystem))) tmy3_testfile = os.path.join(pvlib_abspath, 'data', '703165TY.csv') - tmy3_data, tmy3_metadata = tmy.readtmy3(tmy3_testfile) + tmy3_data, tmy3_metadata = tmy.read_tmy3(tmy3_testfile) expected = {'tz': -9.0, 'albedo': 0.1, 'altitude': 7.0, @@ -40,9 +40,10 @@ def test_systemdef_tmy3(): def test_systemdef_tmy2(): - pvlib_abspath = os.path.dirname(os.path.abspath(inspect.getfile(tmy))) + from pvlib.iotools import tmy + pvlib_abspath = os.path.dirname(os.path.abspath(inspect.getfile(pvsystem))) tmy2_testfile = os.path.join(pvlib_abspath, 'data', '12839.tm2') - tmy2_data, tmy2_metadata = tmy.readtmy2(tmy2_testfile) + tmy2_data, tmy2_metadata = tmy.read_tmy2(tmy2_testfile) expected = {'tz': -5, 'albedo': 0.1, diff --git a/pvlib/test/test_tmy.py b/pvlib/test/test_tmy.py index fdb1c2771f..5df92450bf 100644 --- a/pvlib/test/test_tmy.py +++ b/pvlib/test/test_tmy.py @@ -2,40 +2,60 @@ import os from pandas.util.testing import network +import pytest + +from pvlib._deprecation import pvlibDeprecationWarning +from pvlib.iotools import tmy + +from conftest import fail_on_pvlib_version + test_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) tmy3_testfile = os.path.join(test_dir, '../data/703165TY.csv') tmy2_testfile = os.path.join(test_dir, '../data/12839.tm2') -from pvlib import tmy + +@fail_on_pvlib_version('0.7') +def test_deprecated_07(): + with pytest.warns(pvlibDeprecationWarning): + from pvlib.tmy import readtmy2 + readtmy2(tmy2_testfile) + with pytest.warns(pvlibDeprecationWarning): + from pvlib.tmy import readtmy3 + readtmy3(tmy3_testfile) -def test_readtmy3(): - tmy.readtmy3(tmy3_testfile) +def test_read_tmy3(): + tmy.read_tmy3(tmy3_testfile) + @network -def test_readtmy3_remote(): +def test_read_tmy3_remote(): url = 'http://rredc.nrel.gov/solar/old_data/nsrdb/1991-2005/data/tmy3/703165TYA.CSV' - tmy.readtmy3(url) - -def test_readtmy3_recolumn(): - data, meta = tmy.readtmy3(tmy3_testfile) + tmy.read_tmy3(url) + + +def test_read_tmy3_recolumn(): + data, meta = tmy.read_tmy3(tmy3_testfile) assert 'GHISource' in data.columns - -def test_readtmy3_norecolumn(): - data, meta = tmy.readtmy3(tmy3_testfile, recolumn=False) + + +def test_read_tmy3_norecolumn(): + data, meta = tmy.read_tmy3(tmy3_testfile, recolumn=False) assert 'GHI source' in data.columns - -def test_readtmy3_coerce_year(): + + +def test_read_tmy3_coerce_year(): coerce_year = 1987 - data, meta = tmy.readtmy3(tmy3_testfile, coerce_year=coerce_year) + data, meta = tmy.read_tmy3(tmy3_testfile, coerce_year=coerce_year) assert (data.index.year == 1987).all() - -def test_readtmy3_no_coerce_year(): + + +def test_read_tmy3_no_coerce_year(): coerce_year = None - data, meta = tmy.readtmy3(tmy3_testfile, coerce_year=coerce_year) + data, meta = tmy.read_tmy3(tmy3_testfile, coerce_year=coerce_year) assert 1997 and 1999 in data.index.year - -def test_readtmy2(): - tmy.readtmy2(tmy2_testfile) - + + +def test_read_tmy2(): + tmy.read_tmy2(tmy2_testfile) diff --git a/pvlib/tmy.py b/pvlib/tmy.py index 17cf316f28..a9ea8164f9 100644 --- a/pvlib/tmy.py +++ b/pvlib/tmy.py @@ -1,516 +1,13 @@ """ -Import functions for TMY2 and TMY3 data files. +Deprecated version of pvlib.iotools.tmy """ -import re -import datetime -import dateutil -import io -try: - from urllib2 import urlopen, Request -except ImportError: - from urllib.request import urlopen, Request +from pvlib.iotools import read_tmy2, read_tmy3 -import pandas as pd +from pvlib._deprecation import deprecated +readtmy2 = deprecated('0.6.1', alternative='iotools.read_tmy2', + name='readtmy2', removal='0.7')(read_tmy2) -def readtmy3(filename=None, coerce_year=None, recolumn=True): - ''' - Read a TMY3 file in to a pandas dataframe. - - Note that values contained in the metadata dictionary are unchanged - from the TMY3 file (i.e. units are retained). In the case of any - discrepencies between this documentation and the TMY3 User's Manual - [1], the TMY3 User's Manual takes precedence. - - The TMY3 files were updated in Jan. 2015. This function requires the - use of the updated files. - - Parameters - ---------- - filename : None or string, default None - If None, attempts to use a Tkinter file browser. A string can be - a relative file path, absolute file path, or url. - - coerce_year : None or int, default None - If supplied, the year of the data will be set to this value. - - recolumn : bool, default True - If True, apply standard names to TMY3 columns. Typically this - results in stripping the units from the column name. - - Returns - ------- - Tuple of the form (data, metadata). - - data : DataFrame - A pandas dataframe with the columns described in the table - below. For more detailed descriptions of each component, please - consult the TMY3 User's Manual ([1]), especially tables 1-1 - through 1-6. - - metadata : dict - The site metadata available in the file. - - Notes - ----- - - The returned structures have the following fields. - - =============== ====== =================== - key format description - =============== ====== =================== - altitude Float site elevation - latitude Float site latitudeitude - longitude Float site longitudeitude - Name String site name - State String state - TZ Float UTC offset - USAF Int USAF identifier - =============== ====== =================== - - ============================= ====================================================================================================================================================== - TMYData field description - ============================= ====================================================================================================================================================== - TMYData.Index A pandas datetime index. NOTE, the index is currently timezone unaware, and times are set to local standard time (daylight savings is not indcluded) - TMYData.ETR Extraterrestrial horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - TMYData.ETRN Extraterrestrial normal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - TMYData.GHI Direct and diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - TMYData.GHISource See [1], Table 1-4 - TMYData.GHIUncertainty Uncertainty based on random and bias error estimates see [2] - TMYData.DNI Amount of direct normal radiation (modeled) recv'd during 60 mintues prior to timestamp, Wh/m^2 - TMYData.DNISource See [1], Table 1-4 - TMYData.DNIUncertainty Uncertainty based on random and bias error estimates see [2] - TMYData.DHI Amount of diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - TMYData.DHISource See [1], Table 1-4 - TMYData.DHIUncertainty Uncertainty based on random and bias error estimates see [2] - TMYData.GHillum Avg. total horizontal illuminance recv'd during the 60 minutes prior to timestamp, lx - TMYData.GHillumSource See [1], Table 1-4 - TMYData.GHillumUncertainty Uncertainty based on random and bias error estimates see [2] - TMYData.DNillum Avg. direct normal illuminance recv'd during the 60 minutes prior to timestamp, lx - TMYData.DNillumSource See [1], Table 1-4 - TMYData.DNillumUncertainty Uncertainty based on random and bias error estimates see [2] - TMYData.DHillum Avg. horizontal diffuse illuminance recv'd during the 60 minutes prior to timestamp, lx - TMYData.DHillumSource See [1], Table 1-4 - TMYData.DHillumUncertainty Uncertainty based on random and bias error estimates see [2] - TMYData.Zenithlum Avg. luminance at the sky's zenith during the 60 minutes prior to timestamp, cd/m^2 - TMYData.ZenithlumSource See [1], Table 1-4 - TMYData.ZenithlumUncertainty Uncertainty based on random and bias error estimates see [1] section 2.10 - TMYData.TotCld Amount of sky dome covered by clouds or obscuring phenonema at time stamp, tenths of sky - TMYData.TotCldSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.TotCldUnertainty See [1], Table 1-6 - TMYData.OpqCld Amount of sky dome covered by clouds or obscuring phenonema that prevent observing the sky at time stamp, tenths of sky - TMYData.OpqCldSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.OpqCldUncertainty See [1], Table 1-6 - TMYData.DryBulb Dry bulb temperature at the time indicated, deg C - TMYData.DryBulbSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.DryBulbUncertainty See [1], Table 1-6 - TMYData.DewPoint Dew-point temperature at the time indicated, deg C - TMYData.DewPointSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.DewPointUncertainty See [1], Table 1-6 - TMYData.RHum Relatitudeive humidity at the time indicated, percent - TMYData.RHumSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.RHumUncertainty See [1], Table 1-6 - TMYData.Pressure Station pressure at the time indicated, 1 mbar - TMYData.PressureSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.PressureUncertainty See [1], Table 1-6 - TMYData.Wdir Wind direction at time indicated, degrees from north (360 = north; 0 = undefined,calm) - TMYData.WdirSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.WdirUncertainty See [1], Table 1-6 - TMYData.Wspd Wind speed at the time indicated, meter/second - TMYData.WspdSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.WspdUncertainty See [1], Table 1-6 - TMYData.Hvis Distance to discernable remote objects at time indicated (7777=unlimited), meter - TMYData.HvisSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.HvisUncertainty See [1], Table 1-6 - TMYData.CeilHgt Height of cloud base above local terrain (7777=unlimited), meter - TMYData.CeilHgtSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.CeilHgtUncertainty See [1], Table 1-6 - TMYData.Pwat Total precipitable water contained in a column of unit cross section from earth to top of atmosphere, cm - TMYData.PwatSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.PwatUncertainty See [1], Table 1-6 - TMYData.AOD The broadband aerosol optical depth per unit of air mass due to extinction by aerosol component of atmosphere, unitless - TMYData.AODSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.AODUncertainty See [1], Table 1-6 - TMYData.Alb The ratio of reflected solar irradiance to global horizontal irradiance, unitless - TMYData.AlbSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.AlbUncertainty See [1], Table 1-6 - TMYData.Lprecipdepth The amount of liquid precipitation observed at indicated time for the period indicated in the liquid precipitation quantity field, millimeter - TMYData.Lprecipquantity The period of accumulatitudeion for the liquid precipitation depth field, hour - TMYData.LprecipSource See [1], Table 1-5, 8760x1 cell array of strings - TMYData.LprecipUncertainty See [1], Table 1-6 - TMYData.PresWth Present weather code, see [2]. - TMYData.PresWthSource Present weather code source, see [2]. - TMYData.PresWthUncertainty Present weather code uncertainty, see [2]. - ============================= ====================================================================================================================================================== - - References - ---------- - - [1] Wilcox, S and Marion, W. "Users Manual for TMY3 Data Sets". - NREL/TP-581-43156, Revised May 2008. - - [2] Wilcox, S. (2007). National Solar Radiation Database 1991 2005 - Update: Users Manual. 472 pp.; NREL Report No. TP-581-41364. - ''' - - if filename is None: - try: - filename = _interactive_load() - except ImportError: - raise ImportError('Interactive load failed. Tkinter not supported ' - 'on this system. Try installing X-Quartz and ' - 'reloading') - - head = ['USAF', 'Name', 'State', 'TZ', 'latitude', 'longitude', 'altitude'] - - if filename.startswith('http'): - request = Request(filename, headers={'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 ' - 'Safari/537.36'}) - response = urlopen(request) - csvdata = io.StringIO(response.read().decode(errors='ignore')) - else: - # assume it's accessible via the file system - csvdata = open(filename, 'r') - - # read in file metadata, advance buffer to second line - firstline = csvdata.readline() - if 'Request Rejected' in firstline: - raise IOError('Remote server rejected TMY file request') - - meta = dict(zip(head, firstline.rstrip('\n').split(","))) - - # convert metadata strings to numeric types - meta['altitude'] = float(meta['altitude']) - meta['latitude'] = float(meta['latitude']) - meta['longitude'] = float(meta['longitude']) - meta['TZ'] = float(meta['TZ']) - meta['USAF'] = int(meta['USAF']) - - # use pandas to read the csv file/stringio buffer - # header is actually the second line in file, but tell pandas to look for - # header information on the 1st line (0 indexing) because we've already - # advanced past the true first line with the readline call above. - data = pd.read_csv( - csvdata, header=0, - parse_dates={'datetime': ['Date (MM/DD/YYYY)', 'Time (HH:MM)']}, - date_parser=lambda *x: _parsedate(*x, year=coerce_year), - index_col='datetime') - - if recolumn: - data = _recolumn(data) # rename to standard column names - - data = data.tz_localize(int(meta['TZ']*3600)) - - return data, meta - - -def _interactive_load(): - import Tkinter - from tkFileDialog import askopenfilename - Tkinter.Tk().withdraw() # Start interactive file input - return askopenfilename() - - -def _parsedate(ymd, hour, year=None): - # stupidly complicated due to TMY3's usage of hour 24 - # and dateutil's inability to handle that. - offset_hour = int(hour[:2]) - 1 - offset_datetime = '{} {}:00'.format(ymd, offset_hour) - offset_date = dateutil.parser.parse(offset_datetime) - true_date = offset_date + dateutil.relativedelta.relativedelta(hours=1) - if year is not None: - true_date = true_date.replace(year=year) - return true_date - - -def _recolumn(tmy3_dataframe): - """ - Rename the columns of the TMY3 DataFrame. - - Parameters - ---------- - tmy3_dataframe : DataFrame - inplace : bool - passed to DataFrame.rename() - - Returns - ------- - Recolumned DataFrame. - """ - raw_columns = 'ETR (W/m^2),ETRN (W/m^2),GHI (W/m^2),GHI source,GHI uncert (%),DNI (W/m^2),DNI source,DNI uncert (%),DHI (W/m^2),DHI source,DHI uncert (%),GH illum (lx),GH illum source,Global illum uncert (%),DN illum (lx),DN illum source,DN illum uncert (%),DH illum (lx),DH illum source,DH illum uncert (%),Zenith lum (cd/m^2),Zenith lum source,Zenith lum uncert (%),TotCld (tenths),TotCld source,TotCld uncert (code),OpqCld (tenths),OpqCld source,OpqCld uncert (code),Dry-bulb (C),Dry-bulb source,Dry-bulb uncert (code),Dew-point (C),Dew-point source,Dew-point uncert (code),RHum (%),RHum source,RHum uncert (code),Pressure (mbar),Pressure source,Pressure uncert (code),Wdir (degrees),Wdir source,Wdir uncert (code),Wspd (m/s),Wspd source,Wspd uncert (code),Hvis (m),Hvis source,Hvis uncert (code),CeilHgt (m),CeilHgt source,CeilHgt uncert (code),Pwat (cm),Pwat source,Pwat uncert (code),AOD (unitless),AOD source,AOD uncert (code),Alb (unitless),Alb source,Alb uncert (code),Lprecip depth (mm),Lprecip quantity (hr),Lprecip source,Lprecip uncert (code),PresWth (METAR code),PresWth source,PresWth uncert (code)' - - new_columns = [ - 'ETR', 'ETRN', 'GHI', 'GHISource', 'GHIUncertainty', - 'DNI', 'DNISource', 'DNIUncertainty', 'DHI', 'DHISource', - 'DHIUncertainty', 'GHillum', 'GHillumSource', 'GHillumUncertainty', - 'DNillum', 'DNillumSource', 'DNillumUncertainty', 'DHillum', - 'DHillumSource', 'DHillumUncertainty', 'Zenithlum', - 'ZenithlumSource', 'ZenithlumUncertainty', 'TotCld', 'TotCldSource', - 'TotCldUnertainty', 'OpqCld', 'OpqCldSource', 'OpqCldUncertainty', - 'DryBulb', 'DryBulbSource', 'DryBulbUncertainty', 'DewPoint', - 'DewPointSource', 'DewPointUncertainty', 'RHum', 'RHumSource', - 'RHumUncertainty', 'Pressure', 'PressureSource', - 'PressureUncertainty', 'Wdir', 'WdirSource', 'WdirUncertainty', - 'Wspd', 'WspdSource', 'WspdUncertainty', 'Hvis', 'HvisSource', - 'HvisUncertainty', 'CeilHgt', 'CeilHgtSource', 'CeilHgtUncertainty', - 'Pwat', 'PwatSource', 'PwatUncertainty', 'AOD', 'AODSource', - 'AODUncertainty', 'Alb', 'AlbSource', 'AlbUncertainty', - 'Lprecipdepth', 'Lprecipquantity', 'LprecipSource', - 'LprecipUncertainty', 'PresWth', 'PresWthSource', - 'PresWthUncertainty'] - - mapping = dict(zip(raw_columns.split(','), new_columns)) - - return tmy3_dataframe.rename(columns=mapping) - - -def readtmy2(filename): - ''' - Read a TMY2 file in to a DataFrame. - - Note that values contained in the DataFrame are unchanged from the - TMY2 file (i.e. units are retained). Time/Date and location data - imported from the TMY2 file have been modified to a "friendlier" - form conforming to modern conventions (e.g. N latitude is postive, E - longitude is positive, the "24th" hour of any day is technically the - "0th" hour of the next day). In the case of any discrepencies - between this documentation and the TMY2 User's Manual [1], the TMY2 - User's Manual takes precedence. - - Parameters - ---------- - filename : None or string - If None, attempts to use a Tkinter file browser. A string can be - a relative file path, absolute file path, or url. - - Returns - ------- - Tuple of the form (data, metadata). - - data : DataFrame - A dataframe with the columns described in the table below. For a - more detailed descriptions of each component, please consult the - TMY2 User's Manual ([1]), especially tables 3-1 through 3-6, and - Appendix B. - - metadata : dict - The site metadata available in the file. - - Notes - ----- - - The returned structures have the following fields. - - ============= ================================== - key description - ============= ================================== - WBAN Site identifier code (WBAN number) - City Station name - State Station state 2 letter designator - TZ Hours from Greenwich - latitude Latitude in decimal degrees - longitude Longitude in decimal degrees - altitude Site elevation in meters - ============= ================================== - - ============================ ========================================================================================================================================================================== - TMYData field description - ============================ ========================================================================================================================================================================== - index Pandas timeseries object containing timestamps - year - month - day - hour - ETR Extraterrestrial horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - ETRN Extraterrestrial normal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - GHI Direct and diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - GHISource See [1], Table 3-3 - GHIUncertainty See [1], Table 3-4 - DNI Amount of direct normal radiation (modeled) recv'd during 60 mintues prior to timestamp, Wh/m^2 - DNISource See [1], Table 3-3 - DNIUncertainty See [1], Table 3-4 - DHI Amount of diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2 - DHISource See [1], Table 3-3 - DHIUncertainty See [1], Table 3-4 - GHillum Avg. total horizontal illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux (e.g. value of 50 = 5000 lux) - GHillumSource See [1], Table 3-3 - GHillumUncertainty See [1], Table 3-4 - DNillum Avg. direct normal illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux - DNillumSource See [1], Table 3-3 - DNillumUncertainty See [1], Table 3-4 - DHillum Avg. horizontal diffuse illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux - DHillumSource See [1], Table 3-3 - DHillumUncertainty See [1], Table 3-4 - Zenithlum Avg. luminance at the sky's zenith during the 60 minutes prior to timestamp, units of 10 Cd/m^2 (e.g. value of 700 = 7,000 Cd/m^2) - ZenithlumSource See [1], Table 3-3 - ZenithlumUncertainty See [1], Table 3-4 - TotCld Amount of sky dome covered by clouds or obscuring phenonema at time stamp, tenths of sky - TotCldSource See [1], Table 3-5, 8760x1 cell array of strings - TotCldUnertainty See [1], Table 3-6 - OpqCld Amount of sky dome covered by clouds or obscuring phenonema that prevent observing the sky at time stamp, tenths of sky - OpqCldSource See [1], Table 3-5, 8760x1 cell array of strings - OpqCldUncertainty See [1], Table 3-6 - DryBulb Dry bulb temperature at the time indicated, in tenths of degree C (e.g. 352 = 35.2 C). - DryBulbSource See [1], Table 3-5, 8760x1 cell array of strings - DryBulbUncertainty See [1], Table 3-6 - DewPoint Dew-point temperature at the time indicated, in tenths of degree C (e.g. 76 = 7.6 C). - DewPointSource See [1], Table 3-5, 8760x1 cell array of strings - DewPointUncertainty See [1], Table 3-6 - RHum Relative humidity at the time indicated, percent - RHumSource See [1], Table 3-5, 8760x1 cell array of strings - RHumUncertainty See [1], Table 3-6 - Pressure Station pressure at the time indicated, 1 mbar - PressureSource See [1], Table 3-5, 8760x1 cell array of strings - PressureUncertainty See [1], Table 3-6 - Wdir Wind direction at time indicated, degrees from east of north (360 = 0 = north; 90 = East; 0 = undefined,calm) - WdirSource See [1], Table 3-5, 8760x1 cell array of strings - WdirUncertainty See [1], Table 3-6 - Wspd Wind speed at the time indicated, in tenths of meters/second (e.g. 212 = 21.2 m/s) - WspdSource See [1], Table 3-5, 8760x1 cell array of strings - WspdUncertainty See [1], Table 3-6 - Hvis Distance to discernable remote objects at time indicated (7777=unlimited, 9999=missing data), in tenths of kilometers (e.g. 341 = 34.1 km). - HvisSource See [1], Table 3-5, 8760x1 cell array of strings - HvisUncertainty See [1], Table 3-6 - CeilHgt Height of cloud base above local terrain (7777=unlimited, 88888=cirroform, 99999=missing data), in meters - CeilHgtSource See [1], Table 3-5, 8760x1 cell array of strings - CeilHgtUncertainty See [1], Table 3-6 - Pwat Total precipitable water contained in a column of unit cross section from Earth to top of atmosphere, in millimeters - PwatSource See [1], Table 3-5, 8760x1 cell array of strings - PwatUncertainty See [1], Table 3-6 - AOD The broadband aerosol optical depth (broadband turbidity) in thousandths on the day indicated (e.g. 114 = 0.114) - AODSource See [1], Table 3-5, 8760x1 cell array of strings - AODUncertainty See [1], Table 3-6 - SnowDepth Snow depth in centimeters on the day indicated, (999 = missing data). - SnowDepthSource See [1], Table 3-5, 8760x1 cell array of strings - SnowDepthUncertainty See [1], Table 3-6 - LastSnowfall Number of days since last snowfall (maximum value of 88, where 88 = 88 or greater days; 99 = missing data) - LastSnowfallSource See [1], Table 3-5, 8760x1 cell array of strings - LastSnowfallUncertainty See [1], Table 3-6 - PresentWeather See [1], Appendix B, an 8760x1 cell array of strings. Each string contains 10 numeric values. The string can be parsed to determine each of 10 observed weather metrics. - ============================ ========================================================================================================================================================================== - - References - ---------- - - [1] Marion, W and Urban, K. "Wilcox, S and Marion, W. "User's Manual - for TMY2s". NREL 1995. - ''' - - if filename is None: - try: - filename = _interactive_load() - except ImportError: - raise ImportError('Interactive load failed. Tkinter not supported ' - 'on this system. Try installing X-Quartz and ' - 'reloading') - - string = '%2d%2d%2d%2d%4d%4d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%2d%1s%1d%2d%1s%1d%4d%1s%1d%4d%1s%1d%3d%1s%1d%4d%1s%1d%3d%1s%1d%3d%1s%1d%4d%1s%1d%5d%1s%1d%10d%3d%1s%1d%3d%1s%1d%3d%1s%1d%2d%1s%1d' - columns = 'year,month,day,hour,ETR,ETRN,GHI,GHISource,GHIUncertainty,DNI,DNISource,DNIUncertainty,DHI,DHISource,DHIUncertainty,GHillum,GHillumSource,GHillumUncertainty,DNillum,DNillumSource,DNillumUncertainty,DHillum,DHillumSource,DHillumUncertainty,Zenithlum,ZenithlumSource,ZenithlumUncertainty,TotCld,TotCldSource,TotCldUnertainty,OpqCld,OpqCldSource,OpqCldUncertainty,DryBulb,DryBulbSource,DryBulbUncertainty,DewPoint,DewPointSource,DewPointUncertainty,RHum,RHumSource,RHumUncertainty,Pressure,PressureSource,PressureUncertainty,Wdir,WdirSource,WdirUncertainty,Wspd,WspdSource,WspdUncertainty,Hvis,HvisSource,HvisUncertainty,CeilHgt,CeilHgtSource,CeilHgtUncertainty,PresentWeather,Pwat,PwatSource,PwatUncertainty,AOD,AODSource,AODUncertainty,SnowDepth,SnowDepthSource,SnowDepthUncertainty,LastSnowfall,LastSnowfallSource,LastSnowfallUncertaint' - hdr_columns = 'WBAN,City,State,TZ,latitude,longitude,altitude' - - tmy2, tmy2_meta = _read_tmy2(string, columns, hdr_columns, filename) - - return tmy2, tmy2_meta - - -def _parsemeta_tmy2(columns, line): - """Retrieves metadata from the top line of the tmy2 file. - - Parameters - ---------- - columns : string - String of column headings in the header - - line : string - Header string containing DataFrame - - Returns - ------- - meta : Dict of metadata contained in the header string - """ - # Remove duplicated spaces, and read in each element - rawmeta = " ".join(line.split()).split(" ") - meta = rawmeta[:3] # take the first string entries - meta.append(int(rawmeta[3])) - # Convert to decimal notation with S negative - longitude = ( - float(rawmeta[5]) + float(rawmeta[6])/60) * (2*(rawmeta[4] == 'N') - 1) - # Convert to decimal notation with W negative - latitude = ( - float(rawmeta[8]) + float(rawmeta[9])/60) * (2*(rawmeta[7] == 'E') - 1) - meta.append(longitude) - meta.append(latitude) - meta.append(float(rawmeta[10])) - - # Creates a dictionary of metadata - meta_dict = dict(zip(columns.split(','), meta)) - return meta_dict - - -def _read_tmy2(string, columns, hdr_columns, fname): - head = 1 - date = [] - with open(fname) as infile: - fline = 0 - for line in infile: - # Skip the header - if head != 0: - meta = _parsemeta_tmy2(hdr_columns, line) - head -= 1 - continue - # Reset the cursor and array for each line - cursor = 1 - part = [] - for marker in string.split('%'): - # Skip the first line of markers - if marker == '': - continue - - # Read the next increment from the marker list - increment = int(re.findall('\d+', marker)[0]) - - # Extract the value from the line in the file - val = (line[cursor:cursor+increment]) - # increment the cursor by the length of the read value - cursor = cursor+increment - - # Determine the datatype from the marker string - if marker[-1] == 'd': - try: - val = float(val) - except: - raise Exception('WARNING: In {} Read value is not an ' - 'integer " {} " '.format(fname, val)) - elif marker[-1] == 's': - try: - val = str(val) - except: - raise Exception('WARNING: In {} Read value is not a ' - 'string " {} " '.format(fname, val)) - else: - raise Exception('WARNING: In {} Improper column DataFrame ' - '" %{} " '.format(__name__, marker)) - - part.append(val) - - if fline == 0: - axes = [part] - year = part[0]+1900 - fline = 1 - else: - axes.append(part) - - # Create datetime objects from read data - date.append(datetime.datetime(year=int(year), - month=int(part[1]), - day=int(part[2]), - hour=int(part[3])-1)) - - data = pd.DataFrame( - axes, index=date, - columns=columns.split(',')).tz_localize(int(meta['TZ']*3600)) - - return data, meta +readtmy3 = deprecated('0.6.1', alternative='iotools.read_tmy2', + name='readtmy3', removal='0.7')(read_tmy3)