Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace freq '1M' with '1MS' #2266

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Replace freq '1M' with '1MS' #2266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
05fc1d1
Replace freq '1M' with '1ME'
AdamRJensen Oct 16, 2024
1242b48
Switch ME to MS
AdamRJensen Oct 16, 2024
2228d8b
Modify turbidity tess
AdamRJensen Oct 17, 2024
bfe8a5c
Correct date_range in srml and bsrn
AdamRJensen Oct 17, 2024
cbf9224
Remove label option for sodapro functions
AdamRJensen Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions pvlib/iotools/bsrn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,7 @@ def get_bsrn(station, start, end, username, password,
end = pd.to_datetime(end)

# Generate list files to download based on start/end (SSSMMYY.dat.gz)
filenames = pd.date_range(
start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M')\
filenames = pd.date_range(start, end, freq='1MS')\
.strftime(f"{station}%m%y.dat.gz").tolist()

# Create FTP connection
Expand Down
54 changes: 22 additions & 32 deletions pvlib/iotools/sodapro.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,20 +32,20 @@

# Dictionary mapping time steps to CAMS time step format
TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H',
'1d': 'P01D', '1M': 'P01M'}
'1d': 'P01D', '1MS': 'P01M'}

TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24}

SUMMATION_PERIOD_TO_TIME_STEP = {'0 year 0 month 0 day 0 h 1 min 0 s': '1min',
'0 year 0 month 0 day 0 h 15 min 0 s': '15min', # noqa
'0 year 0 month 0 day 1 h 0 min 0 s': '1h',
'0 year 0 month 1 day 0 h 0 min 0 s': '1d',
'0 year 1 month 0 day 0 h 0 min 0 s': '1M'}
'0 year 1 month 0 day 0 h 0 min 0 s': '1MS'}


def get_cams(latitude, longitude, start, end, email, identifier='mcclear',
altitude=None, time_step='1h', time_ref='UT', verbose=False,
integrated=False, label=None, map_variables=True,
integrated=False, map_variables=True,
server=URL, timeout=30):
"""Retrieve irradiance and clear-sky time series from CAMS.

Expand Down Expand Up @@ -79,7 +79,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear',
altitude: float, optional
Altitude in meters. If not specified, then the altitude is determined
from the NASA SRTM database
time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h'
time_step: str, {'1min', '15min', '1h', '1d', '1MS'}, default: '1h'
Time step of the time series, either 1 minute, 15 minute, hourly,
daily, or monthly.
time_ref: str, {'UT', 'TST'}, default: 'UT'
Expand All @@ -90,9 +90,6 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear',
integrated: boolean, default False
Whether to return radiation parameters as integrated values (Wh/m^2)
or as average irradiance values (W/m^2) (pvlib preferred units)
label : {'right', 'left'}, optional
Which bin edge label to label time-step with. The default is 'left' for
all time steps except for '1M' which has a default of 'right'.
map_variables: bool, default: True
When true, renames columns of the DataFrame to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Expand Down Expand Up @@ -120,7 +117,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear',
======================== ====== =========================================
**Mapped field names are returned when the map_variables argument is True**
---------------------------------------------------------------------------
Observation period str Beginning/end of time period
Observation period str Start of time period
TOA, ghi_extra float Horizontal radiation at top of atmosphere
Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal
Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal
Expand Down Expand Up @@ -231,12 +228,12 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear',
# Successful requests returns a csv data file
else:
fbuf = io.StringIO(res.content.decode('utf-8'))
data, metadata = parse_cams(fbuf, integrated=integrated, label=label,
data, metadata = parse_cams(fbuf, integrated=integrated,
map_variables=map_variables)
return data, metadata


def parse_cams(fbuf, integrated=False, label=None, map_variables=True):
def parse_cams(fbuf, integrated=False, map_variables=True):
"""
Parse a file-like buffer with data in the format of a CAMS Radiation or
McClear file. The CAMS solar radiation services are described in [1]_.
Expand All @@ -248,9 +245,6 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True):
integrated: boolean, default False
Whether to return radiation parameters as integrated values (Wh/m^2)
or as average irradiance values (W/m^2) (pvlib preferred units)
label : {'right', 'left'}, optional
Which bin edge label to label time-step with. The default is 'left' for
all time steps except for '1M' which has a default of 'right'.
map_variables: bool, default: True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Expand All @@ -262,6 +256,10 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True):
metadata: dict
Metadata available in the file.

Notes
-----
The index timestamps correspond to the start/left of the interval.

See Also
--------
pvlib.iotools.read_cams, pvlib.iotools.get_cams
Expand Down Expand Up @@ -301,26 +299,17 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True):
obs_period = data['Observation period'].str.split('/')

# Set index as the start observation time (left) and localize to UTC
if (label == 'left') | ((label is None) & (time_step != '1M')):
data.index = pd.to_datetime(obs_period.str[0], utc=True)
# Set index as the stop observation time (right) and localize to UTC
# default label for monthly data is 'right' following Pandas' convention
elif (label == 'right') | ((label is None) & (time_step == '1M')):
data.index = pd.to_datetime(obs_period.str[1], utc=True)

# For time_steps '1d' and '1M', drop timezone and round to nearest midnight
if (time_step == '1d') | (time_step == '1M'):
data.index = pd.to_datetime(obs_period.str[0], utc=True)

# For time_steps '1d' and '1MS' drop timezone and round to nearest midnight
if (time_step == '1d') | time_step.endswith('MS'): # noqa
data.index = pd.DatetimeIndex(data.index.date)
# For monthly data with 'right' label, the index should be the last
# date of the month and not the first date of the following month
if (time_step == '1M') & (label != 'left'):
data.index = data.index - pd.Timedelta(days=1)

if not integrated: # Convert radiation values from Wh/m2 to W/m2
integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS
if c in data.columns]

if time_step == '1M':
if time_step.endswith('MS'):
time_delta = (pd.to_datetime(obs_period.str[1])
- pd.to_datetime(obs_period.str[0]))
hours = time_delta.dt.total_seconds()/60/60
Expand All @@ -336,7 +325,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True):
return data, metadata


def read_cams(filename, integrated=False, label=None, map_variables=True):
def read_cams(filename, integrated=False, map_variables=True):
"""
Read a CAMS Radiation or McClear file into a pandas DataFrame.

Expand All @@ -349,9 +338,6 @@ def read_cams(filename, integrated=False, label=None, map_variables=True):
integrated: boolean, default False
Whether to return radiation parameters as integrated values (Wh/m^2)
or as average irradiance values (W/m^2) (pvlib preferred units)
label : {'right', 'left}, optional
Which bin edge label to label time-step with. The default is 'left' for
all time steps except for '1M' which has a default of 'right'.
map_variables: bool, default: True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Expand All @@ -368,11 +354,15 @@ def read_cams(filename, integrated=False, label=None, map_variables=True):
--------
pvlib.iotools.parse_cams, pvlib.iotools.get_cams

Notes
-----
The index timestamps correspond to the start/left of the interval.

References
----------
.. [1] `CAMS solar radiation documentation
<https://atmosphere.copernicus.eu/solar-radiation>`_
"""
with open(str(filename), 'r') as fbuf:
content = parse_cams(fbuf, integrated, label, map_variables)
content = parse_cams(fbuf, integrated, map_variables)
return content
3 changes: 1 addition & 2 deletions pvlib/iotools/srml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
end = pd.to_datetime(end)

# Generate list of months
months = pd.date_range(
start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M')
months = pd.date_range(start, end, freq='1MS')
months_str = months.strftime('%y%m')

# Generate list of filenames
Expand Down
21 changes: 4 additions & 17 deletions pvlib/tests/iotools/test_sodapro.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

index_verbose = pd.date_range('2020-06-01 12', periods=4, freq='1min',
tz='UTC')
index_monthly = pd.date_range('2020-01-01', periods=4, freq='1M')
index_monthly = pd.date_range('2020-01-01', periods=4, freq='1MS')


dtypes_mcclear_verbose = [
Expand Down Expand Up @@ -172,19 +172,6 @@ def test_read_cams(testfile, index, columns, values, dtypes):
assert_frame_equal(out, expected, check_less_precise=True)


def test_read_cams_integrated_unmapped_label():
# Default label is 'left' for 1 minute time resolution, hence 1 minute is
# added for label='right'
expected = generate_expected_dataframe(
values_radiation_verbose_integrated,
columns_radiation_verbose_unmapped,
index_verbose+pd.Timedelta(minutes=1), dtypes=dtypes_radiation_verbose)
out, metadata = sodapro.read_cams(testfile_radiation_verbose,
integrated=True, label='right',
map_variables=False)
assert_frame_equal(out, expected, check_less_precise=True)


def test_read_cams_metadata():
_, metadata = sodapro.read_cams(testfile_mcclear_monthly, integrated=False)
assert metadata['Time reference'] == 'Universal time (UT)'
Expand All @@ -193,7 +180,7 @@ def test_read_cams_metadata():
assert metadata['longitude'] == 12.5251
assert metadata['altitude'] == 39.0
assert metadata['radiation_unit'] == 'W/m^2'
assert metadata['time_step'] == '1M'
assert metadata['time_step'] == '1MS'


@pytest.mark.parametrize('testfile,index,columns,values,dtypes,identifier', [
Expand Down Expand Up @@ -224,7 +211,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes,
email='[email protected]',
identifier=identifier,
altitude=80,
time_step='1M',
time_step='1MS',
verbose=False,
integrated=False)
expected = generate_expected_dataframe(values, columns, index, dtypes)
Expand All @@ -240,7 +227,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes,
email='[email protected]',
identifier=identifier,
altitude=80,
time_step='1M',
time_step='1MS',
verbose=True)


Expand Down
16 changes: 10 additions & 6 deletions pvlib/tests/test_clearsky.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,9 @@ def test_lookup_linke_turbidity_nointerp():


def test_lookup_linke_turbidity_months():
times = pd.date_range(start='2014-04-01', end='2014-07-01',
freq='1M', tz='America/Phoenix')
times = pd.date_range(start='2014-05-01', end='2014-07-01',
freq='1MS', tz='America/Phoenix',
) - pd.Timedelta(days=1)
expected = pd.Series(
np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times
)
Expand All @@ -234,8 +235,9 @@ def test_lookup_linke_turbidity_months():


def test_lookup_linke_turbidity_months_leapyear():
times = pd.date_range(start='2016-04-01', end='2016-07-01',
freq='1M', tz='America/Phoenix')
times = pd.date_range(start='2016-05-01', end='2016-07-01',
freq='1MS', tz='America/Phoenix',
) - pd.Timedelta(days=1)
expected = pd.Series(
np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times
)
Expand All @@ -245,14 +247,16 @@ def test_lookup_linke_turbidity_months_leapyear():

def test_lookup_linke_turbidity_nointerp_months():
times = pd.date_range(start='2014-04-10', end='2014-07-10',
freq='1M', tz='America/Phoenix')
freq='1MS', tz='America/Phoenix',
) - pd.Timedelta(days=1)
expected = pd.Series(np.array([2.85, 2.95, 3.]), index=times)
out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875,
interp_turbidity=False)
assert_series_equal(expected, out)
# changing the dates shouldn't matter if interp=False
times = pd.date_range(start='2014-04-05', end='2014-07-05',
freq='1M', tz='America/Phoenix')
freq='1MS', tz='America/Phoenix',
) - pd.Timedelta(days=1)
out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875,
interp_turbidity=False)
assert_series_equal(expected, out)
Expand Down
Loading