From 05fc1d14b88f4c472e5965c62093ee22508090b3 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 17 Oct 2024 01:11:46 +0200 Subject: [PATCH 1/5] Replace freq '1M' with '1ME' --- pvlib/iotools/bsrn.py | 2 +- pvlib/iotools/srml.py | 2 +- pvlib/tests/iotools/test_sodapro.py | 2 +- pvlib/tests/test_clearsky.py | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pvlib/iotools/bsrn.py b/pvlib/iotools/bsrn.py index 43fdbe919f..fa6ef9b81e 100644 --- a/pvlib/iotools/bsrn.py +++ b/pvlib/iotools/bsrn.py @@ -160,7 +160,7 @@ def get_bsrn(station, start, end, username, password, # Generate list files to download based on start/end (SSSMMYY.dat.gz) filenames = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M')\ + start, end.replace(day=1) + pd.DateOffset(months=1), freq='1ME')\ .strftime(f"{station}%m%y.dat.gz").tolist() # Create FTP connection diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py index 728c3a7093..8f0becc970 100644 --- a/pvlib/iotools/srml.py +++ b/pvlib/iotools/srml.py @@ -237,7 +237,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True, # Generate list of months months = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M') + start, end.replace(day=1) + pd.DateOffset(months=1), freq='1ME') months_str = months.strftime('%y%m') # Generate list of filenames diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 83983d2b68..9fddaf7766 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -19,7 +19,7 @@ index_verbose = pd.date_range('2020-06-01 12', periods=4, freq='1min', tz='UTC') -index_monthly = pd.date_range('2020-01-01', periods=4, freq='1M') +index_monthly = pd.date_range('2020-01-01', periods=4, freq='1ME') dtypes_mcclear_verbose = [ diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index 23d39a5205..ea590fc699 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -225,7 +225,7 @@ def test_lookup_linke_turbidity_nointerp(): def test_lookup_linke_turbidity_months(): times = pd.date_range(start='2014-04-01', end='2014-07-01', - freq='1M', tz='America/Phoenix') + freq='1ME', tz='America/Phoenix') expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -235,7 +235,7 @@ def test_lookup_linke_turbidity_months(): def test_lookup_linke_turbidity_months_leapyear(): times = pd.date_range(start='2016-04-01', end='2016-07-01', - freq='1M', tz='America/Phoenix') + freq='1ME', tz='America/Phoenix') expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -245,14 +245,14 @@ def test_lookup_linke_turbidity_months_leapyear(): def test_lookup_linke_turbidity_nointerp_months(): times = pd.date_range(start='2014-04-10', end='2014-07-10', - freq='1M', tz='America/Phoenix') + freq='1ME', tz='America/Phoenix') expected = pd.Series(np.array([2.85, 2.95, 3.]), index=times) out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out) # changing the dates shouldn't matter if interp=False times = pd.date_range(start='2014-04-05', end='2014-07-05', - freq='1M', tz='America/Phoenix') + freq='1ME', tz='America/Phoenix') out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out) From 1242b48837417287a3057a7aafd77235f485a34e Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 17 Oct 2024 01:21:55 +0200 Subject: [PATCH 2/5] Switch ME to MS --- pvlib/iotools/bsrn.py | 2 +- pvlib/iotools/srml.py | 2 +- pvlib/tests/iotools/test_sodapro.py | 2 +- pvlib/tests/test_clearsky.py | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pvlib/iotools/bsrn.py b/pvlib/iotools/bsrn.py index fa6ef9b81e..21d69400d9 100644 --- a/pvlib/iotools/bsrn.py +++ b/pvlib/iotools/bsrn.py @@ -160,7 +160,7 @@ def get_bsrn(station, start, end, username, password, # Generate list files to download based on start/end (SSSMMYY.dat.gz) filenames = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1ME')\ + start, end.replace(day=1) + pd.DateOffset(months=1), freq='1MS')\ .strftime(f"{station}%m%y.dat.gz").tolist() # Create FTP connection diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py index 8f0becc970..0b53548b72 100644 --- a/pvlib/iotools/srml.py +++ b/pvlib/iotools/srml.py @@ -237,7 +237,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True, # Generate list of months months = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1ME') + start, end.replace(day=1) + pd.DateOffset(months=1), freq='1MS') months_str = months.strftime('%y%m') # Generate list of filenames diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 9fddaf7766..120854bbc4 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -19,7 +19,7 @@ index_verbose = pd.date_range('2020-06-01 12', periods=4, freq='1min', tz='UTC') -index_monthly = pd.date_range('2020-01-01', periods=4, freq='1ME') +index_monthly = pd.date_range('2020-01-01', periods=4, freq='1MS') dtypes_mcclear_verbose = [ diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index ea590fc699..d4eba1781e 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -225,7 +225,7 @@ def test_lookup_linke_turbidity_nointerp(): def test_lookup_linke_turbidity_months(): times = pd.date_range(start='2014-04-01', end='2014-07-01', - freq='1ME', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix') expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -235,7 +235,7 @@ def test_lookup_linke_turbidity_months(): def test_lookup_linke_turbidity_months_leapyear(): times = pd.date_range(start='2016-04-01', end='2016-07-01', - freq='1ME', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix') expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -245,14 +245,14 @@ def test_lookup_linke_turbidity_months_leapyear(): def test_lookup_linke_turbidity_nointerp_months(): times = pd.date_range(start='2014-04-10', end='2014-07-10', - freq='1ME', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix') expected = pd.Series(np.array([2.85, 2.95, 3.]), index=times) out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out) # changing the dates shouldn't matter if interp=False times = pd.date_range(start='2014-04-05', end='2014-07-05', - freq='1ME', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix') out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out) From 2228d8bd4698fbd80354c36d124f2a69bfc8a800 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:16:47 +0200 Subject: [PATCH 3/5] Modify turbidity tess --- pvlib/tests/test_clearsky.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pvlib/tests/test_clearsky.py b/pvlib/tests/test_clearsky.py index d4eba1781e..d737e385c1 100644 --- a/pvlib/tests/test_clearsky.py +++ b/pvlib/tests/test_clearsky.py @@ -224,8 +224,9 @@ def test_lookup_linke_turbidity_nointerp(): def test_lookup_linke_turbidity_months(): - times = pd.date_range(start='2014-04-01', end='2014-07-01', - freq='1MS', tz='America/Phoenix') + times = pd.date_range(start='2014-05-01', end='2014-07-01', + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -234,8 +235,9 @@ def test_lookup_linke_turbidity_months(): def test_lookup_linke_turbidity_months_leapyear(): - times = pd.date_range(start='2016-04-01', end='2016-07-01', - freq='1MS', tz='America/Phoenix') + times = pd.date_range(start='2016-05-01', end='2016-07-01', + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) expected = pd.Series( np.array([2.89918032787, 2.97540983607, 3.19672131148]), index=times ) @@ -245,14 +247,16 @@ def test_lookup_linke_turbidity_months_leapyear(): def test_lookup_linke_turbidity_nointerp_months(): times = pd.date_range(start='2014-04-10', end='2014-07-10', - freq='1MS', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) expected = pd.Series(np.array([2.85, 2.95, 3.]), index=times) out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out) # changing the dates shouldn't matter if interp=False times = pd.date_range(start='2014-04-05', end='2014-07-05', - freq='1MS', tz='America/Phoenix') + freq='1MS', tz='America/Phoenix', + ) - pd.Timedelta(days=1) out = clearsky.lookup_linke_turbidity(times, 32.125, -110.875, interp_turbidity=False) assert_series_equal(expected, out) From bfe8a5c49b32375f7be56f5a9ebc2fab22f77f40 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:22:41 +0200 Subject: [PATCH 4/5] Correct date_range in srml and bsrn --- pvlib/iotools/bsrn.py | 3 +-- pvlib/iotools/srml.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/bsrn.py b/pvlib/iotools/bsrn.py index 21d69400d9..ffe09191d5 100644 --- a/pvlib/iotools/bsrn.py +++ b/pvlib/iotools/bsrn.py @@ -159,8 +159,7 @@ def get_bsrn(station, start, end, username, password, end = pd.to_datetime(end) # Generate list files to download based on start/end (SSSMMYY.dat.gz) - filenames = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1MS')\ + filenames = pd.date_range(start, end, freq='1MS')\ .strftime(f"{station}%m%y.dat.gz").tolist() # Create FTP connection diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py index 0b53548b72..3bcb1f09e9 100644 --- a/pvlib/iotools/srml.py +++ b/pvlib/iotools/srml.py @@ -236,8 +236,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True, end = pd.to_datetime(end) # Generate list of months - months = pd.date_range( - start, end.replace(day=1) + pd.DateOffset(months=1), freq='1MS') + months = pd.date_range(start, end, freq='1MS') months_str = months.strftime('%y%m') # Generate list of filenames From cbf9224459e8d2232de2bd403fa17eda868e213d Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:23:02 +0200 Subject: [PATCH 5/5] Remove label option for sodapro functions --- pvlib/iotools/sodapro.py | 54 ++++++++++++----------------- pvlib/tests/iotools/test_sodapro.py | 19 ++-------- 2 files changed, 25 insertions(+), 48 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index b9922af4b8..21cda88ea7 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -32,7 +32,7 @@ # Dictionary mapping time steps to CAMS time step format TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', - '1d': 'P01D', '1M': 'P01M'} + '1d': 'P01D', '1MS': 'P01M'} TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} @@ -40,12 +40,12 @@ '0 year 0 month 0 day 0 h 15 min 0 s': '15min', # noqa '0 year 0 month 0 day 1 h 0 min 0 s': '1h', '0 year 0 month 1 day 0 h 0 min 0 s': '1d', - '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} + '0 year 1 month 0 day 0 h 0 min 0 s': '1MS'} def get_cams(latitude, longitude, start, end, email, identifier='mcclear', altitude=None, time_step='1h', time_ref='UT', verbose=False, - integrated=False, label=None, map_variables=True, + integrated=False, map_variables=True, server=URL, timeout=30): """Retrieve irradiance and clear-sky time series from CAMS. @@ -79,7 +79,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', altitude: float, optional Altitude in meters. If not specified, then the altitude is determined from the NASA SRTM database - time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' + time_step: str, {'1min', '15min', '1h', '1d', '1MS'}, default: '1h' Time step of the time series, either 1 minute, 15 minute, hourly, daily, or monthly. time_ref: str, {'UT', 'TST'}, default: 'UT' @@ -90,9 +90,6 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label : {'right', 'left'}, optional - Which bin edge label to label time-step with. The default is 'left' for - all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the DataFrame to pvlib variable names where applicable. See variable :const:`VARIABLE_MAP`. @@ -120,7 +117,7 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', ======================== ====== ========================================= **Mapped field names are returned when the map_variables argument is True** --------------------------------------------------------------------------- - Observation period str Beginning/end of time period + Observation period str Start of time period TOA, ghi_extra float Horizontal radiation at top of atmosphere Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal @@ -231,12 +228,12 @@ def get_cams(latitude, longitude, start, end, email, identifier='mcclear', # Successful requests returns a csv data file else: fbuf = io.StringIO(res.content.decode('utf-8')) - data, metadata = parse_cams(fbuf, integrated=integrated, label=label, + data, metadata = parse_cams(fbuf, integrated=integrated, map_variables=map_variables) return data, metadata -def parse_cams(fbuf, integrated=False, label=None, map_variables=True): +def parse_cams(fbuf, integrated=False, map_variables=True): """ Parse a file-like buffer with data in the format of a CAMS Radiation or McClear file. The CAMS solar radiation services are described in [1]_. @@ -248,9 +245,6 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label : {'right', 'left'}, optional - Which bin edge label to label time-step with. The default is 'left' for - all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable :const:`VARIABLE_MAP`. @@ -262,6 +256,10 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): metadata: dict Metadata available in the file. + Notes + ----- + The index timestamps correspond to the start/left of the interval. + See Also -------- pvlib.iotools.read_cams, pvlib.iotools.get_cams @@ -301,26 +299,17 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): obs_period = data['Observation period'].str.split('/') # Set index as the start observation time (left) and localize to UTC - if (label == 'left') | ((label is None) & (time_step != '1M')): - data.index = pd.to_datetime(obs_period.str[0], utc=True) - # Set index as the stop observation time (right) and localize to UTC - # default label for monthly data is 'right' following Pandas' convention - elif (label == 'right') | ((label is None) & (time_step == '1M')): - data.index = pd.to_datetime(obs_period.str[1], utc=True) - - # For time_steps '1d' and '1M', drop timezone and round to nearest midnight - if (time_step == '1d') | (time_step == '1M'): + data.index = pd.to_datetime(obs_period.str[0], utc=True) + + # For time_steps '1d' and '1MS' drop timezone and round to nearest midnight + if (time_step == '1d') | time_step.endswith('MS'): # noqa data.index = pd.DatetimeIndex(data.index.date) - # For monthly data with 'right' label, the index should be the last - # date of the month and not the first date of the following month - if (time_step == '1M') & (label != 'left'): - data.index = data.index - pd.Timedelta(days=1) if not integrated: # Convert radiation values from Wh/m2 to W/m2 integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns] - if time_step == '1M': + if time_step.endswith('MS'): time_delta = (pd.to_datetime(obs_period.str[1]) - pd.to_datetime(obs_period.str[0])) hours = time_delta.dt.total_seconds()/60/60 @@ -336,7 +325,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): return data, metadata -def read_cams(filename, integrated=False, label=None, map_variables=True): +def read_cams(filename, integrated=False, map_variables=True): """ Read a CAMS Radiation or McClear file into a pandas DataFrame. @@ -349,9 +338,6 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label : {'right', 'left}, optional - Which bin edge label to label time-step with. The default is 'left' for - all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable :const:`VARIABLE_MAP`. @@ -368,11 +354,15 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): -------- pvlib.iotools.parse_cams, pvlib.iotools.get_cams + Notes + ----- + The index timestamps correspond to the start/left of the interval. + References ---------- .. [1] `CAMS solar radiation documentation `_ """ with open(str(filename), 'r') as fbuf: - content = parse_cams(fbuf, integrated, label, map_variables) + content = parse_cams(fbuf, integrated, map_variables) return content diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 120854bbc4..cc8b682977 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -172,19 +172,6 @@ def test_read_cams(testfile, index, columns, values, dtypes): assert_frame_equal(out, expected, check_less_precise=True) -def test_read_cams_integrated_unmapped_label(): - # Default label is 'left' for 1 minute time resolution, hence 1 minute is - # added for label='right' - expected = generate_expected_dataframe( - values_radiation_verbose_integrated, - columns_radiation_verbose_unmapped, - index_verbose+pd.Timedelta(minutes=1), dtypes=dtypes_radiation_verbose) - out, metadata = sodapro.read_cams(testfile_radiation_verbose, - integrated=True, label='right', - map_variables=False) - assert_frame_equal(out, expected, check_less_precise=True) - - def test_read_cams_metadata(): _, metadata = sodapro.read_cams(testfile_mcclear_monthly, integrated=False) assert metadata['Time reference'] == 'Universal time (UT)' @@ -193,7 +180,7 @@ def test_read_cams_metadata(): assert metadata['longitude'] == 12.5251 assert metadata['altitude'] == 39.0 assert metadata['radiation_unit'] == 'W/m^2' - assert metadata['time_step'] == '1M' + assert metadata['time_step'] == '1MS' @pytest.mark.parametrize('testfile,index,columns,values,dtypes,identifier', [ @@ -224,7 +211,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, email='pvlib-admin@googlegroups.com', identifier=identifier, altitude=80, - time_step='1M', + time_step='1MS', verbose=False, integrated=False) expected = generate_expected_dataframe(values, columns, index, dtypes) @@ -240,7 +227,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, email='pvlib-admin@googlegroups.com', identifier=identifier, altitude=80, - time_step='1M', + time_step='1MS', verbose=True)