Add get_surfrad iotools function (#2836)

* generate get_surfrad

* function documentation

* fix linter

* linter v2

* add tests

* fix test linter

* minor error

* linter again

* Update docs/sphinx/source/whatsnew/v0.15.3.rst

Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com>

* Update pvlib/iotools/surfrad.py

Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com>

* Update tests/iotools/test_surfrad.py

Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com>

* Update test_surfrad.py

* Update pvlib/iotools/surfrad.py

Co-authored-by: Kevin Anderson <kevin.anderso@gmail.com>

---------

Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com>
Co-authored-by: Kevin Anderson <kevin.anderso@gmail.com>

Author: 3 people

docs/sphinx/source/reference/iotools.rst

@@ -176,6 +176,7 @@ A solar radiation network in the USA, run by NOAA.
    :toctree: generated/
 
    iotools.read_surfrad
+   iotools.get_surfrad
 
 
 MIDC

docs/sphinx/source/whatsnew/v0.15.3.rst

@@ -50,6 +50,9 @@ Enhancements
   :py:func:`~pvlib.iotools.get_nsrdb_psm4_polar_tmy`.
   (:issue:`2639`, :pull:`2807`)
 * Ensure all timezones are available in all OSs. (:issue:`2795`, :pull:`2809`)
+* Add :py:func:`pvlib.iotools.get_surfrad` for retrieving irradiance data
+  from NOAA's SURFRAD network.
+  (:issue:`1155`, :pull:`2836`)
 * Implement the ANTS-2D bifacial irradiance model in
   :py:func:`pvlib.bifacial.ants2d.get_irradiance`. (:pull:`2740`)
 * Add parameters ``g0`` and ``g1`` to allow segmented ground surfaces in
@@ -109,3 +112,4 @@ Contributors
 * Jason Lun Leung (:ghuser:`jason-rpkt`)
 * Leonardo Scappatura (:ghuser:`Leonard013`)
 * Carolina Crespo (:ghuser:`cbcrespo`)
+* Ioannis Sifnaios (:ghuser:`IoannisSifnaios`)

pvlib/iotools/__init__.py

@@ -3,6 +3,7 @@
 from pvlib.iotools.srml import read_srml  # noqa: F401
 from pvlib.iotools.srml import get_srml  # noqa: F401
 from pvlib.iotools.surfrad import read_surfrad  # noqa: F401
+from pvlib.iotools.surfrad import get_surfrad  # noqa: F401
 from pvlib.iotools.midc import read_midc  # noqa: F401
 from pvlib.iotools.midc import read_midc_raw_data_from_nrel  # noqa: F401
 from pvlib.iotools.crn import read_crn  # noqa: F401

pvlib/iotools/surfrad.py

@@ -5,6 +5,9 @@
 from urllib.request import urlopen, Request
 import pandas as pd
 import numpy as np
+import warnings
+import urllib.error
+
 
 SURFRAD_COLUMNS = [
     'year', 'jday', 'month', 'day', 'hour', 'minute', 'dt', 'zen',
@@ -181,3 +184,99 @@ def _format_index(data):
     data.index = index
     data = data.tz_localize('UTC')
     return data
+
+
+def get_surfrad(station, start, end, map_variables=True,
+                url="https://gml.noaa.gov/aftp/data/radiation/surfrad/"):
+    """
+    Request data from NOAA SURFRAD and read it into a DataFrame.
+
+    The SURFRAD network is described in [1]_. The README files are located in
+    the station directories in the SURFRAD data archives [2]_. In addition to
+    the FTP server, the SURFRAD files are also available via HTTP access [3]_.
+
+    Data is returned for complete days, including ``start`` and ``end``.
+
+    Parameters
+    ----------
+    station : str
+        Three-letter SURFRAD station abbreviation.
+    start : datetime-like
+        First day of the requested period.
+    end : datetime-like
+        Last day of the requested period.
+    map_variables : bool, default True
+        Passed through to :py:func:`~pvlib.iotools.read_surfrad`:
+        whether to rename columns to pvlib variable names
+        (e.g. ``'dw_solar'`` -> ``'ghi'``).
+    url : str, default 'https://gml.noaa.gov/aftp/data/radiation/surfrad/'
+        Base URL of the SURFRAD archive.
+
+    Returns
+    -------
+    data : pd.DataFrame
+        Dataframe with data from SURFRAD.
+    meta : dict
+        Metadata.
+
+    See Also
+    --------
+    pvlib.iotools.read_surfrad
+
+    Notes
+    -----
+    Missing days (e.g. before a station's operational start date, or gaps
+    in the archive) are skipped with a warning rather than raising an error.
+
+    Examples
+    --------
+    >>> data, meta = pvlib.iotools.get_surfrad(
+    ...     station='bon', start='2020-01-01', end='2020-01-31')
+
+    References
+    ----------
+    .. [1] NOAA Earth System Research Laboratory Surface Radiation Budget
+       Network
+       `SURFRAD Homepage <https://www.esrl.noaa.gov/gmd/grad/surfrad/>`_
+    .. [2] NOAA SURFRAD Data Archive
+       `SURFRAD Archive <ftp://aftp.cmdl.noaa.gov/data/radiation/surfrad/>`_
+    .. [3] `NOAA SURFRAD HTTP Index
+       <https://gml.noaa.gov/aftp/data/radiation/surfrad/>`_
+    """
+    start = pd.to_datetime(start)
+    end = pd.to_datetime(end)
+
+    dates = pd.date_range(start.floor('D'), end, freq='D')
+    station = station.lower()
+
+    filenames = [
+        f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat"  # noqa: E231,E501
+        for d in dates
+    ]
+
+    dfs = []
+    file_metadata = None
+    for f in filenames:
+        try:
+            dfi, file_metadata = read_surfrad(url + f,
+                                              map_variables=VARIABLE_MAP)
+            dfs.append(dfi)
+
+        except urllib.error.HTTPError:
+            warnings.warn(f"The following file was not found: {f}")
+
+    if not dfs:
+        raise ValueError(
+            f"No data retrieved for station '{station}' between "
+            f"{start.date()} and {end.date()}. Check the station code "
+            "and date range."
+        )
+
+    data = pd.concat(dfs, axis='rows')
+    meta = {
+        'station': station,
+        'filenames': filenames,
+        # all files should share metadata, so just take it from the last one
+        **file_metadata,
+    }
+    return data, meta

tests/iotools/test_surfrad.py

@@ -2,7 +2,12 @@
 import pytest
 
 from pvlib.iotools import surfrad
-from tests.conftest import TESTS_DATA_DIR, RERUNS, RERUNS_DELAY
+from tests.conftest import (
+    TESTS_DATA_DIR,
+    assert_frame_equal,
+    RERUNS,
+    RERUNS_DELAY,
+)
 
 testfile = TESTS_DATA_DIR / 'surfrad-slv16001.dat'
 network_testfile = ('ftp://aftp.cmdl.noaa.gov/data/radiation/surfrad/'
@@ -73,3 +78,41 @@ def test_read_surfrad_metadata():
                 'tz': 'UTC'}
     _, metadata = surfrad.read_surfrad(testfile)
     assert metadata == expected
+
+
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_surfrad():
+    df, meta = surfrad.get_surfrad('slv', '2016-01-01', '2016-01-01')
+
+    assert meta['station'] == 'slv'
+    assert isinstance(meta['filenames'], list)
+
+    assert len(df) == 1440
+    assert df.index[0] == pd.to_datetime('2016-01-01 00:00+00:00')
+    assert df.index[-1] == pd.to_datetime('2016-01-01 23:59+00:00')
+
+    expected, _ = surfrad.read_surfrad(testfile)
+    assert_frame_equal(df, expected)
+
+
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_surfrad_missing_day():
+    # SURFRAD's Alamosa station data begins 2014-07-28 (slv14209.dat), so
+    # requesting the day before that will raise a warning
+    message = 'The following file was not found: slv/2014/slv14208.dat'
+    with pytest.warns(UserWarning, match=message):
+        df, meta = surfrad.get_surfrad('slv', '2014-07-27', '2014-07-28')
+
+    # but the data for 2014-07-28 is still returned
+    assert not df.empty
+
+
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_surfrad_no_data():
+    message = "No data retrieved for station 'xxx'"
+    with pytest.warns(UserWarning):
+        with pytest.raises(ValueError, match=message):
+            surfrad.get_surfrad('xxx', '2016-01-01', '2016-01-01')