Updated interpolate from_scene_et_fraction function to match core app…

…roach

The time and mask bands are now computed inside the interpolate call instead of needing to be set on the scene collection ahead of time.

.github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: Run Tests
+name: Tests
 
 on:
   push:

openet/sims/image.py

@@ -110,7 +110,7 @@ def __init__(
         # Build date properties from the system:time_start
         self._date = ee.Date(self._time_start)
         self._year = ee.Number(self._date.get('year'))
-        self._start_date = ee.Date(utils.date_to_time_0utc(self._date))
+        self._start_date = ee.Date(utils.date_0utc(self._date).millis())
         self._end_date = self._start_date.advance(1, 'day')
         self._doy = self._date.getRelative('day', 'year').add(1).int()
 
@@ -345,7 +345,7 @@ def time(self):
         """
         return (
             self.mask
-            .double().multiply(0).add(utils.date_to_time_0utc(self._date))
+            .double().multiply(0).add(utils.date_0utc(self._date).millis())
             .rename(['time']).set(self._properties)
         )
 

openet/sims/interpolate.py

@@ -41,6 +41,19 @@ def from_scene_et_fraction(
         interp_days : int, str, optional
             Number of extra days before the start date and after the end date
             to include in the interpolation calculation. The default is 32.
+        use_joins : bool, optional
+            If True, use joins to link the target and source collections.
+            If False, the source collection will be filtered for each target image.
+            This parameter is passed through to interpolate.daily().
+        et_reference_source : str
+            Reference ET collection ID.
+        et_reference_band : str
+            Reference ET band name.
+        et_reference_factor : float, None, optional
+            Reference ET scaling factor.  The default is 1.0 which is
+            equivalent to no scaling.
+        et_reference_resample : {'nearest', 'bilinear', 'bicubic', None}, optional
+            Reference ET resampling.  The default is 'nearest'.
         estimate_soil_evaporation: bool
             Compute daily Ke values by simulating water balance in evaporable
             zone. Default is False.
@@ -49,9 +62,7 @@ def from_scene_et_fraction(
             soil water state.  This value will be added to the interp_days when
             setting the interpolation start date.  Default is 0 days.
     model_args : dict
-        Parameters from the MODEL section of the INI file.  The reference
-        source and parameters will need to be set here if computing
-        reference ET or actual ET.
+        Parameters from the MODEL section of the INI file.
     t_interval : {'daily', 'monthly', 'annual', 'custom'}
         Time interval over which to interpolate and aggregate values
         The 'custom' interval will aggregate all days within the start and end
@@ -68,6 +79,10 @@ def from_scene_et_fraction(
     ------
     ValueError
 
+    Notes
+    -----
+    This function assumes that "mask" and "time" bands are not in the scene collection.
+
     """
     # Check whether to compute daily Ke
     if 'estimate_soil_evaporation' in interp_args.keys():
@@ -153,7 +168,7 @@ def from_scene_et_fraction(
     # Supporting reading the parameters from both the interp_args and model_args dictionaries
     # Check interp_args then model_args, but eventually drop support for reading from model_args
     # Assume that if source and band are both set, the parameters in that section should be used
-    if 'et_reference_source' in interp_args.keys() and 'et_reference_band' in interp_args.keys():
+    if ('et_reference_source' in interp_args.keys()) and ('et_reference_band' in interp_args.keys()):
         et_reference_source = interp_args['et_reference_source']
         et_reference_band = interp_args['et_reference_band']
         if not et_reference_source or not et_reference_band:
@@ -177,7 +192,7 @@ def from_scene_et_fraction(
             et_reference_resample = 'nearest'
             logging.debug('et_reference_resample was not set, default to nearest')
 
-    elif 'et_reference_source' in model_args.keys() and 'et_reference_band' in model_args.keys():
+    elif ('et_reference_source' in model_args.keys()) and ('et_reference_band' in model_args.keys()):
         et_reference_source = model_args['et_reference_source']
         et_reference_band = model_args['et_reference_band']
         if not et_reference_source or not et_reference_band:
@@ -206,7 +221,7 @@ def from_scene_et_fraction(
 
     # Check if collection already has et_reference provided
     #   if not, get it from the collection
-    if type(et_reference_source) is str and et_reference_source.lower() == 'provided':
+    if (type(et_reference_source) is str) and (et_reference_source.lower() == 'provided'):
         daily_et_ref_coll = scene_coll.map(lambda x: x.select('et_reference'))
     elif type(et_reference_source) is str:
         # Assume a string source is a single image collection ID
@@ -235,31 +250,71 @@ def et_reference_adjust(input_img):
                 .copyProperties(input_img)
                 .set({'system:time_start': input_img.get('system:time_start')})
             )
-
         daily_et_ref_coll = daily_et_ref_coll.map(et_reference_adjust)
 
     # Initialize variable list to only variables that can be interpolated
     interp_vars = list(set(_interp_vars) & set(variables))
 
     # To return ET, the ETf must be interpolated
-    if 'et' in variables and 'et_fraction' not in interp_vars:
-        interp_vars.append('et_fraction')
+    if ('et' in variables) and ('et_fraction' not in interp_vars):
+        interp_vars = interp_vars + ['et_fraction']
 
     # With the current interpolate.daily() function,
     #   something has to be interpolated in order to return et_reference
-    if 'et_reference' in variables and 'et_fraction' not in interp_vars:
-        interp_vars.append('et_fraction')
-
-    # The time band is always needed for interpolation
-    interp_vars.append('time')
+    if ('et_reference' in variables) and ('et_fraction' not in interp_vars):
+        interp_vars = interp_vars + ['et_fraction']
 
     # The NDVI band is always needed for the soil water balance
-    if estimate_soil_evaporation and 'ndvi' not in interp_vars:
-        interp_vars.append('ndvi')
+    if estimate_soil_evaporation and ('ndvi' not in interp_vars):
+        interp_vars = interp_vars + ['ndvi']
+
+    # TODO: Look into implementing et_fraction clamping here
+    #   (similar to et_actual below)
+
+    def interpolate_prep(img):
+        """Prep WRS2 scene images for interpolation
+
+        "Unscale" the images using the "scale_factor" property and convert to double.
+        Add a mask and time band to each image in the scene_coll since
+            interpolator is assuming time and mask bands exist.
+        The interpolation could be modified to get the mask from the
+            time band instead of setting it here.
+        The time image must be the 0 UTC time
+
+        """
+        mask_img = (
+            img.select(['et_fraction']).multiply(0).add(1).updateMask(1).uint8()
+            .rename(['mask'])
+        )
+        time_img = (
+            img.select(['et_fraction']).double().multiply(0)
+            .add(utils.date_0utc(ee.Date(img.get('system:time_start'))).millis())
+            .rename(['time'])
+        )
+
+        # Set the default scale factor to 1 if the image does not have the property
+        scale_factor = (
+            ee.Dictionary({'scale_factor': img.get('scale_factor')})
+            .combine({'scale_factor': 1.0}, overwrite=False)
+        )
+
+        return (
+            img.select(interp_vars)
+            .double().multiply(ee.Number(scale_factor.get('scale_factor')))
+            .addBands([mask_img, time_img])
+            .set({
+                'system:time_start': ee.Number(img.get('system:time_start')),
+                'system:index': ee.String(img.get('system:index')),
+            })
+        )
 
     # Filter scene collection to the interpolation range
-    # This probably isn't needed since scene_coll was built to this range
-    scene_coll = scene_coll.filterDate(interp_start_date, interp_end_date)
+    #   This probably isn't needed since scene_coll was built to this range
+    # Then add the time and mask bands needed for interpolation
+    scene_coll = ee.ImageCollection(
+        scene_coll.filterDate(interp_start_date, interp_end_date)
+        .map(interpolate_prep)
+    )
 
     # For count, compute the composite/mosaic image for the mask band only
     if 'count' in variables:
@@ -284,7 +339,7 @@ def et_reference_adjust(input_img):
     #   but is returning the target (ETo) band
     daily_coll = openet.core.interpolate.daily(
         target_coll=daily_et_ref_coll,
-        source_coll=scene_coll.select(interp_vars),
+        source_coll=scene_coll.select(interp_vars + ['time']),
         interp_method=interp_method,
         interp_days=interp_days,
         use_joins=use_joins,
@@ -345,8 +400,10 @@ def aggregate_image(agg_start_date, agg_end_date, date_format):
 
         if ('et_reference' in variables) or ('et_fraction' in variables):
             et_reference_img = (
-                daily_et_ref_coll.filterDate(agg_start_date, agg_end_date)
-                .select(['et_reference']).sum()
+                daily_et_ref_coll
+                .filterDate(agg_start_date, agg_end_date)
+                .select(['et_reference'])
+                .sum()
             )
             if et_reference_resample and (et_reference_resample in ['bilinear', 'bicubic']):
                 et_reference_img = (

openet/sims/tests/test_a_utils.py

@@ -82,8 +82,8 @@ def test_point_coll_value(image_id, image_date, xy, scale, expected, tol):
 
     ]
 )
-def test_date_to_time_0utc(input, expected):
-    assert utils.getinfo(utils.date_to_time_0utc(ee.Date(input))) == expected
+def test_date_0utc(input, expected):
+    assert utils.getinfo(utils.date_0utc(ee.Date(input)).millis()) == expected
 
 
 @pytest.mark.parametrize(

openet/sims/tests/test_d_interpolate.py

@@ -32,34 +32,53 @@ def scene_coll(variables, etf=[0.4, 0.4, 0.4], et=[5, 5, 5], ndvi=[0.6, 0.6, 0.6
     )
     mask = img.add(1).updateMask(1).uint8()
 
-    # The "date" is used for the time band since it needs to be the 0 UTC time
+    # # The "date" is used for the time band since it needs to be the 0 UTC time
+    # date1 = ee.Number(ee.Date.fromYMD(2017, 7, 8).millis())
+    # date2 = ee.Number(ee.Date.fromYMD(2017, 7, 16).millis())
+    # date3 = ee.Number(ee.Date.fromYMD(2017, 7, 24).millis())
+
     # The "time" is advanced to match the typical Landsat overpass time
-    date1 = ee.Number(ee.Date.fromYMD(2017, 7, 8).millis())
-    date2 = ee.Number(ee.Date.fromYMD(2017, 7, 16).millis())
-    date3 = ee.Number(ee.Date.fromYMD(2017, 7, 24).millis())
     time1 = ee.Number(ee.Date.fromYMD(2017, 7, 8).advance(18, 'hours').millis())
     time2 = ee.Number(ee.Date.fromYMD(2017, 7, 16).advance(18, 'hours').millis())
     time3 = ee.Number(ee.Date.fromYMD(2017, 7, 24).advance(18, 'hours').millis())
 
-    # Mask and time bands currently get added on to the scene collection
-    #   and images are unscaled just before interpolating in the export tool
-    scene_coll = ee.ImageCollection([
-        ee.Image([img.add(etf[0]), img.add(et[0]), img.add(ndvi[0]), img.add(date1), mask])
-            .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask'])
-            .set({'system:index': 'LE07_044033_20170708', 'system:time_start': time1}),
-        ee.Image([img.add(etf[1]), img.add(et[1]), img.add(ndvi[1]), img.add(date2), mask])
-            .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask'])
-            .set({'system:index': 'LC08_044033_20170716', 'system:time_start': time2}),
-        ee.Image([img.add(etf[2]), img.add(et[2]), img.add(ndvi[2]), img.add(date3), mask])
-            .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask'])
-            .set({'system:index': 'LE07_044033_20170724', 'system:time_start': time3}),
+    # TODO: Add code to convert et, et_fraction, and ndvi to lists if they
+    #   are set as a single value
+
+    # Don't add mask or time band to scene collection
+    # since they are now added in the interpolation calls
+    scene_coll = ee.ImageCollection.fromImages([
+        ee.Image([img.add(etf[0]), img.add(et[0]), img.add(ndvi[0])])
+        .rename(['et_fraction', 'et', 'ndvi'])
+        .set({'system:index': 'LE07_044033_20170708', 'system:time_start': time1}),
+        ee.Image([img.add(etf[1]), img.add(et[1]), img.add(ndvi[1])])
+        .rename(['et_fraction', 'et', 'ndvi'])
+        .set({'system:index': 'LC08_044033_20170716', 'system:time_start': time2}),
+        ee.Image([img.add(etf[2]), img.add(et[2]), img.add(ndvi[2])])
+        .rename(['et_fraction', 'et', 'ndvi'])
+        .set({'system:index': 'LE07_044033_20170724', 'system:time_start': time3}),
     ])
+
+    # # Mask and time bands currently get added on to the scene collection
+    # #   and images are unscaled just before interpolating in the export tool
+    # scene_coll = ee.ImageCollection([
+    #     ee.Image([img.add(etf[0]), img.add(et[0]), img.add(ndvi[0]), img.add(date1), mask])
+    #     .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask'])
+    #     .set({'system:index': 'LE07_044033_20170708', 'system:time_start': time1}),
+    #     ee.Image([img.add(etf[1]), img.add(et[1]), img.add(ndvi[1]), img.add(date2), mask])
+    #     .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask'])
+    #     .set({'system:index': 'LC08_044033_20170716', 'system:time_start': time2}),
+    #     ee.Image([img.add(etf[2]), img.add(et[2]), img.add(ndvi[2]), img.add(date3), mask])
+    #     .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask'])
+    #     .set({'system:index': 'LE07_044033_20170724', 'system:time_start': time3}),
+    # ])
+
     return scene_coll.select(variables)
 
 
 def test_from_scene_et_fraction_t_interval_daily_values(tol=0.0001):
     output_coll = interpolate.from_scene_et_fraction(
-        scene_coll(['et_fraction', 'ndvi', 'time', 'mask'], ndvi=[0.2, 0.4, 0.6]),
+        scene_coll(['et_fraction', 'ndvi'], ndvi=[0.2, 0.4, 0.6]),
         start_date='2017-07-01', end_date='2017-08-01',
         variables=['et', 'et_reference', 'et_fraction', 'ndvi'],
         interp_args={'interp_method': 'linear', 'interp_days': 32},
@@ -90,7 +109,7 @@ def test_from_scene_et_fraction_t_interval_daily_values(tol=0.0001):
 
 def test_from_scene_et_fraction_t_interval_monthly_values(tol=0.0001):
     output_coll = interpolate.from_scene_et_fraction(
-        scene_coll(['et_fraction', 'ndvi', 'time', 'mask']),
+        scene_coll(['et_fraction', 'ndvi']),
         start_date='2017-07-01', end_date='2017-08-01',
         variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'],
         interp_args={'interp_method': 'linear', 'interp_days': 32},
@@ -112,7 +131,7 @@ def test_from_scene_et_fraction_t_interval_monthly_values(tol=0.0001):
 
 def test_from_scene_et_fraction_t_interval_custom_values(tol=0.0001):
     output_coll = interpolate.from_scene_et_fraction(
-        scene_coll(['et_fraction', 'ndvi', 'time', 'mask']),
+        scene_coll(['et_fraction', 'ndvi']),
         start_date='2017-07-01', end_date='2017-08-01',
         variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'],
         interp_args={'interp_method': 'linear', 'interp_days': 32},
@@ -134,7 +153,7 @@ def test_from_scene_et_fraction_t_interval_custom_values(tol=0.0001):
 
 def test_from_scene_et_fraction_t_interval_monthly_et_reference_factor(tol=0.0001):
     output_coll = interpolate.from_scene_et_fraction(
-        scene_coll(['et_fraction', 'ndvi', 'time', 'mask']),
+        scene_coll(['et_fraction', 'ndvi']),
         start_date='2017-07-01', end_date='2017-08-01',
         variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'],
         interp_args={'interp_method': 'linear', 'interp_days': 32},
@@ -156,7 +175,7 @@ def test_from_scene_et_fraction_t_interval_monthly_et_reference_factor(tol=0.000
 
 def test_from_scene_et_fraction_t_interval_monthly_et_reference_resample(tol=0.0001):
     output_coll = interpolate.from_scene_et_fraction(
-        scene_coll(['et_fraction', 'ndvi', 'time', 'mask']),
+        scene_coll(['et_fraction', 'ndvi']),
         start_date='2017-07-01', end_date='2017-08-01',
         variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'],
         interp_args={'interp_method': 'linear', 'interp_days': 32},
@@ -183,7 +202,7 @@ def test_from_scene_et_fraction_t_interval_monthly_et_reference_resample(tol=0.0
 def test_from_scene_et_fraction_t_interval_monthly_interp_args_et_reference(tol=0.0001):
     # Check that the et_reference parameters can be set through the interp_args
     output_coll = interpolate.from_scene_et_fraction(
-        scene_coll(['et_fraction', 'ndvi', 'time', 'mask']),
+        scene_coll(['et_fraction', 'ndvi']),
         start_date='2017-07-01', end_date='2017-08-01',
         variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'],
         interp_args={'interp_method': 'linear', 'interp_days': 32,
@@ -208,7 +227,7 @@ def test_from_scene_et_fraction_t_interval_bad_value():
     # Function should raise a ValueError if t_interval is not supported
     with pytest.raises(ValueError):
         interpolate.from_scene_et_fraction(
-            scene_coll(['et', 'time', 'mask']),
+            scene_coll(['et']),
             start_date='2017-07-01', end_date='2017-08-01', variables=['et'],
             interp_args={'interp_method': 'linear', 'interp_days': 32},
             model_args={'et_reference_source': 'IDAHO_EPSCOR/GRIDMET',
@@ -223,7 +242,7 @@ def test_from_scene_et_fraction_t_interval_no_value():
     # Function should raise an Exception if t_interval is not set
     with pytest.raises(TypeError):
         interpolate.from_scene_et_fraction(
-            scene_coll(['et', 'time', 'mask']),
+            scene_coll(['et']),
             start_date='2017-07-01', end_date='2017-08-01',
             variables=['et', 'et_reference', 'et_fraction', 'count'],
             interp_args={'interp_method': 'linear', 'interp_days': 32},

openet/sims/utils.py

@@ -74,22 +74,19 @@ def point_coll_value(coll, xy, scale=1):
     # return pd.DataFrame.from_dict(info_dict)
 
 
-def date_to_time_0utc(date):
-    """Get the 0 UTC time_start for a date
+def date_0utc(date):
+    """Get the 0 UTC date for a date
 
     Parameters
     ----------
     date : ee.Date
 
     Returns
     -------
-    ee.Number
+    ee.Date
 
     """
-    return ee.Date.fromYMD(date.get('year'), date.get('month'), date.get('day')).millis()
-    # Extra operations are needed since update() does not set milliseconds to 0.
-    # return date.update(hour=0, minute=0, second=0).millis()\
-    #     .divide(1000).floor().multiply(1000)
+    return ee.Date.fromYMD(date.get('year'), date.get('month'), date.get('day'))
 
 
 def is_number(x):

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openet-sims"
-version = "0.2.1"
+version = "0.2.2"
 authors = [
     { name = "Alberto Guzman", email = "[email protected]" },
 ]
@@ -16,8 +16,8 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
-    "earthengine-api >= 0.1.364",
-    "openet-core >= 0.4.0",
+    "earthengine-api >= 0.1.392",
+    "openet-core >= 0.5.0",
 ]
 
 [project.urls]