From 519dc6b0701b26f7f6f7cbceef09779dd78fe40d Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Sat, 25 May 2024 09:21:37 -0700 Subject: [PATCH] Updated interpolate from_scene_et_fraction function to match core approach The time and mask bands are now computed inside the interpolate call instead of needing to be set on the scene collection ahead of time. --- .github/workflows/tests.yml | 2 +- openet/sims/image.py | 4 +- openet/sims/interpolate.py | 99 +++++++++++++++++++------ openet/sims/tests/test_a_utils.py | 4 +- openet/sims/tests/test_d_interpolate.py | 67 +++++++++++------ openet/sims/utils.py | 11 +-- pyproject.toml | 6 +- 7 files changed, 133 insertions(+), 60 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b5fb73..332f4bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: Run Tests +name: Tests on: push: diff --git a/openet/sims/image.py b/openet/sims/image.py index 7335b2c..048bcd0 100644 --- a/openet/sims/image.py +++ b/openet/sims/image.py @@ -110,7 +110,7 @@ def __init__( # Build date properties from the system:time_start self._date = ee.Date(self._time_start) self._year = ee.Number(self._date.get('year')) - self._start_date = ee.Date(utils.date_to_time_0utc(self._date)) + self._start_date = ee.Date(utils.date_0utc(self._date).millis()) self._end_date = self._start_date.advance(1, 'day') self._doy = self._date.getRelative('day', 'year').add(1).int() @@ -345,7 +345,7 @@ def time(self): """ return ( self.mask - .double().multiply(0).add(utils.date_to_time_0utc(self._date)) + .double().multiply(0).add(utils.date_0utc(self._date).millis()) .rename(['time']).set(self._properties) ) diff --git a/openet/sims/interpolate.py b/openet/sims/interpolate.py index 43345ee..7ff6c05 100644 --- a/openet/sims/interpolate.py +++ b/openet/sims/interpolate.py @@ -41,6 +41,19 @@ def from_scene_et_fraction( interp_days : int, str, optional Number of extra days before the start date and after the end date to include in the interpolation calculation. The default is 32. + use_joins : bool, optional + If True, use joins to link the target and source collections. + If False, the source collection will be filtered for each target image. + This parameter is passed through to interpolate.daily(). + et_reference_source : str + Reference ET collection ID. + et_reference_band : str + Reference ET band name. + et_reference_factor : float, None, optional + Reference ET scaling factor. The default is 1.0 which is + equivalent to no scaling. + et_reference_resample : {'nearest', 'bilinear', 'bicubic', None}, optional + Reference ET resampling. The default is 'nearest'. estimate_soil_evaporation: bool Compute daily Ke values by simulating water balance in evaporable zone. Default is False. @@ -49,9 +62,7 @@ def from_scene_et_fraction( soil water state. This value will be added to the interp_days when setting the interpolation start date. Default is 0 days. model_args : dict - Parameters from the MODEL section of the INI file. The reference - source and parameters will need to be set here if computing - reference ET or actual ET. + Parameters from the MODEL section of the INI file. t_interval : {'daily', 'monthly', 'annual', 'custom'} Time interval over which to interpolate and aggregate values The 'custom' interval will aggregate all days within the start and end @@ -68,6 +79,10 @@ def from_scene_et_fraction( ------ ValueError + Notes + ----- + This function assumes that "mask" and "time" bands are not in the scene collection. + """ # Check whether to compute daily Ke if 'estimate_soil_evaporation' in interp_args.keys(): @@ -153,7 +168,7 @@ def from_scene_et_fraction( # Supporting reading the parameters from both the interp_args and model_args dictionaries # Check interp_args then model_args, but eventually drop support for reading from model_args # Assume that if source and band are both set, the parameters in that section should be used - if 'et_reference_source' in interp_args.keys() and 'et_reference_band' in interp_args.keys(): + if ('et_reference_source' in interp_args.keys()) and ('et_reference_band' in interp_args.keys()): et_reference_source = interp_args['et_reference_source'] et_reference_band = interp_args['et_reference_band'] if not et_reference_source or not et_reference_band: @@ -177,7 +192,7 @@ def from_scene_et_fraction( et_reference_resample = 'nearest' logging.debug('et_reference_resample was not set, default to nearest') - elif 'et_reference_source' in model_args.keys() and 'et_reference_band' in model_args.keys(): + elif ('et_reference_source' in model_args.keys()) and ('et_reference_band' in model_args.keys()): et_reference_source = model_args['et_reference_source'] et_reference_band = model_args['et_reference_band'] if not et_reference_source or not et_reference_band: @@ -206,7 +221,7 @@ def from_scene_et_fraction( # Check if collection already has et_reference provided # if not, get it from the collection - if type(et_reference_source) is str and et_reference_source.lower() == 'provided': + if (type(et_reference_source) is str) and (et_reference_source.lower() == 'provided'): daily_et_ref_coll = scene_coll.map(lambda x: x.select('et_reference')) elif type(et_reference_source) is str: # Assume a string source is a single image collection ID @@ -235,31 +250,71 @@ def et_reference_adjust(input_img): .copyProperties(input_img) .set({'system:time_start': input_img.get('system:time_start')}) ) - daily_et_ref_coll = daily_et_ref_coll.map(et_reference_adjust) # Initialize variable list to only variables that can be interpolated interp_vars = list(set(_interp_vars) & set(variables)) # To return ET, the ETf must be interpolated - if 'et' in variables and 'et_fraction' not in interp_vars: - interp_vars.append('et_fraction') + if ('et' in variables) and ('et_fraction' not in interp_vars): + interp_vars = interp_vars + ['et_fraction'] # With the current interpolate.daily() function, # something has to be interpolated in order to return et_reference - if 'et_reference' in variables and 'et_fraction' not in interp_vars: - interp_vars.append('et_fraction') - - # The time band is always needed for interpolation - interp_vars.append('time') + if ('et_reference' in variables) and ('et_fraction' not in interp_vars): + interp_vars = interp_vars + ['et_fraction'] # The NDVI band is always needed for the soil water balance - if estimate_soil_evaporation and 'ndvi' not in interp_vars: - interp_vars.append('ndvi') + if estimate_soil_evaporation and ('ndvi' not in interp_vars): + interp_vars = interp_vars + ['ndvi'] + + # TODO: Look into implementing et_fraction clamping here + # (similar to et_actual below) + + def interpolate_prep(img): + """Prep WRS2 scene images for interpolation + + "Unscale" the images using the "scale_factor" property and convert to double. + Add a mask and time band to each image in the scene_coll since + interpolator is assuming time and mask bands exist. + The interpolation could be modified to get the mask from the + time band instead of setting it here. + The time image must be the 0 UTC time + + """ + mask_img = ( + img.select(['et_fraction']).multiply(0).add(1).updateMask(1).uint8() + .rename(['mask']) + ) + time_img = ( + img.select(['et_fraction']).double().multiply(0) + .add(utils.date_0utc(ee.Date(img.get('system:time_start'))).millis()) + .rename(['time']) + ) + + # Set the default scale factor to 1 if the image does not have the property + scale_factor = ( + ee.Dictionary({'scale_factor': img.get('scale_factor')}) + .combine({'scale_factor': 1.0}, overwrite=False) + ) + + return ( + img.select(interp_vars) + .double().multiply(ee.Number(scale_factor.get('scale_factor'))) + .addBands([mask_img, time_img]) + .set({ + 'system:time_start': ee.Number(img.get('system:time_start')), + 'system:index': ee.String(img.get('system:index')), + }) + ) # Filter scene collection to the interpolation range - # This probably isn't needed since scene_coll was built to this range - scene_coll = scene_coll.filterDate(interp_start_date, interp_end_date) + # This probably isn't needed since scene_coll was built to this range + # Then add the time and mask bands needed for interpolation + scene_coll = ee.ImageCollection( + scene_coll.filterDate(interp_start_date, interp_end_date) + .map(interpolate_prep) + ) # For count, compute the composite/mosaic image for the mask band only if 'count' in variables: @@ -284,7 +339,7 @@ def et_reference_adjust(input_img): # but is returning the target (ETo) band daily_coll = openet.core.interpolate.daily( target_coll=daily_et_ref_coll, - source_coll=scene_coll.select(interp_vars), + source_coll=scene_coll.select(interp_vars + ['time']), interp_method=interp_method, interp_days=interp_days, use_joins=use_joins, @@ -345,8 +400,10 @@ def aggregate_image(agg_start_date, agg_end_date, date_format): if ('et_reference' in variables) or ('et_fraction' in variables): et_reference_img = ( - daily_et_ref_coll.filterDate(agg_start_date, agg_end_date) - .select(['et_reference']).sum() + daily_et_ref_coll + .filterDate(agg_start_date, agg_end_date) + .select(['et_reference']) + .sum() ) if et_reference_resample and (et_reference_resample in ['bilinear', 'bicubic']): et_reference_img = ( diff --git a/openet/sims/tests/test_a_utils.py b/openet/sims/tests/test_a_utils.py index ff2a79a..ad823d4 100644 --- a/openet/sims/tests/test_a_utils.py +++ b/openet/sims/tests/test_a_utils.py @@ -82,8 +82,8 @@ def test_point_coll_value(image_id, image_date, xy, scale, expected, tol): ] ) -def test_date_to_time_0utc(input, expected): - assert utils.getinfo(utils.date_to_time_0utc(ee.Date(input))) == expected +def test_date_0utc(input, expected): + assert utils.getinfo(utils.date_0utc(ee.Date(input)).millis()) == expected @pytest.mark.parametrize( diff --git a/openet/sims/tests/test_d_interpolate.py b/openet/sims/tests/test_d_interpolate.py index 0497706..c0a25fa 100644 --- a/openet/sims/tests/test_d_interpolate.py +++ b/openet/sims/tests/test_d_interpolate.py @@ -32,34 +32,53 @@ def scene_coll(variables, etf=[0.4, 0.4, 0.4], et=[5, 5, 5], ndvi=[0.6, 0.6, 0.6 ) mask = img.add(1).updateMask(1).uint8() - # The "date" is used for the time band since it needs to be the 0 UTC time + # # The "date" is used for the time band since it needs to be the 0 UTC time + # date1 = ee.Number(ee.Date.fromYMD(2017, 7, 8).millis()) + # date2 = ee.Number(ee.Date.fromYMD(2017, 7, 16).millis()) + # date3 = ee.Number(ee.Date.fromYMD(2017, 7, 24).millis()) + # The "time" is advanced to match the typical Landsat overpass time - date1 = ee.Number(ee.Date.fromYMD(2017, 7, 8).millis()) - date2 = ee.Number(ee.Date.fromYMD(2017, 7, 16).millis()) - date3 = ee.Number(ee.Date.fromYMD(2017, 7, 24).millis()) time1 = ee.Number(ee.Date.fromYMD(2017, 7, 8).advance(18, 'hours').millis()) time2 = ee.Number(ee.Date.fromYMD(2017, 7, 16).advance(18, 'hours').millis()) time3 = ee.Number(ee.Date.fromYMD(2017, 7, 24).advance(18, 'hours').millis()) - # Mask and time bands currently get added on to the scene collection - # and images are unscaled just before interpolating in the export tool - scene_coll = ee.ImageCollection([ - ee.Image([img.add(etf[0]), img.add(et[0]), img.add(ndvi[0]), img.add(date1), mask]) - .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) - .set({'system:index': 'LE07_044033_20170708', 'system:time_start': time1}), - ee.Image([img.add(etf[1]), img.add(et[1]), img.add(ndvi[1]), img.add(date2), mask]) - .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) - .set({'system:index': 'LC08_044033_20170716', 'system:time_start': time2}), - ee.Image([img.add(etf[2]), img.add(et[2]), img.add(ndvi[2]), img.add(date3), mask]) - .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) - .set({'system:index': 'LE07_044033_20170724', 'system:time_start': time3}), + # TODO: Add code to convert et, et_fraction, and ndvi to lists if they + # are set as a single value + + # Don't add mask or time band to scene collection + # since they are now added in the interpolation calls + scene_coll = ee.ImageCollection.fromImages([ + ee.Image([img.add(etf[0]), img.add(et[0]), img.add(ndvi[0])]) + .rename(['et_fraction', 'et', 'ndvi']) + .set({'system:index': 'LE07_044033_20170708', 'system:time_start': time1}), + ee.Image([img.add(etf[1]), img.add(et[1]), img.add(ndvi[1])]) + .rename(['et_fraction', 'et', 'ndvi']) + .set({'system:index': 'LC08_044033_20170716', 'system:time_start': time2}), + ee.Image([img.add(etf[2]), img.add(et[2]), img.add(ndvi[2])]) + .rename(['et_fraction', 'et', 'ndvi']) + .set({'system:index': 'LE07_044033_20170724', 'system:time_start': time3}), ]) + + # # Mask and time bands currently get added on to the scene collection + # # and images are unscaled just before interpolating in the export tool + # scene_coll = ee.ImageCollection([ + # ee.Image([img.add(etf[0]), img.add(et[0]), img.add(ndvi[0]), img.add(date1), mask]) + # .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) + # .set({'system:index': 'LE07_044033_20170708', 'system:time_start': time1}), + # ee.Image([img.add(etf[1]), img.add(et[1]), img.add(ndvi[1]), img.add(date2), mask]) + # .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) + # .set({'system:index': 'LC08_044033_20170716', 'system:time_start': time2}), + # ee.Image([img.add(etf[2]), img.add(et[2]), img.add(ndvi[2]), img.add(date3), mask]) + # .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) + # .set({'system:index': 'LE07_044033_20170724', 'system:time_start': time3}), + # ]) + return scene_coll.select(variables) def test_from_scene_et_fraction_t_interval_daily_values(tol=0.0001): output_coll = interpolate.from_scene_et_fraction( - scene_coll(['et_fraction', 'ndvi', 'time', 'mask'], ndvi=[0.2, 0.4, 0.6]), + scene_coll(['et_fraction', 'ndvi'], ndvi=[0.2, 0.4, 0.6]), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'ndvi'], interp_args={'interp_method': 'linear', 'interp_days': 32}, @@ -90,7 +109,7 @@ def test_from_scene_et_fraction_t_interval_daily_values(tol=0.0001): def test_from_scene_et_fraction_t_interval_monthly_values(tol=0.0001): output_coll = interpolate.from_scene_et_fraction( - scene_coll(['et_fraction', 'ndvi', 'time', 'mask']), + scene_coll(['et_fraction', 'ndvi']), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'], interp_args={'interp_method': 'linear', 'interp_days': 32}, @@ -112,7 +131,7 @@ def test_from_scene_et_fraction_t_interval_monthly_values(tol=0.0001): def test_from_scene_et_fraction_t_interval_custom_values(tol=0.0001): output_coll = interpolate.from_scene_et_fraction( - scene_coll(['et_fraction', 'ndvi', 'time', 'mask']), + scene_coll(['et_fraction', 'ndvi']), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'], interp_args={'interp_method': 'linear', 'interp_days': 32}, @@ -134,7 +153,7 @@ def test_from_scene_et_fraction_t_interval_custom_values(tol=0.0001): def test_from_scene_et_fraction_t_interval_monthly_et_reference_factor(tol=0.0001): output_coll = interpolate.from_scene_et_fraction( - scene_coll(['et_fraction', 'ndvi', 'time', 'mask']), + scene_coll(['et_fraction', 'ndvi']), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'], interp_args={'interp_method': 'linear', 'interp_days': 32}, @@ -156,7 +175,7 @@ def test_from_scene_et_fraction_t_interval_monthly_et_reference_factor(tol=0.000 def test_from_scene_et_fraction_t_interval_monthly_et_reference_resample(tol=0.0001): output_coll = interpolate.from_scene_et_fraction( - scene_coll(['et_fraction', 'ndvi', 'time', 'mask']), + scene_coll(['et_fraction', 'ndvi']), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'], interp_args={'interp_method': 'linear', 'interp_days': 32}, @@ -183,7 +202,7 @@ def test_from_scene_et_fraction_t_interval_monthly_et_reference_resample(tol=0.0 def test_from_scene_et_fraction_t_interval_monthly_interp_args_et_reference(tol=0.0001): # Check that the et_reference parameters can be set through the interp_args output_coll = interpolate.from_scene_et_fraction( - scene_coll(['et_fraction', 'ndvi', 'time', 'mask']), + scene_coll(['et_fraction', 'ndvi']), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'ndvi', 'count'], interp_args={'interp_method': 'linear', 'interp_days': 32, @@ -208,7 +227,7 @@ def test_from_scene_et_fraction_t_interval_bad_value(): # Function should raise a ValueError if t_interval is not supported with pytest.raises(ValueError): interpolate.from_scene_et_fraction( - scene_coll(['et', 'time', 'mask']), + scene_coll(['et']), start_date='2017-07-01', end_date='2017-08-01', variables=['et'], interp_args={'interp_method': 'linear', 'interp_days': 32}, model_args={'et_reference_source': 'IDAHO_EPSCOR/GRIDMET', @@ -223,7 +242,7 @@ def test_from_scene_et_fraction_t_interval_no_value(): # Function should raise an Exception if t_interval is not set with pytest.raises(TypeError): interpolate.from_scene_et_fraction( - scene_coll(['et', 'time', 'mask']), + scene_coll(['et']), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'count'], interp_args={'interp_method': 'linear', 'interp_days': 32}, diff --git a/openet/sims/utils.py b/openet/sims/utils.py index a4250b4..9a13826 100644 --- a/openet/sims/utils.py +++ b/openet/sims/utils.py @@ -74,8 +74,8 @@ def point_coll_value(coll, xy, scale=1): # return pd.DataFrame.from_dict(info_dict) -def date_to_time_0utc(date): - """Get the 0 UTC time_start for a date +def date_0utc(date): + """Get the 0 UTC date for a date Parameters ---------- @@ -83,13 +83,10 @@ def date_to_time_0utc(date): Returns ------- - ee.Number + ee.Date """ - return ee.Date.fromYMD(date.get('year'), date.get('month'), date.get('day')).millis() - # Extra operations are needed since update() does not set milliseconds to 0. - # return date.update(hour=0, minute=0, second=0).millis()\ - # .divide(1000).floor().multiply(1000) + return ee.Date.fromYMD(date.get('year'), date.get('month'), date.get('day')) def is_number(x): diff --git a/pyproject.toml b/pyproject.toml index b0cbfbb..74da9ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openet-sims" -version = "0.2.1" +version = "0.2.2" authors = [ { name = "Alberto Guzman", email = "aguzman@csumb.edu" }, ] @@ -16,8 +16,8 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "earthengine-api >= 0.1.364", - "openet-core >= 0.4.0", + "earthengine-api >= 0.1.392", + "openet-core >= 0.5.0", ] [project.urls]