Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial support for setting custom et band names from_scene_et_actual #19

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Initial support for setting custom et band names from_scene_et_actual #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b210d70
Initial support for setting custom et band names in interpolate.from_…
cgmorton Oct 25, 2022
64f66be
Misc formatting
cgmorton Jun 7, 2023
49b487c
Merge branch 'main' into interpolate-et-set-band-name
cgmorton Jun 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions openet/core/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,12 @@ def landsat_c1_toa_cloud_mask(
.Or(qa_img.rightShift(7).bitwiseAnd(3).gte(shadow_confidence))
if snow_flag:
cloud_mask = cloud_mask.Or(
qa_img.rightShift(9).bitwiseAnd(3).gte(snow_confidence))
qa_img.rightShift(9).bitwiseAnd(3).gte(snow_confidence)
)
if cirrus_flag:
cloud_mask = cloud_mask.Or(
qa_img.rightShift(11).bitwiseAnd(3).gte(cirrus_confidence))
qa_img.rightShift(11).bitwiseAnd(3).gte(cirrus_confidence)
)

# Set cloudy pixels to 0 and clear to 1
return cloud_mask.Not()
Expand Down
113 changes: 63 additions & 50 deletions openet/core/interpolate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,8 @@ def _linear(image):
# The closest image in time should be on "top"
# CGM - Is the previous collection already sorted?
# prev_qm_img = prev_qm_coll.mosaic()
prev_qm_img = prev_qm_coll.sort('system:time_start', True)\
.mosaic()
next_qm_img = next_qm_coll.sort('system:time_start', False)\
.mosaic()
prev_qm_img = prev_qm_coll.sort('system:time_start', True).mosaic()
next_qm_img = next_qm_coll.sort('system:time_start', False).mosaic()

# DEADBEEF - It might be easier to interpolate all bands instead of
# separating the value and time bands
Expand Down Expand Up @@ -247,8 +245,7 @@ def _linear(image):


# @deprecated
def aggregate_daily(image_coll, start_date=None, end_date=None,
agg_type='mean'):
def aggregate_daily(image_coll, start_date=None, end_date=None, agg_type='mean'):
return aggregate_to_daily(image_coll, start_date, end_date, agg_type)


Expand Down Expand Up @@ -401,8 +398,7 @@ def from_scene_et_fraction(scene_coll, start_date, end_date, variables,
if t_interval.lower() not in ['daily', 'monthly', 'annual', 'custom']:
raise ValueError('unsupported t_interval: {}'.format(t_interval))
elif interp_method.lower() not in ['linear']:
raise ValueError('unsupported interp_method: {}'.format(
interp_method))
raise ValueError('unsupported interp_method: {}'.format(interp_method))

if ((type(interp_days) is str or type(interp_days) is float) and
utils.is_number(interp_days)):
Expand Down Expand Up @@ -472,8 +468,8 @@ def from_scene_et_fraction(scene_coll, start_date, end_date, variables,
# # raise ValueError('et_reference_resample was not set')

if type(et_reference_source) is str:
# Assume a string source is an single image collection ID
# not an list of collection IDs or ee.ImageCollection
# Assume a string source is a single image collection ID
# not a list of collection IDs or ee.ImageCollection
daily_et_ref_coll = ee.ImageCollection(et_reference_source) \
.filterDate(start_date, end_date) \
.select([et_reference_band], ['et_reference'])
Expand All @@ -488,7 +484,7 @@ def from_scene_et_fraction(scene_coll, start_date, end_date, variables,

# Scale reference ET images (if necessary)
# CGM - Resampling is not working correctly so not including for now
if (et_reference_factor and et_reference_factor != 1):
if et_reference_factor and et_reference_factor != 1:
def et_reference_adjust(input_img):
return input_img.multiply(et_reference_factor) \
.copyProperties(input_img) \
Expand Down Expand Up @@ -521,8 +517,9 @@ def et_reference_adjust(input_img):
# For count, compute the composite/mosaic image for the mask band only
if 'count' in variables:
aggregate_coll = aggregate_to_daily(
image_coll = scene_coll.select(['mask']),
start_date=start_date, end_date=end_date)
image_coll=scene_coll.select(['mask']),
start_date=start_date, end_date=end_date
)

# The following is needed because the aggregate collection can be
# empty if there are no scenes in the target date range but there
Expand All @@ -531,20 +528,22 @@ def et_reference_adjust(input_img):
# bands will be which causes a non-homogeneous image collection.
aggregate_coll = aggregate_coll.merge(
ee.Image.constant(0).rename(['mask'])
.set({'system:time_start': ee.Date(start_date).millis()}))
.set({'system:time_start': ee.Date(start_date).millis()})
)

# Interpolate to a daily time step
daily_coll = daily(
target_coll=daily_et_ref_coll,
source_coll=scene_coll.select(interp_vars),
interp_method=interp_method, interp_days=interp_days,
interp_method=interp_method,
interp_days=interp_days,
use_joins=use_joins,
compute_product=False,
)

# The interpolate.daily() function can/will return the product of
# the source and target image named as "{source_band}_1".
# The problem with this approach is that is will drop any other bands
# The problem with this approach is that it will drop any other bands
# that are being interpolated (such as the ndvi).
# daily_coll = daily_coll.select(['et_fraction_1'], ['et'])

Expand All @@ -554,8 +553,7 @@ def et_reference_adjust(input_img):
# if 'et' in variables or 'et_fraction' in variables:
def compute_et(img):
"""This function assumes ETr and ETf are present"""
et_img = img.select(['et_fraction']) \
.multiply(img.select(['et_reference']))
et_img = img.select(['et_fraction']).multiply(img.select(['et_reference']))
return img.addBands(et_img.double().rename('et'))
daily_coll = daily_coll.map(compute_et)

Expand Down Expand Up @@ -598,8 +596,8 @@ def aggregate_image(agg_start_date, agg_end_date, date_format):
if 'et_fraction' in variables:
# Compute average et fraction over the aggregation period
image_list.append(
et_img.divide(et_reference_img).rename(
['et_fraction']).float())
et_img.divide(et_reference_img).rename(['et_fraction']).float()
)
if 'ndvi' in variables:
# Compute average ndvi over the aggregation period
ndvi_img = daily_coll \
Expand Down Expand Up @@ -629,7 +627,8 @@ def agg_daily(daily_img):
return aggregate_image(
agg_start_date=agg_start_date,
agg_end_date=ee.Date(agg_start_date).advance(1, 'day'),
date_format='YYYYMMdd')
date_format='YYYYMMdd',
)

return ee.ImageCollection(daily_coll.map(agg_daily))

Expand All @@ -647,7 +646,8 @@ def agg_monthly(agg_start_date):
return aggregate_image(
agg_start_date=agg_start_date,
agg_end_date=ee.Date(agg_start_date).advance(1, 'month'),
date_format='YYYYMM')
date_format='YYYYMM',
)

return ee.ImageCollection(month_list.map(agg_monthly))

Expand All @@ -664,27 +664,32 @@ def agg_annual(agg_start_date):
return aggregate_image(
agg_start_date=agg_start_date,
agg_end_date=ee.Date(agg_start_date).advance(1, 'year'),
date_format='YYYY')
date_format='YYYY',
)

return ee.ImageCollection(year_list.map(agg_annual))

elif t_interval.lower() == 'custom':
# Returning an ImageCollection to be consistent
return ee.ImageCollection(aggregate_image(
agg_start_date=start_date, agg_end_date=end_date,
date_format='YYYYMMdd'))
agg_start_date=start_date,
agg_end_date=end_date,
date_format='YYYYMMdd',
))


def from_scene_et_actual(scene_coll, start_date, end_date, variables,
interp_args, model_args, t_interval,
use_joins=False,
use_joins=False, et_band_name='et',
):
"""Interpolate from a precomputed collection of Landsat actual ET scenes

Parameters
----------
scene_coll : ee.ImageCollection
Non-daily 'et' images that will be interpolated.
Non-daily actual ET source images that will be interpolated from.
If the band name is different than 'et', it will need to be set using
the 'et_band_name' function parameter.
start_date : str
ISO format start date.
end_date : str
Expand Down Expand Up @@ -719,6 +724,8 @@ def from_scene_et_actual(scene_coll, start_date, end_date, variables,
If True, use joins to link the target and source collections.
If False, the source collection will be filtered for each target image.
This parameter is passed through to interpolate.daily().
et_band_name : str, optional
The actual ET band name. The default is "et".

Returns
-------
Expand Down Expand Up @@ -760,8 +767,7 @@ def from_scene_et_actual(scene_coll, start_date, end_date, variables,
if t_interval.lower() not in ['daily', 'monthly', 'annual', 'custom']:
raise ValueError('unsupported t_interval: {}'.format(t_interval))
elif interp_method.lower() not in ['linear']:
raise ValueError('unsupported interp_method: {}'.format(
interp_method))
raise ValueError('unsupported interp_method: {}'.format(interp_method))

if ((type(interp_days) is str or type(interp_days) is float) and
utils.is_number(interp_days)):
Expand Down Expand Up @@ -825,16 +831,16 @@ def from_scene_et_actual(scene_coll, start_date, end_date, variables,
# 'et_reference_resample was not set, default to nearest')
# # raise ValueError('et_reference_resample was not set')

# Assume a string source is an single image collection ID
# not an list of collection IDs or ee.ImageCollection
# Assume a string source is a single image collection ID
# not a list of collection IDs or ee.ImageCollection
daily_et_ref_coll_id = model_args['et_reference_source']
daily_et_ref_coll = ee.ImageCollection(daily_et_ref_coll_id) \
.filterDate(start_date, end_date) \
.select([model_args['et_reference_band']], ['et_reference'])

# Scale reference ET images (if necessary)
# CGM - Resampling is not working correctly so not including for now
if (et_reference_factor and et_reference_factor != 1):
if et_reference_factor and et_reference_factor != 1:
def et_reference_adjust(input_img):
return input_img.multiply(et_reference_factor) \
.copyProperties(input_img) \
Expand Down Expand Up @@ -869,35 +875,37 @@ def et_reference_adjust(input_img):
.filterDate(interp_start_date, interp_end_date) \
.select([interp_args['interp_band']])

interp_vars = ['et'] + ['mask', 'time']

# For count, compute the composite/mosaic image for the mask band only
if 'count' in variables:
aggregate_coll = aggregate_to_daily(
image_coll=scene_coll.select(['mask']),
start_date=start_date, end_date=end_date)
start_date=start_date,
end_date=end_date,
)
# The following is needed because the aggregate collection can be
# empty if there are no scenes in the target date range but there
# are scenes in the interpolation date range.
# Without this the count image will not be built but the other
# bands will be which causes a non-homogeneous image collection.
aggregate_coll = aggregate_coll.merge(
ee.Image.constant(0).rename(['mask'])
.set({'system:time_start': ee.Date(start_date).millis()}))
.set({'system:time_start': ee.Date(start_date).millis()})
)

# It might be more efficient to join the target collection to the scenes
def normalize_et(img):
img_date = ee.Date(img.get('system:time_start')) \
.update(hour=0, minute=0, second=0)
img_date = ee.Date(img_date.millis().divide(1000).floor().multiply(1000))
target_img = ee.Image(daily_target_coll \
.filterDate(img_date, img_date.advance(1, 'day')).first())
target_img = ee.Image(
daily_target_coll.filterDate(img_date, img_date.advance(1, 'day')).first()
)

# CGM - This is causing weird artifacts in the output images
# if interp_args['interp_resample'].lower() in ['bilinear', 'bicubic']:
# target_img = target_img.resample(interp_args['interp_resample'])

et_norm_img = img.select(['et']).divide(target_img).rename(['et_norm'])
et_norm_img = img.select([et_band_name]).divide(target_img).rename(['et_norm'])

# Clamp the normalized ET image (et_fraction)
if 'et_fraction_max' in interp_args.keys():
Expand All @@ -910,13 +918,12 @@ def normalize_et(img):
# float(interp_args['et_fraction_min']),
# float(interp_args['et_fraction_max']))

return img.addBands([
et_norm_img.double(), target_img.rename(['norm'])])
return img.addBands([et_norm_img.double(), target_img.rename(['norm'])])

# The time band is always needed for interpolation
scene_coll = scene_coll \
.filterDate(interp_start_date, interp_end_date) \
.select(interp_vars) \
.select([et_band_name, 'mask', 'time']) \
.map(normalize_et)

# # Join the target (normalization) image to the scene images
Expand All @@ -939,7 +946,8 @@ def normalize_et(img):
daily_coll = daily(
target_coll=daily_target_coll.filterDate(start_date, end_date),
source_coll=scene_coll.select(['et_norm', 'time']),
interp_method=interp_method, interp_days=interp_days,
interp_method=interp_method,
interp_days=interp_days,
use_joins=use_joins,
compute_product=True,
)
Expand Down Expand Up @@ -1001,8 +1009,8 @@ def aggregate_image(agg_start_date, agg_end_date, date_format):
if 'et_fraction' in variables:
# Compute average et fraction over the aggregation period
image_list.append(
et_img.divide(et_reference_img)
.rename(['et_fraction']).float())
et_img.divide(et_reference_img).rename(['et_fraction']).float()
)
# if 'ndvi' in variables:
# # Compute average ndvi over the aggregation period
# ndvi_img = daily_coll \
Expand Down Expand Up @@ -1032,7 +1040,8 @@ def agg_daily(daily_img):
return aggregate_image(
agg_start_date=agg_start_date,
agg_end_date=ee.Date(agg_start_date).advance(1, 'day'),
date_format='YYYYMMdd')
date_format='YYYYMMdd',
)

return ee.ImageCollection(daily_coll.map(agg_daily))

Expand All @@ -1050,7 +1059,8 @@ def agg_monthly(agg_start_date):
return aggregate_image(
agg_start_date=agg_start_date,
agg_end_date=ee.Date(agg_start_date).advance(1, 'month'),
date_format='YYYYMM')
date_format='YYYYMM',
)

return ee.ImageCollection(month_list.map(agg_monthly))

Expand All @@ -1067,15 +1077,18 @@ def agg_annual(agg_start_date):
return aggregate_image(
agg_start_date=agg_start_date,
agg_end_date=ee.Date(agg_start_date).advance(1, 'year'),
date_format='YYYY')
date_format='YYYY',
)

return ee.ImageCollection(year_list.map(agg_annual))

elif t_interval.lower() == 'custom':
# Returning an ImageCollection to be consistent
return ee.ImageCollection(aggregate_image(
agg_start_date=start_date, agg_end_date=end_date,
date_format='YYYYMMdd'))
agg_start_date=start_date,
agg_end_date=end_date,
date_format='YYYYMMdd',
))


# @deprecated
Expand Down
Loading