diff --git a/openet/core/api.py b/openet/core/api.py index 0812869..b3161c2 100644 --- a/openet/core/api.py +++ b/openet/core/api.py @@ -1,4 +1,4 @@ -import sys +# import sys import ee @@ -22,7 +22,7 @@ def collection( t_interval, geometry, **kwargs - ): + ): """Generic OpenET Collection Parameters diff --git a/openet/core/common.py b/openet/core/common.py index 9e52aa9..2c3f7ee 100644 --- a/openet/core/common.py +++ b/openet/core/common.py @@ -1,6 +1,3 @@ -import re -import warnings - import ee @@ -547,4 +544,3 @@ def ged_emis(ged_col): return L8_LST.rename('surface_temperature') # .set('system:time_start', image.get('system:time_start')) - diff --git a/openet/core/ensemble.py b/openet/core/ensemble.py index 1c36c74..7313e8a 100644 --- a/openet/core/ensemble.py +++ b/openet/core/ensemble.py @@ -1,9 +1,7 @@ -import pprint - import ee -from . import utils -# import openet.core.utils as utils +# from . import utils +# # import openet.core.utils as utils model_index = { 'disalexi': 1, diff --git a/openet/core/interpolate.py b/openet/core/interpolate.py index 01f9f3c..c404308 100644 --- a/openet/core/interpolate.py +++ b/openet/core/interpolate.py @@ -2,14 +2,20 @@ import logging import ee -from dateutil.relativedelta import * +from dateutil.relativedelta import relativedelta from . import utils # import openet.core.utils as utils -def daily(target_coll, source_coll, interp_days=32, interp_method='linear', - use_joins=False, compute_product=False): +def daily( + target_coll, + source_coll, + interp_days=32, + interp_method='linear', + use_joins=False, + compute_product=False + ): """Interpolate non-daily source images to a daily target image collection Parameters @@ -252,8 +258,12 @@ def aggregate_daily(image_coll, start_date=None, end_date=None, return aggregate_to_daily(image_coll, start_date, end_date, agg_type) -def aggregate_to_daily(image_coll, start_date=None, end_date=None, - agg_type='mean'): +def aggregate_to_daily( + image_coll, + start_date=None, + end_date=None, + agg_type='mean' + ): """Aggregate images by day without using joins The primary purpose of this function is to join separate Landsat images @@ -288,10 +298,12 @@ def aggregate_to_daily(image_coll, start_date=None, end_date=None, test_coll = image_coll.filterDate(ee.Date(start_date), ee.Date(end_date)) elif start_date: test_coll = image_coll.filter(ee.Filter.greaterThanOrEquals( - 'system:time_start', ee.Date(start_date).millis())) + 'system:time_start', ee.Date(start_date).millis() + )) elif end_date: test_coll = image_coll.filter(ee.Filter.lessThan( - 'system:time_start', ee.Date(end_date).millis())) + 'system:time_start', ee.Date(end_date).millis() + )) else: test_coll = image_coll @@ -321,10 +333,15 @@ def aggregate_func(date_str): return ee.ImageCollection(date_list.map(aggregate_func)) -def from_scene_et_fraction(scene_coll, start_date, end_date, variables, - interp_args, model_args, t_interval, - use_joins=False, - ): +def from_scene_et_fraction( + scene_coll, + start_date, end_date, + variables, + interp_args, + model_args, + t_interval, + use_joins=False, + ): """Interpolate from a precomputed collection of Landsat ET fraction scenes Parameters @@ -521,8 +538,10 @@ def et_reference_adjust(input_img): # For count, compute the composite/mosaic image for the mask band only if 'count' in variables: aggregate_coll = aggregate_to_daily( - image_coll = scene_coll.select(['mask']), - start_date=start_date, end_date=end_date) + image_coll=scene_coll.select(['mask']), + start_date=start_date, + end_date=end_date, + ) # The following is needed because the aggregate collection can be # empty if there are no scenes in the target date range but there @@ -531,13 +550,15 @@ def et_reference_adjust(input_img): # bands will be which causes a non-homogeneous image collection. aggregate_coll = aggregate_coll.merge( ee.Image.constant(0).rename(['mask']) - .set({'system:time_start': ee.Date(start_date).millis()})) + .set({'system:time_start': ee.Date(start_date).millis()}) + ) # Interpolate to a daily time step daily_coll = daily( target_coll=daily_et_ref_coll, source_coll=scene_coll.select(interp_vars), - interp_method=interp_method, interp_days=interp_days, + interp_method=interp_method, + interp_days=interp_days, use_joins=use_joins, compute_product=False, ) @@ -554,8 +575,7 @@ def et_reference_adjust(input_img): # if 'et' in variables or 'et_fraction' in variables: def compute_et(img): """This function assumes ETr and ETf are present""" - et_img = img.select(['et_fraction']) \ - .multiply(img.select(['et_reference'])) + et_img = img.select(['et_fraction']).multiply(img.select(['et_reference'])) return img.addBands(et_img.double().rename('et')) daily_coll = daily_coll.map(compute_et) @@ -598,8 +618,8 @@ def aggregate_image(agg_start_date, agg_end_date, date_format): if 'et_fraction' in variables: # Compute average et fraction over the aggregation period image_list.append( - et_img.divide(et_reference_img).rename( - ['et_fraction']).float()) + et_img.divide(et_reference_img).rename(['et_fraction']).float() + ) if 'ndvi' in variables: # Compute average ndvi over the aggregation period ndvi_img = daily_coll \ @@ -629,7 +649,8 @@ def agg_daily(daily_img): return aggregate_image( agg_start_date=agg_start_date, agg_end_date=ee.Date(agg_start_date).advance(1, 'day'), - date_format='YYYYMMdd') + date_format='YYYYMMdd', + ) return ee.ImageCollection(daily_coll.map(agg_daily)) @@ -647,7 +668,8 @@ def agg_monthly(agg_start_date): return aggregate_image( agg_start_date=agg_start_date, agg_end_date=ee.Date(agg_start_date).advance(1, 'month'), - date_format='YYYYMM') + date_format='YYYYMM', + ) return ee.ImageCollection(month_list.map(agg_monthly)) @@ -664,21 +686,30 @@ def agg_annual(agg_start_date): return aggregate_image( agg_start_date=agg_start_date, agg_end_date=ee.Date(agg_start_date).advance(1, 'year'), - date_format='YYYY') + date_format='YYYY', + ) return ee.ImageCollection(year_list.map(agg_annual)) elif t_interval.lower() == 'custom': # Returning an ImageCollection to be consistent return ee.ImageCollection(aggregate_image( - agg_start_date=start_date, agg_end_date=end_date, - date_format='YYYYMMdd')) + agg_start_date=start_date, + agg_end_date=end_date, + date_format='YYYYMMdd', + )) -def from_scene_et_actual(scene_coll, start_date, end_date, variables, - interp_args, model_args, t_interval, - use_joins=False, - ): +def from_scene_et_actual( + scene_coll, + start_date, + end_date, + variables, + interp_args, + model_args, + t_interval, + use_joins=False, + ): """Interpolate from a precomputed collection of Landsat actual ET scenes Parameters diff --git a/openet/core/tests/test_common.py b/openet/core/tests/test_common.py index 4c99f15..b6537d1 100644 --- a/openet/core/tests/test_common.py +++ b/openet/core/tests/test_common.py @@ -1,4 +1,4 @@ -import pprint +# import pprint import ee import pytest @@ -293,4 +293,4 @@ def test_landsat_c2_sr_lst_correct_values(image_id, xy, expected, tol=0.25): # original = utils.point_image_value(lst_img, xy, scale=30)['ST_B10'] output_img = common.landsat_c2_sr_lst_correct(input_img, ndvi_img) corrected = utils.point_image_value(output_img, xy, scale=30) - assert abs(corrected['surface_temperature'] - expected) <= tol \ No newline at end of file + assert abs(corrected['surface_temperature'] - expected) <= tol diff --git a/openet/core/tests/test_ensemble.py b/openet/core/tests/test_ensemble.py index 7d6b116..ade765c 100644 --- a/openet/core/tests/test_ensemble.py +++ b/openet/core/tests/test_ensemble.py @@ -1,6 +1,6 @@ # import datetime -import logging -import pprint +# import logging +# import pprint import ee import pytest @@ -11,8 +11,6 @@ # logging.basicConfig(level=logging.DEBUG, format='%(message)s') - - # TODO: Write a test to check that the output bandnames # def test_mad_bandname(): # assert False diff --git a/openet/core/tests/test_interpolate.py b/openet/core/tests/test_interpolate.py index bd8ae93..53110f7 100644 --- a/openet/core/tests/test_interpolate.py +++ b/openet/core/tests/test_interpolate.py @@ -1,6 +1,6 @@ import datetime import logging -import pprint +# import pprint import ee import pytest @@ -23,6 +23,7 @@ def tgt_image(tgt_value, tgt_time): 'system:index': datetime.datetime.utcfromtimestamp( tgt_time / 1000.0).strftime('%Y%m%d')}) + def src_images(src_values, src_times): """Build constant source images from values and times""" src_images = [] @@ -82,22 +83,16 @@ def scene_coll(variables, et_fraction=[0.4, 0.4, 0.4], et=[5, 5, 5], # Mask and time bands currently get added on to the scene collection # and images are unscaled just before interpolating in the export tool - scene_coll = ee.ImageCollection([ - ee.Image([img.add(et_fraction[0]), img.add(et[0]), img.add(ndvi[0]), - img.add(date1), mask])\ + scene_coll = ee.ImageCollection.fromImages([ + ee.Image([img.add(et_fraction[0]), img.add(et[0]), img.add(ndvi[0]), img.add(date1), mask]) .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) - .set({'system:index': 'LE07_044033_20170708', - 'system:time_start': time1}), - ee.Image([img.add(et_fraction[1]), img.add(et[1]), img.add(ndvi[1]), - img.add(date2), mask])\ + .set({'system:index': 'LE07_044033_20170708', 'system:time_start': time1}), + ee.Image([img.add(et_fraction[1]), img.add(et[1]), img.add(ndvi[1]), img.add(date2), mask]) .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) - .set({'system:index': 'LC08_044033_20170716', - 'system:time_start': time2}), - ee.Image([img.add(et_fraction[2]), img.add(et[2]), img.add(ndvi[2]), - img.add(date3), mask])\ + .set({'system:index': 'LC08_044033_20170716', 'system:time_start': time2}), + ee.Image([img.add(et_fraction[2]), img.add(et[2]), img.add(ndvi[2]), img.add(date3), mask]) .rename(['et_fraction', 'et', 'ndvi', 'time', 'mask']) - .set({'system:index': 'LE07_044033_20170724', - 'system:time_start': time3}), + .set({'system:index': 'LE07_044033_20170724', 'system:time_start': time3}), ]) return scene_coll.select(variables) @@ -728,7 +723,7 @@ def test_from_scene_et_actual_daily_et_fraction_max(tol=0.0001): def test_from_scene_et_fraction_t_interval_bad_value(): # Function should raise a ValueError if t_interval is not supported with pytest.raises(ValueError): - output_coll = interpolate.from_scene_et_fraction( + interpolate.from_scene_et_fraction( scene_coll(['et', 'time', 'mask']), start_date='2017-07-01', end_date='2017-08-01', variables=['et'], interp_args={'interp_method': 'linear', 'interp_days': 32}, @@ -742,7 +737,7 @@ def test_from_scene_et_fraction_t_interval_bad_value(): def test_from_scene_et_fraction_t_interval_no_value(): # Function should raise an Exception if t_interval is not set with pytest.raises(TypeError): - output_coll = interpolate.from_scene_et_fraction( + interpolate.from_scene_et_fraction( scene_coll(['et', 'time', 'mask']), start_date='2017-07-01', end_date='2017-08-01', variables=['et', 'et_reference', 'et_fraction', 'count'], @@ -756,7 +751,7 @@ def test_from_scene_et_fraction_t_interval_no_value(): def test_from_scene_et_actual_t_interval_bad_value(): # Function should raise a ValueError if t_interval is not supported with pytest.raises(ValueError): - output_coll = interpolate.from_scene_et_actual( + interpolate.from_scene_et_actual( scene_coll(['et', 'time', 'mask']), start_date='2017-07-01', end_date='2017-08-01', variables=['et'], interp_args={'interp_method': 'linear', 'interp_days': 32, @@ -772,7 +767,7 @@ def test_from_scene_et_actual_t_interval_bad_value(): def test_from_scene_et_actual_t_interval_no_value(): # Function should raise an Exception if t_interval is not set with pytest.raises(TypeError): - output_coll = interpolate.from_scene_et_actual( + interpolate.from_scene_et_actual( scene_coll(['et', 'time', 'mask']), start_date='2017-07-01', end_date='2017-08-01', variables=['et'], interp_args={'interp_method': 'linear', 'interp_days': 32, @@ -787,7 +782,7 @@ def test_from_scene_et_actual_t_interval_no_value(): """ These tests were attempts at making "full" interpolation calls. -They could be removed but are being left in case we want to explore this again +They could be removed but are being left in case we want to explore this again at some point in the future. """ # def test_daily_values_collection_a(): diff --git a/openet/core/utils.py b/openet/core/utils.py index faa0484..d0bb641 100644 --- a/openet/core/utils.py +++ b/openet/core/utils.py @@ -332,7 +332,6 @@ def get_info(ee_obj, max_retries=4): # if output: # break # -# # output = ee_obj.getInfo() # return output @@ -362,7 +361,7 @@ def is_number(x): try: float(x) return True - except: + except ValueError: return False @@ -411,6 +410,7 @@ def parse_int_set(nputstr=""): invalid.add(i) # Report invalid tokens before returning valid selection # print "Invalid set: " + str(invalid) + return selection @@ -450,6 +450,7 @@ def wrs2_set_2_str(tiles): tile_str = json.dumps(tile_dict, sort_keys=True) \ .replace('"', '').replace(' ', '') \ .replace('{', '').replace('}', '') + return tile_str @@ -462,6 +463,7 @@ def wrs2_str_2_set(tile_str, wrs2_fmt='p{:03d}r{:03d}'): rows = str_ranges_2_list(t.split(':')[1].replace(']', '')) for row in sorted(rows): tile_set.add(wrs2_fmt.format(path, int(row))) + return tile_set @@ -493,6 +495,7 @@ def str_ranges_2_list(nputstr=""): except: # not an int and not a range... invalid.add(i) + return sorted(list(selection)) @@ -506,12 +509,13 @@ def list_2_str_ranges(i): for a, b in itertools.groupby(enumerate(sorted(set(i))), lambda pair: pair[1] - pair[0]): b = list(b) # Only create ranges for 3 or more numbers - if b[0][1] != b[-1][1] and abs(b[-1][1] - b[0][1]) > 1 : + if b[0][1] != b[-1][1] and abs(b[-1][1] - b[0][1]) > 1: output.append('{}-{}'.format(b[0][1], b[-1][1])) elif b[0][1] != b[-1][1]: output.append('{},{}'.format(b[0][1], b[-1][1])) else: output.append('{}'.format(b[0][1])) + return ','.join(output)