Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

issue #693 add search for schema and find parameters from the backend #698

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions openeo/internal/processes/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,20 @@ def is_geojson_schema(schema) -> bool:
return any(is_geojson_schema(s) for s in self.schema)
return False

def get_enum_options(self,name):
result = None
if isinstance(self.schema,list):
for item in self.schema:
if name in item:
if result is None:
result = item[name]
else:
raise ValueError("Multiple entries found with name {v}.".format(v=name))
elif isinstance(self.schema,dict):
if name in self.schema:
result = self.schema[name]
return result
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't look compatible with how json schema works.

For example, this methods should allow to extract the options from these example use cases:

For example, the caller should not have to specify a name. The implementation should extract the options from the "enum" listing(s) in the schema data



_NO_DEFAULT = object()

Expand Down Expand Up @@ -127,6 +141,12 @@ def from_json_file(cls, path: Union[str, Path]) -> Process:
with Path(path).open("r") as f:
return cls.from_json(f.read())

def get_parameter(self, name: str) -> Parameter:
for parameter in self.parameters:
if parameter.name == name:
return parameter
raise LookupError(f"Parameter {name!r} not found.")


def parse_all_from_dir(path: Union[str, Path], pattern="*.json") -> Iterator[Process]:
"""Parse all openEO process files in given directory"""
Expand Down
17 changes: 14 additions & 3 deletions openeo/rest/datacube.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
convert_callable_to_pgnode,
get_parameter_names,
)
from openeo.internal.processes.parse import Process
from openeo.internal.warnings import UserDeprecationWarning, deprecated, legacy_alias
from openeo.metadata import (
Band,
Expand Down Expand Up @@ -2767,9 +2768,19 @@ def sar_backscatter(
.. versionadded:: 0.4.9
.. versionchanged:: 0.4.10 replace `orthorectify` and `rtc` arguments with `coefficient`.
"""
coefficient_options = [
"beta0", "sigma0-ellipsoid", "sigma0-terrain", "gamma0-ellipsoid", "gamma0-terrain", None
]
try:
schema = Process.from_dict(self.connection.describe_process("sar_backscatter")).get_parameter("coefficient").schema
coefficient_options = schema.get_enum_options("enum") + [None]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On top of getting the options, we should also get the default value from the schema. It is now hardcoded to "gamma0-terrain" (above), but that should also be dynamic

except Exception as e:
log.warning(f"Failed to extract coefficient options for process `sar_backscatter`: {e}")
coefficient_options = [
"beta0",
"sigma0-ellipsoid",
"sigma0-terrain",
"gamma0-ellipsoid",
"gamma0-terrain",
None,
]
if coefficient not in coefficient_options:
raise OpenEoClientException("Invalid `sar_backscatter` coefficient {c!r}. Should be one of {o}".format(
c=coefficient, o=coefficient_options
Expand Down
74 changes: 74 additions & 0 deletions tests/internal/processes/test_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,35 @@ def test_schema_accepts_geojson(schema, expected):
assert Schema([{"type": "number"}, schema]).accepts_geojson() == expected


@pytest.mark.parametrize(
"schema",
[
Schema([
{"x1": "y1"},
{"x2": "y2"},
{"x3": "y3"},
{"x5": "y3"},
]),
Schema({"x1":"y1","x2":"y2","x3": "y3","x5": "y3"})
],
)
@pytest.mark.parametrize(("key", "expected"), [("x1", "y1"), ("x2", "y2"), ("x4", None)])
def test_get_enum_options(schema, key, expected):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To parameterize this test, I would use more realistic use cases, e.g. the ones linked above

assert schema.get_enum_options(key) == expected

@pytest.mark.parametrize("key",["x2","x3",])
def test_get_enum_options_error(key):
schema = Schema([
{"x1": "y1"},
{"x2": "y2"},
{"x3": "y3"},
{"x3": "y4"},
{"x2": "y1"},
])
with pytest.raises(ValueError):
schema.get_enum_options(key)


def test_parameter():
p = Parameter.from_dict({
"name": "foo",
Expand Down Expand Up @@ -138,6 +167,51 @@ def test_process_from_json():
assert p.returns.schema.schema == {"type": ["number", "null"], "minimum": 0}


@pytest.mark.parametrize(
("key", "expected"), [
("x", Parameter.from_dict({"name": "x", "description": "A number.", "schema": {"type": ["number", "null"]}})),
("y", Parameter.from_dict({"name": "y", "description": "A number.", "schema": {"type": ["number", "null"]}})),
]
)
def test_get_parameter(key, expected):
p = Process.from_dict({
"id": "absolute",
"summary": "Absolute value",
"description": "Computes the absolute value of a real number.",
"categories": ["math"],
"parameters": [
{"name": "x", "description": "A number.", "schema": {"type": ["number", "null"]}},
{"name": "y", "description": "A number.", "schema": {"type": ["number", "null"]}},
],
"returns": {
"description": "The computed absolute value.",
"schema": {"type": ["number", "null"], "minimum": 0}
},
"links": [{"rel": "about", "href": "http://example.com/abs.html"}],
})
assert p.get_parameter(key) == expected


def test_get_parameter_error():
p = Process.from_dict({
"id": "absolute",
"summary": "Absolute value",
"description": "Computes the absolute value of a real number.",
"categories": ["math"],
"parameters": [
{"name": "x", "description": "A number.", "schema": {"type": ["number", "null"]}},
{"name": "y", "description": "A number.", "schema": {"type": ["number", "null"]}},
],
"returns": {
"description": "The computed absolute value.",
"schema": {"type": ["number", "null"], "minimum": 0}
},
"links": [{"rel": "about", "href": "http://example.com/abs.html"}],
})
with pytest.raises(LookupError):
p.get_parameter("z")


def test_parse_remote_process_definition_minimal(requests_mock):
url = "https://example.com/ndvi.json"
requests_mock.get(url, json={"id": "ndvi"})
Expand Down
47 changes: 47 additions & 0 deletions tests/rest/datacube/test_datacube100.py
Original file line number Diff line number Diff line change
Expand Up @@ -2828,6 +2828,53 @@ def test_sar_backscatter_coefficient_invalid(con100):
cube.sar_backscatter(coefficient="unicorn")


def test_sar_backscatter_check_coefficient_backend(con100, requests_mock):
requests_mock.get(API_URL, json={"api_version": "1.0.0"})
processes = [
{
"id": "sar_backscatter",
"description": "Computes backscatter from SAR input",
"summary": "Computes backscatter from SAR input",
"parameters": [
{
"default": "gamma0-terrain",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test should also check that this default value is picked

"description": "Select the radiometric correction coefficient.",
"name": "coefficient",
"schema": [
{
"enum": [
"beta0",
"sigma0-ellipsoid",
"sigma0-terrain",
"gamma0-ellipsoid",
"gamma0-terrain",
],
"type": "string",
},
],
},
],
"returns": {"description": "incremented value", "schema": {"type": "integer"}},
}
]
requests_mock.get(API_URL + "/processes", json={"processes": processes})
cube = con100.load_collection("S2").sar_backscatter()
assert _get_leaf_node(cube) == {
"process_id": "sar_backscatter",
"arguments": {
"data": {"from_node": "loadcollection1"},
"coefficient": "gamma0-terrain",
"elevation_model": None,
"mask": False,
"contributing_area": False,
"local_incidence_angle": False,
"ellipsoid_incidence_angle": False,
"noise_removal": True,
},
"result": True,
}


def test_datacube_from_process(con100):
cube = con100.datacube_from_process("colorize", color="red", size=4)
assert cube.flat_graph() == {
Expand Down