Neat 242 allow arbitrary filters but raise warning that we do not sup…

…port them (#451) * added raw filter * added check if raw filter can be actually parsed * enable export of raw filter to dms schema * change log, bump version * Linting and static code checks * upgrade code * upgrade code * upgrade code --------- Co-authored-by: nikokaoja <[email protected]>
cognitedata · May 14, 2024 · 28eb266 · 28eb266
1 parent 4f7adbe
commit 28eb266
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 9 deletions.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.77.0"
+version="0.77.1"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

diff --git a/cognite/neat/_version.py b/cognite/neat/_version.py
@@ -1 +1 @@
-__version__ = "0.77.0"
+__version__ = "0.77.1"
diff --git a/cognite/neat/rules/models/dms/_exporter.py b/cognite/neat/rules/models/dms/_exporter.py
@@ -133,7 +133,6 @@ def _create_views_with_node_types(
             view_filter = self._create_view_filter(view, dms_view, data_model_type, dms_properties)
 
             view.filter = view_filter.as_dms_filter()
-
             if isinstance(view_filter, NodeTypeFilter):
                 unique_node_types.update(view_filter.nodes)
                 if view.as_id() in parent_views:
@@ -254,6 +253,7 @@ def _create_view_filter(
         dms_properties: list[DMSProperty],
     ) -> DMSFilter:
         selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
+
         if dms_view and dms_view.filter_ and not dms_view.filter_.is_empty:
             # Has Explicit Filter, use it
             return dms_view.filter_

diff --git a/cognite/neat/rules/models/dms/_rules.py b/cognite/neat/rules/models/dms/_rules.py
@@ -41,7 +41,7 @@
     ViewEntityList,
     ViewPropertyEntity,
 )
-from cognite.neat.rules.models.wrapped_entities import HasDataFilter, NodeTypeFilter
+from cognite.neat.rules.models.wrapped_entities import HasDataFilter, NodeTypeFilter, RawFilter
 
 from ._schema import DMSSchema
 
@@ -257,7 +257,7 @@ class DMSView(SheetEntity):
     description: str | None = Field(alias="Description", default=None)
     implements: ViewEntityList | None = Field(None, alias="Implements")
     reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
-    filter_: HasDataFilter | NodeTypeFilter | None = Field(None, alias="Filter")
+    filter_: HasDataFilter | NodeTypeFilter | RawFilter | None = Field(None, alias="Filter")
     in_model: bool = Field(True, alias="In Model")
     class_: ClassEntity = Field(alias="Class (linage)")
 

diff --git a/cognite/neat/rules/models/wrapped_entities.py b/cognite/neat/rules/models/wrapped_entities.py
@@ -1,3 +1,5 @@
+import json
+import re
 from abc import ABC, abstractmethod
 from collections.abc import Collection
 from functools import total_ordering
@@ -37,8 +39,19 @@ def _load(cls, data: Any) -> dict:
     def _parse(cls, data: str) -> dict:
         if data.casefold() == cls.name.casefold():
             return {"inner": None}
-        inner = data[len(cls.name) :].removeprefix("(").removesuffix(")")
-        return {"inner": [cls._inner_cls.load(entry.strip()) for entry in inner.split(",")]}
+
+        # raw filter case:
+        if cls.__name__ == "RawFilter":
+            if match := re.search(r"rawFilter\(([\s\S]*?)\)", data):
+                return {"filter": match.group(1), "inner": None}
+            else:
+                raise ValueError(f"Cannot parse {cls.name} from {data}. Ill formatted raw filter.")
+
+        # nodeType and hasData case:
+        elif inner := data[len(cls.name) :].removeprefix("(").removesuffix(")"):
+            return {"inner": [cls._inner_cls.load(entry.strip()) for entry in inner.split(",")]}
+        else:
+            raise ValueError(f"Cannot parse {cls.name} from {data}")
 
     @model_serializer(when_used="unless-none", return_type=str)
     def as_str(self) -> str:
@@ -164,3 +177,22 @@ def as_dms_filter(self, default: Collection[ContainerId] | None = None) -> dm.Fi
             # Sorting to ensure deterministic order
             containers=sorted(containers, key=lambda container: container.as_tuple())  # type: ignore[union-attr]
         )
+
+
+class RawFilter(DMSFilter):
+    name: ClassVar[str] = "rawFilter"
+    filter: str
+    inner: None = None  # type: ignore[assignment]
+
+    def as_dms_filter(self) -> dm.Filter:  # type: ignore[override]
+        try:
+            return dm.Filter.load(json.loads(self.filter))
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Error loading raw filter: {e}") from e
+
+    @property
+    def is_empty(self) -> bool:
+        return self.filter is None
+
+    def __repr__(self) -> str:
+        return self.filter
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -15,6 +15,11 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.77.1] - 14-05-24
+### Added
+- Support for `RawFilters` allow arbitrary filters to be applied to the data model.
+
+
 ## [0.77.0] - 13-05-24
 ### Changed
 - [BREAKING] The subpackage `cognite.neat.rules.models` is reorganized. All imports using this subpackage must be

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.77.0"
+version = "0.77.1"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

diff --git a/tests/tests_unit/rules/test_models/test_wrapped_entities.py b/tests/tests_unit/rules/test_models/test_wrapped_entities.py
@@ -4,7 +4,24 @@
 from cognite.client import data_modeling as dm
 
 from cognite.neat.rules.models.entities import ContainerEntity, DMSNodeEntity
-from cognite.neat.rules.models.wrapped_entities import DMSFilter, HasDataFilter, NodeTypeFilter, WrappedEntity
+from cognite.neat.rules.models.wrapped_entities import (
+    DMSFilter,
+    HasDataFilter,
+    NodeTypeFilter,
+    RawFilter,
+    WrappedEntity,
+)
+
+RAW_FILTER_EXAMPLE = """{"and": [
+    {
+      "in": {
+        "property": ["yggdrasil_domain_model", "EntityTypeGroup", "entityType"],
+        "values": ["CFIHOS_00000003"]
+      }
+    }
+  ]}"""
+
+RAW_FILTER_CELL_EXAMPLE = f"""rawFilter({RAW_FILTER_EXAMPLE})"""
 
 
 class TestWrappedEntities:
@@ -43,6 +60,11 @@ class TestWrappedEntities:
                     ]
                 ),
             ),
+            (
+                RawFilter,
+                RAW_FILTER_CELL_EXAMPLE,
+                RawFilter(filter=RAW_FILTER_EXAMPLE),
+            ),
         ],
     )
     def test_load(self, cls_: type[WrappedEntity], raw: Any, expected: WrappedEntity) -> None:
@@ -81,3 +103,15 @@ def test_from_dms_filter(self, filter_: dm.Filter, expected: DMSFilter) -> None:
         loaded = DMSFilter.from_dms_filter(filter_)
 
         assert loaded == expected
+
+    def test_has_data_vs_raw_filter(self) -> None:
+        assert (
+            HasDataFilter.load("hasData(space:container1)").as_dms_filter().dump()
+            == RawFilter.load(
+                """rawFilter({"hasData": [{"type": "container",
+                                           "space": "space",
+                                           "externalId": "container1"}]})"""
+            )
+            .as_dms_filter()
+            .dump()
+        )