Update tutorial cdm extension via selection of concepts (#1035)

# Description

Improved Extending Core Data Model tutorial, and made modification to
the code base to assure fully functional data model in CDF. Alpha flags
are renamed to Experimental flag to support tutorial.

## Bump

- [ ] Patch
- [x] Minor
- [ ] Skip

## Changelog
### Added
- Core Data Model transformer that produces data model version that can
be extended/edited by users
- Convenience method for extension of core data model via subsetting of
concepts `..read.cdf.core_data_model` in `NeatSession`

### Changed
- Rename alpha to experimental flags
- Update Extending Core Data Model tutorial in documentation

cognite/neat/_alpha.py

@@ -1,23 +1,24 @@
 import warnings
 
 
-class AlphaWarning(UserWarning):
+class ExperimentalFeatureWarning(UserWarning):
     def __init__(self, feature_name: str):
-        super().__init__(f"Alpha feature '{feature_name}' is subject to change without notice")
+        super().__init__(f"Experimental feature '{feature_name}' is subject to change without notice")
 
     def warn(self) -> None:
         warnings.warn(self, stacklevel=2)
 
 
-class AlphaFlags:
-    manual_rules_edit = AlphaWarning("enable_manual_edit")
-    same_space_properties_only_export = AlphaWarning("same-space-properties-only")
-    standardize_naming = AlphaWarning("standardize_naming")
-    standardize_space_and_version = AlphaWarning("standardize_space_and_version")
-    data_model_subsetting = AlphaWarning("data_model_subsetting")
-    ontology_read = AlphaWarning("ontology_read")
-    imf_read = AlphaWarning("imf_read")
-    dexpi_read = AlphaWarning("dexpi_read")
-    aml_read = AlphaWarning("aml_read")
-    csv_read = AlphaWarning("csv_read")
-    to_ontology = AlphaWarning("to_ontology")
+class ExperimentalFlags:
+    manual_rules_edit = ExperimentalFeatureWarning("enable_manual_edit")
+    same_space_properties_only_export = ExperimentalFeatureWarning("same-space-properties-only")
+    standardize_naming = ExperimentalFeatureWarning("standardize_naming")
+    standardize_space_and_version = ExperimentalFeatureWarning("standardize_space_and_version")
+    data_model_subsetting = ExperimentalFeatureWarning("data_model_subsetting")
+    core_data_model_subsetting = ExperimentalFeatureWarning("core_data_model_subsetting")
+    ontology_read = ExperimentalFeatureWarning("ontology_read")
+    imf_read = ExperimentalFeatureWarning("imf_read")
+    dexpi_read = ExperimentalFeatureWarning("dexpi_read")
+    aml_read = ExperimentalFeatureWarning("aml_read")
+    csv_read = ExperimentalFeatureWarning("csv_read")
+    to_ontology = ExperimentalFeatureWarning("to_ontology")

cognite/neat/_constants.py

@@ -64,6 +64,79 @@ def _is_in_browser() -> bool:
         "cdf_units",
     }
 )
+
+COGNITE_CORE_CONCEPTS = frozenset(
+    {
+        "CogniteFile",
+        "CogniteCubeMap",
+        "CogniteCADRevision",
+        "CognitePointCloudVolume",
+        "Cognite360ImageAnnotation",
+        "CogniteAnnotation",
+        "CogniteUnit",
+        "CogniteAsset",
+        "Cognite3DObject",
+        "Cognite3DRevision",
+        "Cognite360Image",
+        "CogniteDiagramAnnotation",
+        "Cognite360ImageCollection",
+        "Cognite360ImageStation",
+        "CognitePointCloudModel",
+        "CogniteTimeSeries",
+        "Cognite3DTransformation",
+        "CogniteEquipment",
+        "Cognite360ImageModel",
+        "CogniteAssetClass",
+        "CogniteAssetType",
+        "CogniteEquipmentType",
+        "Cognite3DModel",
+        "CogniteCADModel",
+        "CognitePointCloudRevision",
+        "CogniteCADNode",
+        "CogniteFileCategory",
+        "CogniteActivity",
+    }
+)
+
+
+COGNITE_CORE_FEATURES = frozenset(
+    {
+        "CogniteDescribable",
+        "CogniteSourceable",
+        "CogniteSourceSystem",
+        "CogniteSchedulable",
+        "CogniteVisualizable",
+    }
+)
+
+COGNITE_3D_CONCEPTS = frozenset(
+    {
+        "Cognite3DModel",
+        "Cognite3DObject",
+        "Cognite3DRevision",
+        "Cognite3DTransformation",
+        "Cognite360Image",
+        "Cognite360ImageAnnotation",
+        "Cognite360ImageCollection",
+        "Cognite360ImageModel",
+        "Cognite360ImageStation",
+        "CogniteCADModel",
+        "CogniteCADNode",
+        "CogniteCADRevision",
+        "CogniteCubeMap",
+        "CognitePointCloudModel",
+        "CognitePointCloudRevision",
+        "CognitePointCloudVolume",
+    }
+)
+
+COGNITE_ANNOTATION = frozenset(
+    {
+        "CogniteAnnotation",
+        "CogniteDiagramAnnotation",
+    }
+)
+
 DMS_LISTABLE_PROPERTY_LIMIT = 1000
 
 EXAMPLE_RULES = PACKAGE_DIRECTORY / "_rules" / "examples"

cognite/neat/_rules/transformers/_converters.py

@@ -17,6 +17,8 @@
 from cognite.neat._client import NeatClient
 from cognite.neat._client.data_classes.data_modeling import ContainerApplyDict, ViewApplyDict
 from cognite.neat._constants import (
+    COGNITE_CORE_CONCEPTS,
+    COGNITE_CORE_FEATURES,
     COGNITE_MODELS,
     COGNITE_SPACES,
     DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
@@ -1802,6 +1804,76 @@ def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadat
         )
 
 
+class _SubsetEditableCDMRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
+    """Subsets editable CDM rules to only include desired set of CDM concepts.
+
+    !!! note "Platypus UI limitations"
+        This is temporal solution to enable cleaner extension of core data model,
+        assuring that Platypus UI will work correctly, including Data Model Editor,
+        Query Explorer and Search.
+    """
+
+    def __init__(self, views: set[ViewEntity]):
+        if not_in_cognite_core := {view.external_id for view in views} - COGNITE_CORE_CONCEPTS.union(
+            COGNITE_CORE_FEATURES
+        ):
+            raise NeatValueError(
+                f"Concept(s) {', '.join(not_in_cognite_core)} is/are not part of the Cognite Core Data Model. Aborting."
+            )
+
+        self._views = views
+
+    def transform(self, rules: DMSRules) -> DMSRules:
+        # should check to make sure data model is based on the editable CDM
+        # if not raise an error
+
+        subsetted_rules: dict[str, Any] = {
+            "metadata": rules.metadata.model_copy(),
+            "views": SheetList[DMSView](),
+            "properties": SheetList[DMSProperty](),
+            "containers": SheetList[DMSContainer](),
+            "enum": rules.enum,
+            "nodes": rules.nodes,
+        }
+
+        containers_to_keep = set()
+
+        if editable_views_to_keep := self._editable_views_to_keep(rules):
+            for view in rules.views:
+                if view.view in editable_views_to_keep or view.view.space in COGNITE_SPACES:
+                    subsetted_rules["views"].append(view)
+
+            for property_ in rules.properties:
+                if property_.view in editable_views_to_keep and (
+                    isinstance(property_.value_type, DataType)
+                    or isinstance(property_.value_type, DMSUnknownEntity)
+                    or (isinstance(property_.value_type, ViewEntity) and property_.value_type in editable_views_to_keep)
+                ):
+                    subsetted_rules["properties"].append(property_)
+                    if property_.container:
+                        containers_to_keep.add(property_.container)
+
+            if rules.containers:
+                for container in rules.containers:
+                    if container.container in containers_to_keep:
+                        subsetted_rules["containers"].append(container)
+            try:
+                return DMSRules.model_validate(subsetted_rules)
+            except ValidationError as e:
+                raise NeatValueError(f"Cannot subset rules: {e}") from e
+        else:
+            raise NeatValueError("Cannot subset rules: provided data model is not based on Core Data Model")
+
+    def _editable_views_to_keep(self, rules: DMSRules) -> set[ViewEntity]:
+        return {
+            view.view
+            for view in rules.views
+            if view.view.space not in COGNITE_SPACES
+            and view.implements
+            and any(implemented in self._views for implemented in view.implements)
+        }
+
+
 class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
     """Subsets DMSRules to only include the specified views."""
 

cognite/neat/_session/_create.py

@@ -3,7 +3,6 @@
 from cognite.client.data_classes.data_modeling import DataModelIdentifier
 
 from cognite.neat._issues import IssueList
-from cognite.neat._rules.models import DMSRules, InformationRules
 from cognite.neat._rules.models.dms import DMSValidation
 from cognite.neat._rules.transformers import (
     IncludeReferenced,
@@ -29,7 +28,7 @@ def __init__(self, state: SessionState):
     def enterprise_model(
         self,
         data_model_id: DataModelIdentifier,
-        org_name: str = "My",
+        org_name: str = "CopyOf",
         dummy_property: str = "GUID",
     ) -> IssueList:
         """Uses the current data model as a basis to create enterprise data model
@@ -58,7 +57,7 @@ def enterprise_model(
                 - Charts
 
         """
-        last_rules = self._get_last_rules()
+        last_rules = self._state.rule_store.last_verified_rules
         issues = self._state.rule_transform(
             ToEnterpriseModel(
                 new_model_id=data_model_id,
@@ -71,12 +70,6 @@ def enterprise_model(
             self._state.last_reference = last_rules
         return issues
 
-    def _get_last_rules(self) -> InformationRules | DMSRules | None:
-        if not self._state.rule_store.provenance:
-            return None
-        last_entity = self._state.rule_store.provenance[-1].target_entity
-        return last_entity.dms or last_entity.information
-
     def solution_model(
         self,
         data_model_id: DataModelIdentifier,
@@ -110,7 +103,7 @@ def solution_model(
             the containers in the solution data model space.
 
         """
-        last_rules = self._get_last_rules()
+        last_rules = self._state.rule_store.last_verified_rules
         issues = self._state.rule_transform(
             ToSolutionModel(
                 new_model_id=data_model_id,
@@ -144,7 +137,7 @@ def data_product_model(
                 If you set same-space, only the properties of the views in the same space as the data model
                 will be included.
         """
-        last_rules = self._get_last_rules()
+        last_rules = self._state.rule_store.last_verified_rules
         view_ids, container_ids = DMSValidation(
             self._state.rule_store.last_verified_dms_rules
         ).imported_views_and_containers_ids()

cognite/neat/_session/_prepare.py

@@ -4,7 +4,7 @@
 
 from rdflib import URIRef
 
-from cognite.neat._alpha import AlphaFlags
+from cognite.neat._alpha import ExperimentalFlags
 from cognite.neat._graph.transformers import (
     ConnectionToLiteral,
     ConvertLiteral,
@@ -273,7 +273,7 @@ def standardize_naming(self) -> IssueList:
         For properties, the naming will be standardized to camelCase.
         """
         warnings.filterwarnings("default")
-        AlphaFlags.standardize_naming.warn()
+        ExperimentalFlags.standardize_naming.warn()
         return self._state.rule_transform(StandardizeNaming())
 
     def standardize_space_and_version(self) -> IssueList:
@@ -282,5 +282,5 @@ def standardize_space_and_version(self) -> IssueList:
         This method will standardize the space and version in the data model to the Cognite standard.
         """
         warnings.filterwarnings("default")
-        AlphaFlags.standardize_space_and_version.warn()
+        ExperimentalFlags.standardize_space_and_version.warn()
         return self._state.rule_transform(StandardizeSpaceAndVersion())