[NEAT-787] ➕ Introducing min/max for DMS sheet (#1029)

# Description

Please describe the change you have made.

## Bump

- [ ] Patch
- [x] Minor
- [ ] Skip

## Changelog
### Added

- Properties `Min Count` and `Max Count` to DMS sheet.

### Removed

- The properties `Is List` and `Nullable` are now deprecated in the DMS
sheet, and instead derived from the Min Count` and `Max Count`.

cdf.toml

@@ -5,7 +5,7 @@ default_env = "dev"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.8"
+version = "0.4.10"
 
 [alpha_flags]
 classic = true

cognite/neat/_constants.py

@@ -117,9 +117,12 @@ def get_default_prefixes_and_namespaces() -> dict[str, Namespace]:
 
 DEFAULT_DOCS_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/"
 
+# These are the API limits for the DMS API, https://docs.cognite.com/cdf/dm/dm_reference/dm_limits_and_restrictions
 DMS_CONTAINER_PROPERTY_SIZE_LIMIT = 100
 DMS_VIEW_CONTAINER_SIZE_LIMIT = 10
-DMS_DIRECT_RELATION_LIST_LIMIT = 100
+DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT = 100
+DMS_PRIMITIVE_LIST_DEFAULT_LIMIT = 1000
+DMS_CONTAINER_LIST_MAX_LIMIT = 2000
 
 _ASSET_ROOT_PROPERTY = {
     "connection": "direct",

cognite/neat/_graph/loaders/_rdf2dms.py

@@ -21,7 +21,7 @@
 
 from cognite.neat._client import NeatClient
 from cognite.neat._client._api_client import SchemaAPI
-from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT, is_readonly_property
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT, is_readonly_property
 from cognite.neat._issues import IssueList, NeatError, NeatIssue, catch_issues
 from cognite.neat._issues.errors import (
     AuthorizationError,
@@ -417,20 +417,20 @@ def parse_direct_relation(cls, value: list, info: ValidationInfo) -> dict | list
                     ids = (self._create_instance_id(v, "node", stop_on_exception=True) for v in value)
                     result = [id_.dump(camel_case=True, include_instance_type=False) for id_ in ids]
                     # Todo: Account for max_list_limit
-                    if len(result) <= DMS_DIRECT_RELATION_LIST_LIMIT:
+                    if len(result) <= DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT:
                         return result
                     warnings.warn(
                         PropertyDirectRelationLimitWarning(
                             identifier="unknown",
                             resource_type="view property",
                             property_name=cast(str, cls.model_fields[info.field_name].alias or info.field_name),
-                            limit=DMS_DIRECT_RELATION_LIST_LIMIT,
+                            limit=DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
                         ),
                         stacklevel=2,
                     )
                     # To get deterministic results, we sort by space and externalId
                     result.sort(key=lambda x: (x["space"], x["externalId"]))
-                    return result[:DMS_DIRECT_RELATION_LIST_LIMIT]
+                    return result[:DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT]
                 elif value:
                     return self._create_instance_id(value[0], "node", stop_on_exception=True).dump(
                         camel_case=True, include_instance_type=False

cognite/neat/_issues/warnings/__init__.py

@@ -13,7 +13,13 @@
     FileReadWarning,
     FileTypeUnexpectedWarning,
 )
-from ._general import MissingCogniteClientWarning, NeatValueWarning, NotSupportedWarning, RegexViolationWarning
+from ._general import (
+    DeprecatedWarning,
+    MissingCogniteClientWarning,
+    NeatValueWarning,
+    NotSupportedWarning,
+    RegexViolationWarning,
+)
 from ._models import (
     BreakingModelingPrincipleWarning,
     CDFNotSupportedWarning,
@@ -50,6 +56,7 @@
     "CDFAuthWarning",
     "CDFMaxIterationsWarning",
     "CDFNotSupportedWarning",
+    "DeprecatedWarning",
     "FileItemNotSupportedWarning",
     "FileMissingRequiredFieldWarning",
     "FileReadWarning",

cognite/neat/_issues/warnings/_general.py

@@ -34,3 +34,13 @@ class MissingCogniteClientWarning(NeatWarning):
     """Missing Cognite Client required for {functionality}"""
 
     functionality: str
+
+
+@dataclass(unsafe_hash=True)
+class DeprecatedWarning(NeatWarning):
+    """{feature} is deprecated"""
+
+    extra = "{replacement}"
+
+    feature: str
+    replacement: str | None = None

cognite/neat/_issues/warnings/_properties.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import Generic
 
-from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
 from cognite.neat._issues._base import ResourceType
 
 from ._resources import ResourceNeatWarning, T_Identifier, T_ReferenceIdentifier
@@ -79,7 +79,7 @@ class PropertyDirectRelationLimitWarning(PropertyWarning[T_Identifier]):
 
     resource_type = "view"
 
-    limit: int = DMS_DIRECT_RELATION_LIST_LIMIT
+    limit: int = DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
 
 
 @dataclass(unsafe_hash=True)

cognite/neat/_rules/exporters/_rules2excel.py

@@ -170,16 +170,12 @@ def _add_drop_downs(self, workbook: Workbook, no_rows: int = 100) -> None:
         dv_value_types = generate_data_validation(self._helper_sheet_name, "C", no_header_rows=0, no_rows=no_rows)
 
         dv_immutable = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
-        dv_nullable = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
-        dv_is_list = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
         dv_in_model = generate_data_validation(self._helper_sheet_name, "D", no_header_rows=0, no_rows=3)
         dv_used_for = generate_data_validation(self._helper_sheet_name, "E", no_header_rows=0, no_rows=3)
 
         workbook["Properties"].add_data_validation(dv_views)
         workbook["Properties"].add_data_validation(dv_containers)
         workbook["Properties"].add_data_validation(dv_value_types)
-        workbook["Properties"].add_data_validation(dv_nullable)
-        workbook["Properties"].add_data_validation(dv_is_list)
         workbook["Properties"].add_data_validation(dv_immutable)
         workbook["Views"].add_data_validation(dv_in_model)
         workbook["Containers"].add_data_validation(dv_used_for)
@@ -194,12 +190,6 @@ def _add_drop_downs(self, workbook: Workbook, no_rows: int = 100) -> None:
         if column := find_column_with_value(workbook["Properties"], "Value Type"):
             dv_value_types.add(f"{column}{3}:{column}{no_rows * 100}")
 
-        if column := find_column_with_value(workbook["Properties"], "Nullable"):
-            dv_nullable.add(f"{column}{3}:{column}{no_rows * 100}")
-
-        if column := find_column_with_value(workbook["Properties"], "Is List"):
-            dv_is_list.add(f"{column}{3}:{column}{no_rows * 100}")
-
         if column := find_column_with_value(workbook["Properties"], "Immutable"):
             dv_immutable.add(f"{column}{3}:{column}{no_rows * 100}")
 

cognite/neat/_rules/importers/_dms2rules.py

@@ -7,8 +7,12 @@
 from cognite.client import data_modeling as dm
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
 from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
+from cognite.client.data_classes.data_modeling.data_types import (
+    DirectRelation,
+    ListablePropertyType,
+    PropertyTypeWithUnit,
+)
 from cognite.client.data_classes.data_modeling.data_types import Enum as DMSEnum
-from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType, PropertyTypeWithUnit
 from cognite.client.data_classes.data_modeling.views import (
     MultiEdgeConnectionApply,
     MultiReverseDirectRelationApply,
@@ -19,6 +23,7 @@
 from cognite.client.utils import ms_to_datetime
 
 from cognite.neat._client import NeatClient
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT, DMS_PRIMITIVE_LIST_DEFAULT_LIMIT
 from cognite.neat._issues import IssueList, MultiValueError, NeatIssue, catch_issues
 from cognite.neat._issues.errors import (
     FileTypeUnexpectedError,
@@ -336,8 +341,8 @@ def _create_dms_property(
             name=prop.name,
             connection=self._get_connection_type(prop),
             value_type=str(value_type),
-            is_list=self._get_is_list(prop),
-            nullable=self._get_nullable(prop),
+            min_count=self._get_min_count(prop),
+            max_count=self._get_max_count(prop),
             immutable=self._get_immutable(prop),
             default=self._get_default(prop),
             container=(
@@ -419,9 +424,9 @@ def _get_value_type(
             )
             return None
 
-    def _get_nullable(self, prop: ViewPropertyApply) -> bool | None:
+    def _get_min_count(self, prop: ViewPropertyApply) -> int | None:
         if isinstance(prop, dm.MappedPropertyApply):
-            return self._container_prop_unsafe(prop).nullable
+            return int(not self._container_prop_unsafe(prop).nullable)
         else:
             return None
 
@@ -431,15 +436,26 @@ def _get_immutable(self, prop: ViewPropertyApply) -> bool | None:
         else:
             return None
 
-    def _get_is_list(self, prop: ViewPropertyApply) -> bool | None:
+    def _get_max_count(self, prop: ViewPropertyApply) -> int | float | None:
         if isinstance(prop, dm.MappedPropertyApply):
             prop_type = self._container_prop_unsafe(prop).type
-            return isinstance(prop_type, ListablePropertyType) and prop_type.is_list
+            if isinstance(prop_type, ListablePropertyType):
+                if prop_type.is_list is False:
+                    return 1
+                elif isinstance(prop_type.max_list_size, int):
+                    return prop_type.max_list_size
+                elif isinstance(prop_type, DirectRelation):
+                    return DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT
+                else:
+                    return DMS_PRIMITIVE_LIST_DEFAULT_LIMIT
+            else:
+                return 1
         elif isinstance(prop, MultiEdgeConnectionApply | MultiReverseDirectRelationApply):
-            return True
+            return float("inf")
         elif isinstance(prop, SingleEdgeConnectionApply | SingleReverseDirectRelationApply):
-            return False
+            return 1
         else:
+            # Unknown type.
             return None
 
     def _get_default(self, prop: ViewPropertyApply) -> str | None:

cognite/neat/_rules/models/dms/_exporter.py

@@ -20,7 +20,11 @@
     ViewApplyDict,
 )
 from cognite.neat._client.data_classes.schema import DMSSchema
-from cognite.neat._constants import COGNITE_SPACES
+from cognite.neat._constants import (
+    COGNITE_SPACES,
+    DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
+    DMS_PRIMITIVE_LIST_DEFAULT_LIMIT,
+)
 from cognite.neat._issues.errors import NeatTypeError, NeatValueError, ResourceNotFoundError
 from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
 from cognite.neat._issues.warnings.user_modeling import (
@@ -310,7 +314,16 @@ def _create_containers(
 
                 args: dict[str, Any] = {}
                 if issubclass(type_cls, ListablePropertyType):
-                    args["is_list"] = prop.is_list or False
+                    is_list = args["is_list"] = prop.is_list or False
+                    if is_list:
+                        if type_cls is dm.DirectRelation and prop.max_count == DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT:
+                            # Use default of API.
+                            args["max_list_size"] = None
+                        elif type_cls is not dm.DirectRelation and prop.max_count == DMS_PRIMITIVE_LIST_DEFAULT_LIMIT:
+                            # Use default of API.
+                            args["max_list_size"] = None
+                        else:
+                            args["max_list_size"] = prop.max_count
                 if isinstance(prop.value_type, Double | Float) and isinstance(prop.value_type.unit, UnitEntity):
                     args["unit"] = prop.value_type.unit.as_reference()
                 if isinstance(prop.value_type, Enum):

cognite/neat/_rules/models/dms/_rules.py

@@ -1,3 +1,4 @@
+import math
 import warnings
 from collections.abc import Hashable
 from typing import TYPE_CHECKING, Any, ClassVar, Literal
@@ -8,6 +9,7 @@
 from pydantic_core.core_schema import SerializationInfo, ValidationInfo
 
 from cognite.neat._client.data_classes.schema import DMSSchema
+from cognite.neat._constants import DMS_CONTAINER_LIST_MAX_LIMIT
 from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings._general import NeatValueWarning
 from cognite.neat._rules.models._base_rules import (
@@ -102,22 +104,25 @@ class DMSProperty(SheetRow):
         alias="Value Type",
         description="Value type that the property can hold. It takes either subset of CDF primitive types or a View id",
     )
-    nullable: bool | None = Field(
+    min_count: int | None = Field(
+        alias="Min Count",
         default=None,
-        alias="Nullable",
-        description="Used to indicate whether the property is required or not. Only applies to primitive type.",
+        description="Minimum number of values that the property can hold. "
+        "If no value is provided, the default value is  `0`, "
+        "which means that the property is optional.",
+    )
+    max_count: int | float | None = Field(
+        alias="Max Count",
+        default=None,
+        description="Maximum number of values that the property can hold. "
+        "If no value is provided, the default value is  `inf`, "
+        "which means that the property can hold any number of values (listable).",
     )
     immutable: bool | None = Field(
         default=None,
         alias="Immutable",
         description="sed to indicate whether the property is can only be set once. Only applies to primitive type.",
     )
-    is_list: bool | None = Field(
-        default=None,
-        alias="Is List",
-        description="Used to indicate whether the property holds single or multiple values (list). "
-        "Only applies to primitive types.",
-    )
     default: bool | str | int | float | dict | None = Field(
         None, alias="Default", description="Specifies default value for the property."
     )
@@ -147,13 +152,43 @@ class DMSProperty(SheetRow):
         description="Used to make connection between physical and logical data model aspect",
     )
 
+    @property
+    def nullable(self) -> bool | None:
+        """Used to indicate whether the property is required or not. Only applies to primitive type."""
+        return self.min_count in {0, None}
+
+    @property
+    def is_list(self) -> bool | None:
+        """Used to indicate whether the property holds single or multiple values (list). "
+        "Only applies to primitive types."""
+        if self.max_count is None:
+            return None
+        return self.max_count is float("inf") or (isinstance(self.max_count, int | float) and self.max_count > 1)
+
     def _identifier(self) -> tuple[Hashable, ...]:
         return self.view, self.view_property
 
-    @field_validator("nullable")
+    @field_validator("min_count")
     def direct_relation_must_be_nullable(cls, value: Any, info: ValidationInfo) -> None:
-        if info.data.get("connection") == "direct" and value is False:
-            raise ValueError("Direct relation must be nullable")
+        if info.data.get("connection") == "direct" and value not in {0, None}:
+            raise ValueError("Direct relation must have min count set to 0")
+        return value
+
+    @field_validator("max_count", mode="before")
+    def as_integer(cls, value: Any) -> Any:
+        if isinstance(value, float) and not math.isinf(value):
+            return int(value)
+        return value
+
+    @field_validator("max_count")
+    def max_list_size(cls, value: Any, info: ValidationInfo) -> Any:
+        if isinstance(info.data.get("connection"), EdgeEntity | ReverseConnectionEntity):
+            if value is not None and value != float("inf") and not (isinstance(value, int) and value == 1):
+                raise ValueError("Edge and reverse connections must have max count set to inf or 1")
+            return value
+        # We do not have a connection, so we can check the max list size.
+        if isinstance(value, int) and value > DMS_CONTAINER_LIST_MAX_LIMIT:
+            raise ValueError(f"Max list size cannot be greater than {DMS_CONTAINER_LIST_MAX_LIMIT}")
         return value
 
     @field_validator("value_type", mode="after")