feat: yardstick validate subcommand (#380)

add a subcommand "yardstick validate". This allows users of yardstick to specify
validations on result sets, and assert that a candidate tool configuration performs
as well or better than a reference tool configuration for a given set of SBOMs.
Previously, clients had to consume yardstick as a library to achieve this.

---------

Signed-off-by: Will Murphy <[email protected]>

.github/workflows/validations.yaml

@@ -11,7 +11,7 @@ permissions:
 
 env:
   PYTHON_VERSION: "3.11"
-  POETRY_VERSION: "1.3.2"
+  POETRY_VERSION: "1.8.3"
 
 jobs:
 

src/yardstick/__init__.py

@@ -1,7 +1,18 @@
 import logging
 from typing import Callable, Optional
 
-from . import arrange, artifact, capture, cli, comparison, label, store, tool, utils
+from . import (
+    arrange,
+    artifact,
+    capture,
+    cli,
+    comparison,
+    label,
+    store,
+    tool,
+    validate,
+    utils,
+)
 
 __all__ = [
     "arrange",
@@ -12,6 +23,7 @@
     "label",
     "store",
     "tool",
+    "validate",
     "utils",
 ]
 

src/yardstick/artifact.py

@@ -118,6 +118,7 @@ class ScanConfiguration:
     image_digest: str
     tool_name: str
     tool_version: str
+    tool_label: str | None = None
     image_tag: str = ""
     timestamp: datetime.datetime | None = field(
         default=None,
@@ -205,6 +206,7 @@ def new(
             tool_name=tool_obj.id,
             tool_version=tool_obj.version,
             timestamp=timestamp,
+            tool_label=label,
         )
 
 

src/yardstick/cli/cli.py

@@ -8,7 +8,7 @@
 import yaml
 
 from yardstick import store
-from yardstick.cli import config, label, result
+from yardstick.cli import config, label, result, validate
 
 
 @click.option("--verbose", "-v", default=False, help="show logs", is_flag=True)
@@ -126,5 +126,6 @@ def version(_: config.Application):
     print(f"{d.name} {d.version} ({d.locate_file(d.name).parent})")
 
 
+cli.add_command(validate.validate)
 cli.add_command(result.group)
 cli.add_command(label.group)

src/yardstick/cli/config.py

@@ -9,7 +9,7 @@
 import yaml
 from dataclass_wizard import asdict, fromdict  # type: ignore[import]
 
-from yardstick import artifact
+from yardstick import artifact, validate
 from yardstick.store import config as store_config
 
 DEFAULT_CONFIGS = (
@@ -115,11 +115,17 @@ def parse_oci_reference(image: str) -> tuple[str, str, str, str, str]:
         return host, path, repository, tag, digest
 
 
+@dataclass()
+class Validation(validate.GateConfig):
+    name: str = "default"
+
+
 @dataclass()
 class ResultSet:
     description: str = ""
     declared: list[artifact.ScanRequest] = field(default_factory=list)
     matrix: ScanMatrix = field(default_factory=ScanMatrix)
+    validations: list[Validation] = field(default_factory=list)
 
     def images(self) -> list[str]:
         return self.matrix.images + [req.image for req in self.declared]
@@ -151,6 +157,34 @@ class Application:
     default_max_year: int | None = None
     derive_year_from_cve_only: bool = False
 
+    def max_year_for_any_result_set(self, result_sets: list[str]) -> int | None:
+        years = []
+        for result_set in result_sets:
+            m = self.max_year_for_result_set(result_set)
+            if m is not None:
+                years.append(m)
+
+        if not years:
+            return None
+
+        return max(years)
+
+    def max_year_for_result_set(self, result_set: str) -> int | None:
+        """return the max year needed by any validation on the result set, or default_max_year"""
+        rs = self.result_sets.get(result_set, None)
+        years = []
+        if rs is not None:
+            for gate in rs.validations:
+                if gate.max_year is not None:
+                    years.append(gate.max_year)
+                elif self.default_max_year is not None:
+                    years.append(self.default_max_year)
+
+        if years:
+            return max(years)
+
+        return self.default_max_year
+
 
 def clean_dict_keys(d):
     new = {}