fix: update f1 score to not consider Label.Unclear as unlabeled match…

…es (#171) * fix: update f1 score to not consider Label.Unclear as unlabeled matches --------- Signed-off-by: Christopher Phillips <[email protected]>
anchore · Oct 26, 2023 · 95ea523 · 95ea523
1 parent bedecf4
commit 95ea523
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/src/yardstick/comparison.py b/src/yardstick/comparison.py
@@ -122,7 +122,9 @@ def __post_init__(self, result: ScanResult, lineage: list[str]):
             # capture any matches that are in an indeterminate state...
             # note: it is important NOT to use a set here, as multiple instances of the same label will throw
             # the F1 score (e.g. [TP, TP ,TP] != [TP], only the latter is acceptable).
-            if len(label_set) != 1 or Label.Unclear in labels_for_match:
+            # note: we're not considering unclear label to count as an indeterminate match
+            # we want a definite f1 score in the quality gate without being overwhelmed by "disputed" unclear label
+            if len(label_set) != 1:
                 self.matches_with_indeterminate_labels.append(match)
                 # capture TP & FP only beyond this point...
                 continue

diff --git a/tests/unit/test_comparison.py b/tests/unit/test_comparison.py
@@ -195,7 +195,7 @@ def test_comparison_against_labels_indeterminate():
     m3_indeterminate = [
         artifact.LabelEntry(
             label=artifact.Label.Unclear,
-            vulnerability_id="CVE-2021-2222",
+            vulnerability_id="CVE-2020-2222",
             package=package_bash_5,
             **common_label_options,
         ),
@@ -226,8 +226,8 @@ def test_comparison_against_labels_indeterminate():
         lineage=[],
     )
 
-    assert actual.summary.indeterminate == 2
-    assert set(actual.matches_with_indeterminate_labels) == {m2, m3}
+    assert actual.summary.indeterminate == 1
+    assert set(actual.matches_with_indeterminate_labels) == {m2}
     assert actual.summary.f1_score == 1
-    assert actual.summary.f1_score_lower_confidence == 0.5
+    assert actual.summary.f1_score_lower_confidence == 0.6666666666666666
     assert actual.summary.f1_score_upper_confidence == 1