Fix counts, Don't show occurrences from other projects (#359)

* Fix various counts and subqueries that displayed incorrect values

* Pass the project_id to species detail view to filter examples

* Open the correct session for an occurrence link (not the right frame yet)

* Link to a specific capture using a timestamp

ami/main/api/serializers.py

@@ -422,6 +422,7 @@ class Meta:
 class TaxonListSerializer(DefaultSerializer):
     # latest_detection = DetectionNestedSerializer(read_only=True)
     occurrences = serializers.SerializerMethodField()
+    occurrence_images = serializers.SerializerMethodField()
     parent = TaxonParentNestedSerializer(read_only=True)
 
     class Meta:
@@ -446,10 +447,32 @@ def get_occurrences(self, obj):
         Return URL to the occurrences endpoint filtered by this taxon.
         """
 
+        params = {"determination": obj.pk}
+        project_id = self.context.get("request", {}).query_params.get("project")
+        if project_id:
+            params["project"] = project_id
+
         return reverse_with_params(
             "occurrence-list",
             request=self.context.get("request"),
-            params={"determination": obj.pk},
+            params=params,
+        )
+
+    def get_occurrence_images(self, obj):
+        """
+        Call the occurrence_images method on the Taxon model, with arguments.
+        """
+
+        # request = self.context.get("request")
+        # project_id = request.query_params.get("project") if request else None
+        project_id = self.context["request"].query_params["project"]
+        classification_threshold = self.context["request"].query_params.get("threshold", None)
+
+        return obj.occurrence_images(
+            # @TODO pass the request to generate media url & filter by current user's access
+            # request=self.context.get("request"),
+            project_id=project_id,
+            classification_threshold=classification_threshold,
         )
 
 

ami/main/api/views.py

@@ -197,8 +197,20 @@ def get_queryset(self) -> QuerySet:
         if self.action != "list":
             qs = qs.annotate(
                 # detections_count=models.Count("captures__detections", distinct=True),
-                occurrences_count=models.Count("occurrences", distinct=True),
-                taxa_count=models.Count("occurrences__determination", distinct=True),
+                occurrences_count=models.Count(
+                    "occurrences",
+                    distinct=True,
+                    filter=models.Q(
+                        occurrences__determination_score__gte=get_active_classification_threshold(self.request)
+                    ),
+                ),
+                taxa_count=models.Count(
+                    "occurrences__determination",
+                    distinct=True,
+                    filter=models.Q(
+                        occurrences__determination_score__gte=get_active_classification_threshold(self.request)
+                    ),
+                ),
             ).prefetch_related("occurrences", "occurrences__determination")
         return qs
 
@@ -581,9 +593,11 @@ def filter_by_occurrence(self, queryset: QuerySet) -> tuple[QuerySet, bool]:
         """
 
         occurrence_id = self.request.query_params.get("occurrence")
-        project_id = self.request.query_params.get("project")
-        deployment_id = self.request.query_params.get("deployment")
-        event_id = self.request.query_params.get("event")
+        project_id = self.request.query_params.get("project") or self.request.query_params.get("occurrences__project")
+        deployment_id = self.request.query_params.get("deployment") or self.request.query_params.get(
+            "occurrences__deployment"
+        )
+        event_id = self.request.query_params.get("event") or self.request.query_params.get("occurrences__event")
 
         filter_active = any([occurrence_id, project_id, deployment_id, event_id])
 
@@ -601,6 +615,7 @@ def filter_by_occurrence(self, queryset: QuerySet) -> tuple[QuerySet, bool]:
             event = Event.objects.get(id=event_id)
             queryset = super().get_queryset().filter(occurrences__event=event)
 
+        # @TODO need to return the models.Q filter used, so we can use it for counts and related occurrences.
         return queryset, filter_active
 
     def filter_by_classification_threshold(self, queryset: QuerySet) -> QuerySet:
@@ -633,30 +648,34 @@ def get_queryset(self) -> QuerySet:
 
         qs = qs.select_related("parent", "parent__parent")
 
+        # @TODO this should check what the user has access to
+        project_id = self.request.query_params.get("project")
+        taxon_occurrences_query = Occurrence.objects.filter(
+            determination_score__gte=get_active_classification_threshold(self.request),
+            event__isnull=False,
+        ).distinct()
+        taxon_occurrences_count_filter = models.Q(
+            occurrences__determination_score__gte=get_active_classification_threshold(self.request),
+            occurrences__event__isnull=False,
+        )
+        if project_id:
+            taxon_occurrences_query = taxon_occurrences_query.filter(project=project_id)
+            taxon_occurrences_count_filter &= models.Q(occurrences__project=project_id)
+
         if self.action == "retrieve":
-            qs = qs.prefetch_related(
-                Prefetch(
-                    "occurrences",
-                    queryset=Occurrence.objects.filter(
-                        determination_score__gte=get_active_classification_threshold(self.request),
-                        event__isnull=False,
-                    ),
-                )
-            )
+            qs = qs.prefetch_related(Prefetch("occurrences", queryset=taxon_occurrences_query))
+
         if filter_active:
             qs = self.filter_by_classification_threshold(qs)
-
             qs = qs.annotate(
                 occurrences_count=models.Count(
                     "occurrences",
-                    filter=models.Q(
-                        occurrences__determination_score__gte=get_active_classification_threshold(self.request),
-                        occurrences__event__isnull=False,
-                    ),
+                    filter=taxon_occurrences_count_filter,
                     distinct=True,
                 ),
                 last_detected=models.Max("classifications__detection__timestamp"),
             )
+
         elif self.action == "list":
             # If no filter don't return anything related to occurrences
             # @TODO add a project_id filter to all request from the frontend

ami/main/models.py

@@ -432,8 +432,23 @@ def update_calculated_fields(self, save=False):
         self.events_count = self.events.count()
         self.captures_count = self.data_source_total_files or self.captures.count()
         self.detections_count = Detection.objects.filter(Q(source_image__deployment=self)).count()
-        self.occurrences_count = self.occurrences.count()
-        self.taxa_count = Taxon.objects.filter(Q(occurrences__deployment=self)).distinct().count()
+        self.occurrences_count = (
+            self.occurrences.filter(
+                determination_score__gte=DEFAULT_CONFIDENCE_THRESHOLD,
+                event__isnull=False,
+            )
+            .distinct()
+            .count()
+        )
+        self.taxa_count = (
+            Taxon.objects.filter(
+                occurrences__deployment=self,
+                occurrences__determination_score__gte=DEFAULT_CONFIDENCE_THRESHOLD,
+                occurrences__event__isnull=False,
+            )
+            .distinct()
+            .count()
+        )
 
         self.first_capture_timestamp, self.last_capture_timestamp = self.get_first_and_last_timestamps()
 
@@ -1358,6 +1373,8 @@ class Detection(BaseModel):
         blank=True,
     )
     detection_time = models.DateTimeField(null=True, blank=True)
+    # @TODO not sure if this detection score is ever used
+    # I think it was intended to be the score of the detection algorithm (bbox score)
     detection_score = models.FloatField(null=True, blank=True)
     # detection_job = models.ForeignKey(
     #     "Job",
@@ -1867,23 +1884,46 @@ def best_determination_score(self) -> float | None:
         # This is handled by an annotation if we are filtering by project, deployment or event
         return None
 
-    def occurrence_images(self, limit: int | None = 10) -> list[str]:
+    def occurrence_images(
+        self,
+        limit: int | None = 10,
+        project_id: int | None = None,
+        classification_threshold: float | None = None,
+    ) -> list[str]:
         """
         Return one image from each occurrence of this Taxon.
         The image should be from the detection with the highest classification score.
 
         This is used for image thumbnail previews in the species summary view.
+
+        The project ID is an optional filter however
+        @TODO important, this should always filter by what the current user has access to.
+        Use the request.user to filter by the user's access.
+        Use the request to generate the full media URLs.
         """
 
+        classification_threshold = classification_threshold or DEFAULT_CONFIDENCE_THRESHOLD
+
         # Retrieve the URLs using a single optimized query
-        detection_image_paths = (
-            self.occurrences.prefetch_related("detections__classifications")
+        qs = (
+            self.occurrences.prefetch_related(
+                models.Prefetch(
+                    "detections__classifications",
+                    queryset=Classification.objects.filter(score__gte=classification_threshold).order_by("-score"),
+                )
+            )
             .annotate(max_score=models.Max("detections__classifications__score"))
             .filter(detections__classifications__score=models.F("max_score"))
             .order_by("-max_score")
-            .values_list("detections__path", flat=True)[:limit]
         )
+        if project_id is not None:
+            # @TODO this should check the user's access instead
+            qs = qs.filter(project=project_id)
+
+        detection_image_paths = qs.values_list("detections__path", flat=True)[:limit]
 
+        # @TODO should this be done in the serializer?
+        # @TODO better way to get distinct values from an annotated queryset?
         return [get_media_url(path) for path in detection_image_paths if path]
 
     def list_names(self) -> str:
@@ -1913,7 +1953,7 @@ class Meta:
         ]
         verbose_name_plural = "Taxa"
 
-        # Set unique contstraints on name & rank
+        # Set unique constraints on name & rank
         # constraints = [
         #     models.UniqueConstraint(fields=["name", "rank", "parent"], name="unique_name_and_placement"),
         # ]

ui/src/data-services/hooks/species/useSpeciesDetails.ts

@@ -11,7 +11,8 @@ const convertServerRecord = (record: ServerSpeciesDetails) =>
   new SpeciesDetails(record)
 
 export const useSpeciesDetails = (
-  id: string
+  id: string,
+  projectId: string | undefined,
 ): {
   species?: SpeciesDetails
   isLoading: boolean
@@ -21,7 +22,7 @@ export const useSpeciesDetails = (
   const { data, isLoading, isFetching, error } =
     useAuthorizedQuery<SpeciesDetails>({
       queryKey: [API_ROUTES.SPECIES, id],
-      url: `${API_URL}/${API_ROUTES.SPECIES}/${id}/`,
+      url: `${API_URL}/${API_ROUTES.SPECIES}/${id}/?project=${projectId || ''}`,
     })
 
   const species = useMemo(

ui/src/data-services/models/occurrence.ts

@@ -22,6 +22,11 @@ export class Occurrence {
       .map((src: string) => ({ src }))
   }
 
+  get firstAppearanceTimestamp(): string {
+    // Return the first appearance timestamp in ISO format
+    return this._occurrence.first_appearance_timestamp
+  }
+
   get dateLabel(): string {
     const date = new Date(this._occurrence.first_appearance_timestamp)
     return getFormatedDateString({ date })

ui/src/pages/occurrences/occurrence-columns.tsx

@@ -91,7 +91,12 @@ export const columns: (projectId: string) => TableColumn<Occurrence>[] = (
       name: translate(STRING.FIELD_LABEL_DATE),
       sortField: 'first_appearance_timestamp',
       renderCell: (item: Occurrence) => (
-        <Link to={APP_ROUTES.SESSION_DETAILS({ projectId, sessionId: item.id })}>
+        <Link to={getAppRoute({
+          to: APP_ROUTES.SESSION_DETAILS({ projectId, sessionId: item.sessionId }), filters: {
+            occurrence: item.id,
+            timestamp: item.firstAppearanceTimestamp
+          }
+        })}>
           <BasicTableCell value={item.dateLabel} />
         </Link>
       ),
@@ -101,7 +106,12 @@ export const columns: (projectId: string) => TableColumn<Occurrence>[] = (
       sortField: 'first_appearance_time',
       name: translate(STRING.FIELD_LABEL_TIME),
       renderCell: (item: Occurrence) => (
-        <Link to={APP_ROUTES.SESSION_DETAILS({ projectId, sessionId: item.id })}>
+        <Link to={getAppRoute({
+          to: APP_ROUTES.SESSION_DETAILS({ projectId, sessionId: item.sessionId }), filters: {
+            occurrence: item.id,
+            timestamp: item.firstAppearanceTimestamp
+          }
+        })}>
           <BasicTableCell value={item.timeLabel} />
         </Link>
       )

ui/src/pages/species/species.tsx

@@ -92,7 +92,7 @@ const SpeciesDetailsDialog = ({ id }: { id: string }) => {
   const navigate = useNavigate()
   const { projectId } = useParams()
   const { setDetailBreadcrumb } = useContext(BreadcrumbContext)
-  const { species, isLoading } = useSpeciesDetails(id)
+  const { species, isLoading } = useSpeciesDetails(id, projectId)
 
   useEffect(() => {
     setDetailBreadcrumb(species ? { title: species.name } : undefined)

ui/src/utils/getAppRoute.ts

@@ -6,6 +6,7 @@ type FilterType =
   | 'occurrences__event'
   | 'occurrence'
   | 'capture'
+  | 'timestamp'
 
 export const getAppRoute = ({
   to,