From 8ec7de933ce5b80b8a13432e31da3a85b1a4bff3 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sun, 3 Dec 2023 20:39:35 -0800 Subject: [PATCH 1/7] Features for prioritizing & hiding projects --- ami/main/admin.py | 2 ++ ami/main/api/views.py | 2 +- ..._options_alter_project_options_and_more.py | 30 +++++++++++++++++++ ami/main/models.py | 6 ++++ 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 ami/main/migrations/0028_alter_occurrence_options_alter_project_options_and_more.py diff --git a/ami/main/admin.py b/ami/main/admin.py index 58dd4c092..96fb87a22 100644 --- a/ami/main/admin.py +++ b/ami/main/admin.py @@ -55,6 +55,8 @@ class BlogPostAdmin(admin.ModelAdmin[BlogPost]): class ProjectAdmin(admin.ModelAdmin[Project]): """Admin panel example for ``Project`` model.""" + list_display = ("name", "priority", "active", "created_at", "updated_at") + @admin.register(Deployment) class DeploymentAdmin(admin.ModelAdmin[Deployment]): diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 4ad501084..19847e11e 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -102,7 +102,7 @@ class ProjectViewSet(DefaultViewSet): API endpoint that allows projects to be viewed or edited. """ - queryset = Project.objects.prefetch_related("deployments").all() + queryset = Project.objects.filter(active=True).prefetch_related("deployments").all() serializer_class = ProjectSerializer def get_serializer_class(self): diff --git a/ami/main/migrations/0028_alter_occurrence_options_alter_project_options_and_more.py b/ami/main/migrations/0028_alter_occurrence_options_alter_project_options_and_more.py new file mode 100644 index 000000000..4b251ba74 --- /dev/null +++ b/ami/main/migrations/0028_alter_occurrence_options_alter_project_options_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.2 on 2023-12-03 23:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0027_update_occurrence_scores"), + ] + + operations = [ + migrations.AlterModelOptions( + name="occurrence", + options={"ordering": ["-determination_score"]}, + ), + migrations.AlterModelOptions( + name="project", + options={"ordering": ["-priority", "created_at"]}, + ), + migrations.AddField( + model_name="project", + name="active", + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name="project", + name="priority", + field=models.IntegerField(default=1), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 5448e1219..add61992e 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -94,6 +94,9 @@ class Project(BaseModel): taxa: models.QuerySet["Taxon"] taxa_lists: models.QuerySet["TaxaList"] + active = models.BooleanField(default=True) + priority = models.IntegerField(default=1) + def deployments_count(self) -> int: return self.deployments.count() @@ -117,6 +120,9 @@ def summary_data(self): return plots + class Meta: + ordering = ["-priority", "created_at"] + @final class Device(BaseModel): From 4dbc458efa370ead3f7218395d9a44010087ac00 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 7 Dec 2023 02:54:46 +0000 Subject: [PATCH 2/7] Speed up the taxa list view when no project is specified --- ami/main/api/views.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 19847e11e..196003a75 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -537,7 +537,7 @@ def get_serializer_class(self): else: return TaxonSerializer - def filter_by_occurrence(self, queryset: QuerySet) -> QuerySet: + def filter_by_occurrence(self, queryset: QuerySet) -> tuple[QuerySet, bool]: """ Filter taxa by when/where it has occurred. @@ -551,10 +551,12 @@ def filter_by_occurrence(self, queryset: QuerySet) -> QuerySet: deployment_id = self.request.query_params.get("deployment") event_id = self.request.query_params.get("event") + filter_active = any([occurrence_id, project_id, deployment_id, event_id]) + if occurrence_id: occurrence = Occurrence.objects.get(id=occurrence_id) # This query does not need the same filtering as the others - return queryset.filter(occurrences=occurrence).distinct() + return queryset.filter(occurrences=occurrence).distinct(), True elif project_id: project = Project.objects.get(id=project_id) queryset = super().get_queryset().filter(occurrences__project=project) @@ -575,23 +577,35 @@ def filter_by_occurrence(self, queryset: QuerySet) -> QuerySet: if not self.request.query_params.get("ordering"): queryset = queryset.order_by("-best_determination_score") - return queryset + return queryset, filter_active def get_queryset(self) -> QuerySet: qs = super().get_queryset() try: - qs = self.filter_by_occurrence(qs) + qs, filter_active = self.filter_by_occurrence(qs) except exceptions.ObjectDoesNotExist as e: from rest_framework.exceptions import NotFound raise NotFound(detail=str(e)) qs = qs.select_related("parent", "parent__parent") - qs = qs.prefetch_related("occurrences") - qs = qs.annotate( - occurrences_count=models.Count("occurrences", distinct=True), - events_count=models.Count("occurrences__event", distinct=True), - last_detected=models.Max("classifications__detection__timestamp"), - ) + + if filter_active: + qs = qs.prefetch_related("occurrences") + qs = qs.annotate( + occurrences_count=models.Count("occurrences", distinct=True), + events_count=models.Count("occurrences__event", distinct=True), + last_detected=models.Max("classifications__detection__timestamp"), + ) + elif self.action == "list": + # If no filter don't return anything related to occurrences + # @TODO add a project_id filter to all request from the frontend + # event detail views should be filtered by project + qs = qs.prefetch_related(Prefetch("occurrences", queryset=Occurrence.objects.none())) + qs = qs.annotate( + occurrences_count=models.Value(0), + events_count=models.Value(0), + last_detected=models.Value(None, output_field=models.DateTimeField()), + ) return qs From e0fca7838e76589e82c98f800fbfa9257dd3d3fd Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 7 Dec 2023 05:25:15 +0000 Subject: [PATCH 3/7] Speed up session view query until we cache values --- ami/main/api/views.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 196003a75..05496f240 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -149,17 +149,7 @@ class EventViewSet(DefaultViewSet): API endpoint that allows events to be viewed or edited. """ - queryset = ( - Event.objects.select_related("deployment") - .annotate( - captures_count=models.Count("captures", distinct=True), - detections_count=models.Count("captures__detections", distinct=True), - occurrences_count=models.Count("occurrences", distinct=True), - taxa_count=models.Count("occurrences__determination", distinct=True), - duration=models.F("end") - models.F("start"), - ) - .select_related("deployment", "project") - ) # .prefetch_related("captures").all() + queryset = Event.objects.all() serializer_class = EventSerializer filterset_fields = ["deployment", "project"] ordering_fields = [ @@ -184,6 +174,21 @@ def get_serializer_class(self): else: return EventSerializer + def get_queryset(self) -> QuerySet: + qs: QuerySet = super().get_queryset() + qs = qs.annotate( + captures_count=models.Count("captures", distinct=True), + duration=models.F("end") - models.F("start"), + ).select_related("deployment", "project") + + if self.action != "list": + qs = qs.annotate( + detections_count=models.Count("captures__detections", distinct=True), + occurrences_count=models.Count("occurrences", distinct=True), + taxa_count=models.Count("occurrences__determination", distinct=True), + ) + return qs + class SourceImageViewSet(DefaultViewSet): """ From 643e86fc218a23d7bebd58e220d45cbb80bebf5a Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 7 Dec 2023 05:30:53 +0000 Subject: [PATCH 4/7] Disable detection counts on session views all together for now --- ami/main/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 05496f240..3c161a1f6 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -183,7 +183,7 @@ def get_queryset(self) -> QuerySet: if self.action != "list": qs = qs.annotate( - detections_count=models.Count("captures__detections", distinct=True), + # detections_count=models.Count("captures__detections", distinct=True), occurrences_count=models.Count("occurrences", distinct=True), taxa_count=models.Count("occurrences__determination", distinct=True), ) From de256c64aff2faf138b1f8df3553b8224397296b Mon Sep 17 00:00:00 2001 From: Debian Date: Thu, 7 Dec 2023 05:37:51 +0000 Subject: [PATCH 5/7] Skip event counts on taxa for now --- ami/main/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 3c161a1f6..aa28ca236 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -598,7 +598,7 @@ def get_queryset(self) -> QuerySet: qs = qs.prefetch_related("occurrences") qs = qs.annotate( occurrences_count=models.Count("occurrences", distinct=True), - events_count=models.Count("occurrences__event", distinct=True), + # events_count=models.Count("occurrences__event", distinct=True), last_detected=models.Max("classifications__detection__timestamp"), ) elif self.action == "list": From 9593c2a9cd909ab5d4425d11bca8607a11e5d5b2 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 07:33:12 +0000 Subject: [PATCH 6/7] Fixes to deployment counts --- .envs/.local/.django | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.envs/.local/.django b/.envs/.local/.django index 8e1cf358c..2351d52f9 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -12,3 +12,5 @@ REDIS_URL=redis://redis:6379/0 # Flower CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf + +SENDGRID_API_KEY="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" From 88102f71c148b7b08b0e4b0f601bf7f9942c2a5e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 11 Dec 2023 07:37:26 +0000 Subject: [PATCH 7/7] Update local env config --- .envs/.local/.django | 2 -- ami/main/api/views.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.envs/.local/.django b/.envs/.local/.django index 2351d52f9..8e1cf358c 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -12,5 +12,3 @@ REDIS_URL=redis://redis:6379/0 # Flower CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf - -SENDGRID_API_KEY="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" diff --git a/ami/main/api/views.py b/ami/main/api/views.py index aa28ca236..273efe3bd 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -186,7 +186,7 @@ def get_queryset(self) -> QuerySet: # detections_count=models.Count("captures__detections", distinct=True), occurrences_count=models.Count("occurrences", distinct=True), taxa_count=models.Count("occurrences__determination", distinct=True), - ) + ).prefetch_related("occurrences", "occurrences__determination") return qs @@ -608,7 +608,7 @@ def get_queryset(self) -> QuerySet: qs = qs.prefetch_related(Prefetch("occurrences", queryset=Occurrence.objects.none())) qs = qs.annotate( occurrences_count=models.Value(0), - events_count=models.Value(0), + # events_count=models.Value(0), last_detected=models.Value(None, output_field=models.DateTimeField()), )