procrastinate queues

config/settings/base.py

@@ -50,6 +50,8 @@
 DATABASES["default"]["ENGINE"] = "psqlextra.backend"
 POSTGRES_EXTRA_DB_BACKEND_BASE = "django.contrib.gis.db.backends.postgis"
 DATABASES["default"]["ATOMIC_REQUESTS"] = True
+DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=None)
+
 # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-DEFAULT_AUTO_FIELD
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
@@ -106,6 +108,7 @@
     "slippers",
     "fontawesomefree",
     "leaflet",
+    "procrastinate.contrib.django",
 ]
 
 LOCAL_APPS = [
@@ -406,3 +409,6 @@
 DRF_STANDARDIZED_ERRORS = {"ENABLE_IN_DEBUG_FOR_UNHANDLED_EXCEPTIONS": True}
 
 TAILWIND_APP_NAME = "metadata_catalogue.theme"
+
+IPT_SOURCES = env.list("IPT_SOURCES", default=[])
+IPT_SOURCES_CRON = env("IPT_SOURCES_CRON", default="0 0 * * 0")

docker-compose.yml

@@ -10,6 +10,7 @@ x-django-env: &django-env
   DJANGO_ACCOUNT_ALLOW_REGISTRATION: "True"
   USE_DOCKER: "yes"
   DJANGO_BASE_SCHEMA_URL: "http://django:8000"
+  IPT_SOURCES: "https://ipt.nina.no"
 
 x-django-prod-env: &django-prod-env
   <<: *django-env
@@ -93,19 +94,19 @@ services:
       <<: *django-dev-env
       DJANGO_TAILWIND: 1
 
-  # queue:
-  #   <<: *django-prod
-  #   environment:
-  #     <<: *django-prod-env
-  #     WAIT_FOR_HTTP: http://django:8000/ht/
-  #   command: python manage.py qcluster
-
-  # queue-dev:
-  #   <<: *django-dev
-  #   environment:
-  #     <<: *django-dev-env
-  #     WAIT_FOR_HTTP: http://django:8000/ht/
-  #   command: python manage.py qcluster
+  queue:
+    <<: *django-prod
+    environment:
+      <<: *django-prod-env
+      WAIT_FOR_HTTP: http://django:8000/ht/
+    command: python manage.py procrastinate worker
+
+  queue-dev:
+    <<: *django-dev
+    environment:
+      <<: *django-dev-env
+      WAIT_FOR_HTTP: http://django:8000/ht/
+    command: python manage.py procrastinate worker
 
   postgres:
     image: postgis/postgis:16-3.4

metadata_catalogue/datasets/libs/ipt.py

@@ -3,8 +3,8 @@
 
 from bs4 import BeautifulSoup
 from django.db import transaction
-from metadata_catalogue.core.utils import async_task
 from metadata_catalogue.datasets.models import Dataset
+from procrastinate.contrib.django import app
 
 logger = logging.getLogger(__name__)
 
@@ -24,10 +24,9 @@ def rss_to_datasets(rss_content):
                         },
                         fetch_url=archive.text,
                     )
-                    async_task(
-                        "metadata_catalogue.datasets.libs.harvesters.harvest_dataset",
-                        d.id,
-                    )
+                    app.configure_task(
+                        name="harvest_dataset",
+                    ).defer(dataset_id=d.id)
                 else:
                     logger.warn(f'no archive url found for {item.find("title").text}')
             except Exception:

metadata_catalogue/datasets/models.py

@@ -13,7 +13,7 @@
     hook,
 )
 from django.urls import reverse
-from metadata_catalogue.core.utils import async_task
+from procrastinate.contrib.django import app
 from solo.models import SingletonModel
 
 from .libs.iso.mapping import ISOMapping
@@ -498,7 +498,7 @@ class Content(LifecycleModel):
 
     @hook(AFTER_SAVE, when_any=["gdal_vrt_definition"], has_changed=True)
     def check_is_valid(self):
-        async_task("metadata_catalogue.datasets.libs.checks.validate_vrt", self.id)
+        app.configure_task(name="validate_vrt").defer(content_id=self.id)
 
     def __str__(self):
         return str(self.dataset)

metadata_catalogue/datasets/tasks.py

@@ -0,0 +1,24 @@
+from procrastinate.contrib.django import app
+from metadata_catalogue.datasets.libs.harvesters import harvest_ipt, harvest_dataset
+from metadata_catalogue.datasets.libs.checks import validate_vrt
+
+from django.conf import settings
+import logging
+
+
+@app.periodic(cron=settings.IPT_SOURCES_CRON)
+@app.task(name="harvest_ipt")
+def harvest_ipt_task(timestamp: int):
+    for url in settings.IPT_SOURCES:
+        logging.info(f"fetching {url}")
+        harvest_ipt(ipt_url=url)
+
+
+@app.task(name="harvest_dataset")
+def harvest_dataset_task(dataset_id: int):
+    harvest_dataset(dataset_id=dataset_id)
+
+
+@app.task(name="validate_vrt")
+def validate_vrt_task(content_id):
+    validate_vrt(content_id=content_id)

pyproject.toml

@@ -168,6 +168,7 @@ dependencies = [
     "django-tables2",
     "django-leaflet",
     "setuptools>=75.2.0",
+    "procrastinate[django]>=2.15.0",
 ]
 requires-python = ">=3.10"
 name = ""