Merge pull request #514 from rcpch/eatyourpeas/issue509

jersify

project/constants/csv_headings.py

@@ -1,10 +1,22 @@
-CSV_HEADINGS = (
-    # patient
+UNIQUE_IDENTIFIER_ENGLAND = (
     {
         "heading": "NHS Number",
         "model_field": "nhs_number",
         "model": "Patient",
     },
+)
+
+UNIQUE_IDENTIFIER_JERSEY = (
+    {
+        "heading": "Unique Reference Number",
+        "model_field": "unique_reference_number",
+        "model": "Patient",
+    },
+)
+
+
+CSV_HEADING_OBJECTS = (
+    # patient
     {
         "heading": "Date of Birth",
         "model_field": "date_of_birth",
@@ -256,7 +268,7 @@
     },
 )
 
-HEADINGS_LIST = [item["heading"] for item in CSV_HEADINGS]
+# HEADINGS_LIST = [item["heading"] for item in CSV_HEADINGS]
 
 ALL_DATES = [
     "Date of Birth",
@@ -322,8 +334,13 @@
     ("hospital_discharge_date", "Discharge date (Hospital provider spell)"),
 ]
 
-CSV_DATA_TYPES_MINUS_DATES = {
+JERSEY_CSV_DATA_TYPES = {"Unique Reference Number": "string"}
+
+ENGLAND_CSV_DATA_TYPES = {
     "NHS Number": "string",
+}
+
+CSV_DATA_TYPES_MINUS_DATES = {
     "Postcode of usual address": "string",
     "Stated gender": "Int8",
     "Ethnic Category": "string",  # choices are all capital letters
@@ -356,7 +373,6 @@
 }
 
 NONNULL_FIELDS = [
-    "NHS Number",
     "Date of Birth",
     "Diabetes Type",
     "PDU Number",

project/npda/admin.py

@@ -15,7 +15,6 @@
 PaediatricDiabetesUnit = apps.get_model("npda", "PaediatricDiabetesUnit")
 
 
-
 @admin.register(OrganisationEmployer)
 class OrganisationEmployerAdmin(admin.ModelAdmin):
     search_fields = ("name", "pk", "lead_organisation_ods_code", "pz_code")
@@ -28,7 +27,7 @@ class NPDAUserAdmin(admin.ModelAdmin):
 
 @admin.register(Patient)
 class PatientAdmin(admin.ModelAdmin):
-    search_fields = ("nhs_number_icontains", "pk")
+    search_fields = ("nhs_number_icontains", "pk", "unique_reference_number_icontains")
 
 
 @admin.register(PaediatricDiabetesUnit)

project/npda/context_processors.py

@@ -10,6 +10,7 @@ def session_data(request):
         ),
         "can_upload_csv": request.session.get("can_upload_csv", False),
         "pz_code": request.session.get("pz_code", None),
+        "lead_organisation": request.session.get("lead_organisation", None),
         "requested_audit_year": request.session.get("requested_audit_year", None),
         "audit_years": request.session.get("audit_years", []),
         "lead_organisation": request.session.get("lead_organisation", None),

project/npda/dummy_sheets/dummy_sheet_invalid.csv

@@ -1,5 +1,5 @@
 NHS Number,Date of Birth,Postcode of usual address,Stated gender,Ethnic Category,Diabetes Type,Date of Diabetes Diagnosis,Date of leaving service,Reason for leaving service,Death Date,GP Practice Code,PDU Number,Visit/Appointment Date,Patient Height (cm),Patient Weight (kg),Observation Date (Height and weight),Hba1c Value,HbA1c result format,Observation Date: Hba1c Value,Diabetes Treatment at time of Hba1c measurement,"If treatment included insulin pump therapy (i.e. option 3 or 6 selected), was this part of a closed loop system?","At the time of HbA1c measurement, in addition to standard blood glucose monitoring (SBGM), was the patient using any other method of glucose monitoring?",Systolic Blood Pressure,Diastolic Blood pressure,Observation Date (Blood Pressure),Foot Assessment / Examination Date,Retinal Screening date,Retinal Screening Result,Urinary Albumin Level (ACR),Observation Date: Urinary Albumin Level,Albuminuria Stage,Total Cholesterol Level (mmol/l),Observation Date: Total Cholesterol Level,Observation Date: Thyroid Function,"At time of, or following measurement of thyroid function, was the patient prescribed any thyroid treatment?",Observation Date: Coeliac Disease Screening,Has the patient been recommended a Gluten-free diet?,Observation Date - Psychological Screening Assessment,Was the patient assessed as requiring additional psychological/CAMHS support outside of MDT clinics?,Does the patient smoke?,Date of offer of referral to smoking cessation service (if patient is a current smoker),Date of Level 3 carbohydrate counting education received,Was the patient offered an additional appointment with a paediatric dietitian?,Date of additional appointment with dietitian,Was the patient using (or trained to use) blood ketone testing equipment at time of visit?,Date that influenza immunisation was recommended,Date of provision of advice ('sick-day rules') about managing diabetes during intercurrent illness or episodes of hyperglycaemia,Start date (Hospital Provider Spell),Discharge date (Hospital provider spell),Reason for admission,Only complete if DKA selected in previous question: During this DKA admission did the patient receive any of the following therapies?,Only complete if OTHER selected: Reason for admission (free text)
-719 573 0220,26/10/2020,S43 1BN,1,N,1,01/05/2021,,,,G85004,PZ041,11/06/2024,158,58,22/11/2022,50,1,22/11/2022,7,,TRUE,111,83,22/11/2022,22/11/2022,14/06/2022,2,0,22/11/2022,1,4.9,22/11/2022,22/11/2022,TRUE,02/01/2018,FALSE,16/10/2021,TRUE,FALSE,,02/06/2023,TRUE,13/07/2023,FALSE,03/03/2023,03/03/2023,,,3,2,
+123456789,26/10/2020,S43 1BN,1,N,1,01/05/2021,,,,G85004,PZ041,11/06/2024,158,58,22/11/2022,50,1,22/11/2022,7,,TRUE,111,83,22/11/2022,22/11/2022,14/06/2022,2,0,22/11/2022,1,4.9,22/11/2022,22/11/2022,TRUE,02/01/2018,FALSE,16/10/2021,TRUE,FALSE,,02/06/2023,TRUE,13/07/2023,FALSE,03/03/2023,03/03/2023,,,3,2,
 653 765 1948,03/10/2015,BS10 5AH,1,L,1,08/12/2015,,,,G85004,PZ041,19/10/2024,144,40,06/07/2016,59,1,06/07/2016,3,FALSE,FALSE,103,47,06/07/2016,,13/04/2017,2,0,06/07/2016,1,4.9,06/07/2016,,,,TRUE,13/09/2015,TRUE,TRUE,01/09/2022,10/02/2019,TRUE,10/07/2023,FALSE,09/01/2017,09/01/2017,05/11/2016,09/11/2016,1,3,
 758 447 7372,06/12/2016,IV15 9NZ,1,Z,1,26/08/2019,,,,G85004,PZ041,11/05/2024,160,13,01/01/2023,48,1,01/01/2023,8,,TRUE,91,57,01/01/2023,,01/10/2019,1,0,01/01/2023,99,5.7,01/01/2023,01/01/2023,TRUE,01/01/2023,FALSE,25/11/2022,FALSE,TRUE,11/11/2020,21/07/2022,FALSE,,FALSE,09/08/2022,09/08/2022,,,6,3,
 385 576 6320,18/07/2015,BL8 9RU,1,G,1,13/03/2022,,,,G85004,PZ041,02/07/2024,125,19,25/04/2022,103,1,25/04/2022,3,TRUE,TRUE,99,47,25/04/2022,25/04/2022,25/09/2023,2,0,25/04/2022,1,4.9,25/04/2022,,,,FALSE,30/10/2022,FALSE,TRUE,03/06/2022,23/11/2022,FALSE,,FALSE,12/05/2022,12/05/2022,11/08/2022,15/08/2022,4,4,

project/npda/dummy_sheets/npda_csv_submission_template_for_use_from_april_2025.csv

@@ -0,0 +1 @@
+Unique Reference Number,Date of Birth,Postcode of usual address,Stated gender,Ethnic Category,Diabetes Type,Date of Diabetes Diagnosis,Date of leaving service,Reason for leaving service,Death Date,GP Practice Code,PDU Number,Visit/Appointment Date,Patient Height (cm),Patient Weight (kg),Observation Date (Height and weight),Hba1c Value,HbA1c result format,Observation Date: Hba1c Value,Diabetes Treatment at time of Hba1c measurement,"If treatment included insulin pump therapy (i.e. option 3 or 6 selected), was this part of a closed loop system?","At the time of HbA1c measurement, in addition to standard blood glucose monitoring (SBGM), was the patient using any other method of glucose monitoring?",Systolic Blood Pressure,Diastolic Blood pressure,Observation Date (Blood Pressure),Foot Assessment / Examination Date,Retinal Screening date,Retinal Screening Result,Urinary Albumin Level (ACR),Observation Date: Urinary Albumin Level,Albuminuria Stage,Total Cholesterol Level (mmol/l),Observation Date: Total Cholesterol Level,Observation Date: Thyroid Function,"At time of, or following measurement of thyroid function, was the patient prescribed any thyroid treatment?",Observation Date: Coeliac Disease Screening,Has the patient been recommended a Gluten-free diet?,Observation Date - Psychological Screening Assessment,Was the patient assessed as requiring additional psychological/CAMHS support outside of MDT clinics?,Does the patient smoke?,Date of offer of referral to smoking cessation service (if patient is a current smoker),Date of Level 3 carbohydrate counting education received,Was the patient offered an additional appointment with a paediatric dietitian?,Date of additional appointment with dietitian,Was the patient using (or trained to use) blood ketone testing equipment at time of visit?,Date that influenza immunisation was recommended,Date of provision of advice ('sick-day rules') about managing diabetes during intercurrent illness or episodes of hyperglycaemia,Start date (Hospital Provider Spell),Discharge date (Hospital provider spell),Reason for admission,Only complete if DKA selected in previous question: During this DKA admission did the patient receive any of the following therapies?,Only complete if OTHER selected: Reason for admission (free text)

project/npda/forms/patient_form.py

@@ -14,9 +14,8 @@
 from django.apps import apps
 from django.core.exceptions import ValidationError
 
-from project.constants.leave_pdu_reasons import LEAVE_PDU_REASONS
-
 from ...constants.styles.form_styles import *
+from ...constants import LEAVE_PDU_REASONS
 from ..models import Patient, Transfer
 from ..validators import not_in_the_future_validator
 from .external_patient_validators import validate_patient_sync
@@ -30,6 +29,8 @@ class DateInput(forms.DateInput):
 
 class NHSNumberField(forms.CharField):
     def to_python(self, value):
+        if not value:
+            return value
         number = super().to_python(value)
         normalised = nhs_number.standardise_format(number)
 
@@ -41,6 +42,18 @@ def validate(self, value):
             raise ValidationError("Invalid NHS number")
 
 
+class UniqueReferenceNumberField(forms.CharField):
+    def to_python(self, value):
+        if not value:
+            return value
+        number = super().to_python(value)
+        return number
+
+    def validate(self, value):
+        if value and not value.isdigit():
+            raise ValidationError("Invalid Unique Reference Number")
+
+
 class PostcodeField(forms.CharField):
     def to_python(self, value):
         postcode = super().to_python(value)
@@ -60,6 +73,7 @@ class Meta:
         model = Patient
         fields = [
             "nhs_number",
+            "unique_reference_number",
             "sex",
             "date_of_birth",
             "postcode",
@@ -72,13 +86,17 @@ class Meta:
         ]
         field_classes = {
             "nhs_number": NHSNumberField,
+            "unique_reference_number": UniqueReferenceNumberField,
             "postcode": PostcodeField,
             "gp_practice_postcode": PostcodeField,
         }
         widgets = {
             "nhs_number": forms.TextInput(
                 attrs={"class": TEXT_INPUT},
             ),
+            "unique_reference_number": forms.TextInput(
+                attrs={"class": TEXT_INPUT},
+            ),
             "sex": forms.Select(),
             "date_of_birth": DateInput(),
             "postcode": forms.TextInput(attrs={"class": TEXT_INPUT}),
@@ -163,6 +181,23 @@ def clean(self):
         gp_practice_ods_code = cleaned_data.get("gp_practice_ods_code")
         gp_practice_postcode = cleaned_data.get("gp_practice_postcode")
 
+        nhs_number = cleaned_data.get("nhs_number")
+        unique_reference_number = cleaned_data.get("unique_reference_number")
+
+        if not nhs_number and not unique_reference_number:
+            self.add_error(
+                "nhs_number",
+                ValidationError(
+                    "Either NHS Number or Unique Reference Number must be provided."
+                ),
+            )
+            self.add_error(
+                "unique_reference_number",
+                ValidationError(
+                    "Either NHS Number or Unique Reference Number must be provided."
+                ),
+            )
+
         reason_leaving_service = cleaned_data.get("reason_leaving_service")
         date_leaving_service = cleaned_data.get("date_leaving_service")
         if date_leaving_service and not reason_leaving_service:
@@ -246,6 +281,8 @@ def clean(self):
         ]:
             self.handle_async_validation_result(key)
 
+        return cleaned_data
+
     def save(self, commit=True):
         # We deliberately don't call super.save here as it throws ValueError on validation errors
         # and for CSV uploads we don't want that to stop us. As of Django 5.1.5 it doesn't do anything

project/npda/general_functions/csv/csv_download.py

@@ -6,6 +6,7 @@
 
 from ..write_errors_to_xlsx import write_errors_to_xlsx
 
+
 def download_csv(request, submission_id):
     """
     Download a CSV file.
@@ -14,9 +15,12 @@ def download_csv(request, submission_id):
     submission = get_object_or_404(Submission, id=submission_id)
 
     response = HttpResponse(submission.csv_file, content_type="text/csv")
-    response["Content-Disposition"] = f'attachment; filename="{submission.csv_file_name}"'
+    response["Content-Disposition"] = (
+        f'attachment; filename="{submission.csv_file_name}"'
+    )
     return response
 
+
 def download_xlsx(request, submission_id):
     """
     Download a XLSX file.
@@ -32,7 +36,9 @@ def download_xlsx(request, submission_id):
     if submission.errors:
         errors = json.loads(submission.errors)
 
-    xlsx_file = write_errors_to_xlsx(errors or {}, submission.csv_file)
+    is_jersey = submission.paediatric_diabetes_unit.pz_code == "PZ248"
+
+    xlsx_file = write_errors_to_xlsx(errors or {}, submission.csv_file, is_jersey)
 
     response = HttpResponse(xlsx_file, content_type="text/csv")
     response["Content-Disposition"] = f'attachment; filename="{xlsx_file_name}"'

project/npda/general_functions/csv/csv_header.py

@@ -1,11 +1,21 @@
 import csv
 import io
-from project.constants.csv_headings import HEADINGS_LIST
+from project.constants import (
+    CSV_HEADING_OBJECTS,
+    UNIQUE_IDENTIFIER_ENGLAND,
+    UNIQUE_IDENTIFIER_JERSEY,
+)
 
 
-def csv_header():
-    out = io.StringIO()
+def csv_header(is_jersey=False):
+    if is_jersey:
+        HEADINGS_LIST = UNIQUE_IDENTIFIER_JERSEY + CSV_HEADING_OBJECTS
+    else:
+        HEADINGS_LIST = UNIQUE_IDENTIFIER_ENGLAND + CSV_HEADING_OBJECTS
+
+    HEADINGS_LIST = [item["heading"] for item in HEADINGS_LIST]
 
+    out = io.StringIO()
     writer = csv.writer(out)
     writer.writerow(HEADINGS_LIST)
 

project/npda/general_functions/csv/csv_parse.py

@@ -14,8 +14,9 @@
 from project.constants import (
     ALL_DATES,
     CSV_DATA_TYPES_MINUS_DATES,
-    CSV_HEADINGS,
-    HEADINGS_LIST,
+    UNIQUE_IDENTIFIER_ENGLAND,
+    UNIQUE_IDENTIFIER_JERSEY,
+    CSV_HEADING_OBJECTS,
 )
 
 # Logging setup
@@ -31,7 +32,7 @@ class ParsedCSVFile:
     parse_type_error_columns: list[str]
 
 
-def csv_parse(csv_file):
+def csv_parse(csv_file, is_jersey=False):
     """
     Read the csv file and return a pandas dataframe
     Assigns the correct data types to the columns
@@ -44,6 +45,14 @@ def csv_parse(csv_file):
     # If it does, we will use the column names in the csv file
     # The exception is if the first row of the csv file does not match any of the predefined column names, in which case we will reject the csv
 
+    # Define the column names to be used in the csv file: the unique identifier in Jersy is different from the one in England
+    if is_jersey:
+        HEADINGS_LIST = UNIQUE_IDENTIFIER_JERSEY + CSV_HEADING_OBJECTS
+    else:
+        HEADINGS_LIST = UNIQUE_IDENTIFIER_ENGLAND + CSV_HEADING_OBJECTS
+
+    HEADINGS_LIST = [item["heading"] for item in HEADINGS_LIST]
+
     # Convert the predefined column names to lowercase
     lowercase_headings_list = [heading.lower() for heading in HEADINGS_LIST]
 
@@ -139,6 +148,33 @@ def csv_parse(csv_file):
             else:
                 parse_type_error_columns.append(column)
 
+    # Rows where the unique identifier is missing will be removed - count the number of rows before and after: placed here to ensure all columns have been processed
+    total_row_count = df.shape[0]
+    if is_jersey:
+        unique_reference_number_nonnull_row_count = df[
+            "Unique Reference Number"
+        ].count()
+        if unique_reference_number_nonnull_row_count == 0:
+            raise ValueError(
+                "No Unique Reference Numbers found in the file. Please ensure all rows have a unique identifier and upload the file again."
+            )
+        discrepancy = total_row_count - unique_reference_number_nonnull_row_count
+    else:
+        nhs_number_nonnull_row_count = df["NHS Number"].count()
+        if nhs_number_nonnull_row_count == 0:
+            raise ValueError(
+                "No NHS Numbers found in the file. Please ensure all rows have a unique identifier and upload the file again."
+            )
+        discrepancy = total_row_count - nhs_number_nonnull_row_count
+    if discrepancy > 0:
+        if discrepancy == 1:
+            raise ValueError(
+                f"{discrepancy} row has no unique identifier. Please ensure all rows have a unique identifier and upload the file again."
+            )
+        raise ValueError(
+            f"{discrepancy} rows have no unique identifier. Please ensure all rows have a unique identifier and upload the file again."
+        )
+
     return ParsedCSVFile(
         df,
         missing_columns,