fix(backend): export of large files

packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts

@@ -29,6 +29,38 @@ import { domifaConfig } from "../../config";
 
 import { UsagersFilterCriteriaStatut } from "@domifa/common";
 
+let lastCpuUsage = process.cpuUsage();
+let lastTime = Date.now();
+
+const logProcessState = (label: string) => {
+  const used = process.memoryUsage();
+  const currentCpu = process.cpuUsage();
+  const currentTime = Date.now();
+
+  // Calcul du temps écoulé en microseconds
+  const elapsedTime = (currentTime - lastTime) * 1000; // conversion en microsecondes
+
+  // Différence avec la mesure précédente
+  const userDiff = currentCpu.user - lastCpuUsage.user;
+  const systemDiff = currentCpu.system - lastCpuUsage.system;
+
+  // Calcul du pourcentage (temps CPU / temps écoulé)
+  const userPercent = Math.round((userDiff / elapsedTime) * 100);
+  const systemPercent = Math.round((systemDiff / elapsedTime) * 100);
+
+  const processInfo = {
+    État: label,
+    Heure: new Date().toISOString(),
+    Mémoire: `${Math.round(used.rss / 1024 / 1024)} Mo`,
+    "CPU User": `${userPercent}%`,
+    "CPU System": `${systemPercent}%`,
+  };
+
+  console.table(processInfo);
+  lastCpuUsage = currentCpu;
+  lastTime = currentTime;
+};
+
 @UseGuards(AuthGuard("jwt"), AppUserGuard)
 @ApiTags("export")
 @ApiBearerAuth()
@@ -48,6 +80,7 @@ export class ExportStructureUsagersController {
     statut: UsagersFilterCriteriaStatut
   ): Promise<void> {
     const startTime = new Date();
+    logProcessState("Export started");
 
     try {
       await this.appLogsService.create({
@@ -88,7 +121,12 @@ export class ExportStructureUsagersController {
       let currentRowUsagers = 2;
       let currentRowEntretiens = 2;
 
-      const processChunk = async (chunk: StructureUsagerExport[]) => {
+      let processChunk = async (
+        chunk: StructureUsagerExport[],
+        count: number
+      ) => {
+        logProcessState(`Processing chunk (${currentRowUsagers}/${count})`);
+
         const { firstSheetUsagers, secondSheetEntretiens } =
           renderStructureUsagersRows(chunk, user.structure);
 
@@ -103,6 +141,8 @@ export class ExportStructureUsagersController {
           "DATE_DERNIER_PASSAGE",
         ]);
 
+        logProcessState(`Apply date format to first Sheet`);
+
         XLSX.utils.sheet_add_json(wsUsagers, firstSheetUsagers, {
           skipHeader: true,
           origin: currentRowUsagers,
@@ -121,17 +161,32 @@ export class ExportStructureUsagersController {
           cellDates: true,
           dateNF: "DD/MM/YYYY",
         });
+
         currentRowEntretiens += secondSheetEntretiens.length;
         secondSheetEntretiens.length = 0;
+
+        logProcessState(`XLSX.utils.sheet_add_json for second sheet done`);
       };
 
+      logProcessState(`Call to exportByChunks`);
+
       await this.usagersService.exportByChunks(
         user,
         2000,
         statut,
         processChunk
       );
 
+      // Clean some memory
+      wsUsagers.length = 0;
+      wsEntretiens.length = 0;
+      firstSheetHeaders.length = 0;
+      secondSheetHeaders.length = 0;
+      processChunk = null;
+
+      logProcessState(`✅ exportByChunks Done`);
+      logProcessState(`Start  XLSX.write`);
+
       const buffer = XLSX.write(workbook, {
         bookType: "xlsx",
         type: "buffer",
@@ -140,12 +195,16 @@ export class ExportStructureUsagersController {
         WTF: false,
       });
 
+      logProcessState(`✅  XLSX.write DONE, buffer is ready`);
+
       appLogger.info(
         `[EXPORT] [${domifaConfig().envId}] completed in ${
           Date.now() - startTime.getTime()
         }ms`
       );
 
+      logProcessState(`Ready to send buffer`);
+
       res.setHeader(
         "Content-Type",
         "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"

packages/backend/src/usagers/services/usagers.service.ts

@@ -252,7 +252,10 @@ export class UsagersService {
     user: Pick<UserStructureAuthenticated, "id" | "structureId" | "prenom">,
     chunkSize: number = 5000,
     statut: UsagersFilterCriteriaStatut,
-    processChunk: (chunk: StructureUsagerExport[]) => Promise<void>
+    processChunk: (
+      chunk: StructureUsagerExport[],
+      count: number
+    ) => Promise<void>
   ): Promise<void> {
     let skip = 0;
     let total = 0;
@@ -341,14 +344,12 @@ export class UsagersService {
         break;
       }
 
-      await processChunk(chunk);
+      await processChunk(chunk, total);
       total += chunk.length;
       skip += chunk.length;
 
       console.table({
-        timestamp: new Date().toISOString(),
-        processedCount: total,
-        totalCount: parseInt(count, 10),
+        totalCount: `${total}/${parseInt(count, 10)}`,
         progression: `${Math.round((total / parseInt(count, 10)) * 100)}%`,
         chunkSize: chunk.length,
         skip,

packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/applyDateFormat.ts

@@ -1,17 +1,29 @@
 import { StructureCustomDocTags } from "../../../_common/model";
-import { isValid, parse } from "date-fns";
+import { parse } from "date-fns";
+
 export const applyDateFormat = (
   worksheet: StructureCustomDocTags[],
   elements: Array<keyof StructureCustomDocTags>
 ): void => {
-  worksheet.forEach((ws: StructureCustomDocTags) => {
-    elements.forEach((element: keyof StructureCustomDocTags) => {
-      if (ws[element]) {
-        const value = ws[element] as string;
-        ws[element] = isValid(parse(value, "dd/MM/yyyy", new Date()))
-          ? parse(value, "dd/MM/yyyy", new Date())
-          : value;
+  const baseDate = new Date();
+  const dateFormat = "dd/MM/yyyy";
+
+  for (let i = 0; i < worksheet.length; i++) {
+    const ws = worksheet[i];
+
+    for (let j = 0; j < elements.length; j++) {
+      const element = elements[j];
+      const value = ws[element];
+
+      if (!value) continue;
+
+      try {
+        const parsedDate = parse(value as string, dateFormat, baseDate);
+        ws[element] =
+          parsedDate.toString() !== "Invalid Date" ? parsedDate : value;
+      } catch {
+        continue;
       }
-    });
-  });
+    }
+  }
 };