From a15334222e34c9c7e243bb31ff1f130d4855ded6 Mon Sep 17 00:00:00 2001 From: "Yassine R." Date: Mon, 20 Jan 2025 23:38:34 +0100 Subject: [PATCH] fix(backend): export of large files --- .../export-structure-usagers.controller.ts | 61 ++++++++++++++++++- .../src/usagers/services/usagers.service.ts | 11 ++-- .../applyDateFormat.ts | 32 +++++++--- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts b/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts index 3ba47a90d3..67883b7a1e 100644 --- a/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts +++ b/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts @@ -29,6 +29,38 @@ import { domifaConfig } from "../../config"; import { UsagersFilterCriteriaStatut } from "@domifa/common"; +let lastCpuUsage = process.cpuUsage(); +let lastTime = Date.now(); + +const logProcessState = (label: string) => { + const used = process.memoryUsage(); + const currentCpu = process.cpuUsage(); + const currentTime = Date.now(); + + // Calcul du temps écoulé en microseconds + const elapsedTime = (currentTime - lastTime) * 1000; // conversion en microsecondes + + // Différence avec la mesure précédente + const userDiff = currentCpu.user - lastCpuUsage.user; + const systemDiff = currentCpu.system - lastCpuUsage.system; + + // Calcul du pourcentage (temps CPU / temps écoulé) + const userPercent = Math.round((userDiff / elapsedTime) * 100); + const systemPercent = Math.round((systemDiff / elapsedTime) * 100); + + const processInfo = { + État: label, + Heure: new Date().toISOString(), + Mémoire: `${Math.round(used.rss / 1024 / 1024)} Mo`, + "CPU User": `${userPercent}%`, + "CPU System": `${systemPercent}%`, + }; + + console.table(processInfo); + lastCpuUsage = currentCpu; + lastTime = currentTime; +}; + @UseGuards(AuthGuard("jwt"), AppUserGuard) @ApiTags("export") @ApiBearerAuth() @@ -48,6 +80,7 @@ export class ExportStructureUsagersController { statut: UsagersFilterCriteriaStatut ): Promise { const startTime = new Date(); + logProcessState("Export started"); try { await this.appLogsService.create({ @@ -88,7 +121,12 @@ export class ExportStructureUsagersController { let currentRowUsagers = 2; let currentRowEntretiens = 2; - const processChunk = async (chunk: StructureUsagerExport[]) => { + let processChunk = async ( + chunk: StructureUsagerExport[], + count: number + ) => { + logProcessState(`Processing chunk (${currentRowUsagers}/${count})`); + const { firstSheetUsagers, secondSheetEntretiens } = renderStructureUsagersRows(chunk, user.structure); @@ -103,6 +141,8 @@ export class ExportStructureUsagersController { "DATE_DERNIER_PASSAGE", ]); + logProcessState(`Apply date format to first Sheet`); + XLSX.utils.sheet_add_json(wsUsagers, firstSheetUsagers, { skipHeader: true, origin: currentRowUsagers, @@ -121,10 +161,15 @@ export class ExportStructureUsagersController { cellDates: true, dateNF: "DD/MM/YYYY", }); + currentRowEntretiens += secondSheetEntretiens.length; secondSheetEntretiens.length = 0; + + logProcessState(`XLSX.utils.sheet_add_json for second sheet done`); }; + logProcessState(`Call to exportByChunks`); + await this.usagersService.exportByChunks( user, 2000, @@ -132,6 +177,16 @@ export class ExportStructureUsagersController { processChunk ); + // Clean some memory + wsUsagers.length = 0; + wsEntretiens.length = 0; + firstSheetHeaders.length = 0; + secondSheetHeaders.length = 0; + processChunk = null; + + logProcessState(`✅ exportByChunks Done`); + logProcessState(`Start XLSX.write`); + const buffer = XLSX.write(workbook, { bookType: "xlsx", type: "buffer", @@ -140,12 +195,16 @@ export class ExportStructureUsagersController { WTF: false, }); + logProcessState(`✅ XLSX.write DONE, buffer is ready`); + appLogger.info( `[EXPORT] [${domifaConfig().envId}] completed in ${ Date.now() - startTime.getTime() }ms` ); + logProcessState(`Ready to send buffer`); + res.setHeader( "Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" diff --git a/packages/backend/src/usagers/services/usagers.service.ts b/packages/backend/src/usagers/services/usagers.service.ts index cb0dfabef0..e823be5b3e 100644 --- a/packages/backend/src/usagers/services/usagers.service.ts +++ b/packages/backend/src/usagers/services/usagers.service.ts @@ -252,7 +252,10 @@ export class UsagersService { user: Pick, chunkSize: number = 5000, statut: UsagersFilterCriteriaStatut, - processChunk: (chunk: StructureUsagerExport[]) => Promise + processChunk: ( + chunk: StructureUsagerExport[], + count: number + ) => Promise ): Promise { let skip = 0; let total = 0; @@ -341,14 +344,12 @@ export class UsagersService { break; } - await processChunk(chunk); + await processChunk(chunk, total); total += chunk.length; skip += chunk.length; console.table({ - timestamp: new Date().toISOString(), - processedCount: total, - totalCount: parseInt(count, 10), + totalCount: `${total}/${parseInt(count, 10)}`, progression: `${Math.round((total / parseInt(count, 10)) * 100)}%`, chunkSize: chunk.length, skip, diff --git a/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/applyDateFormat.ts b/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/applyDateFormat.ts index baedd33371..67d37e4ff9 100644 --- a/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/applyDateFormat.ts +++ b/packages/backend/src/usagers/services/xlsx-structure-usagers-renderer/applyDateFormat.ts @@ -1,17 +1,29 @@ import { StructureCustomDocTags } from "../../../_common/model"; -import { isValid, parse } from "date-fns"; +import { parse } from "date-fns"; + export const applyDateFormat = ( worksheet: StructureCustomDocTags[], elements: Array ): void => { - worksheet.forEach((ws: StructureCustomDocTags) => { - elements.forEach((element: keyof StructureCustomDocTags) => { - if (ws[element]) { - const value = ws[element] as string; - ws[element] = isValid(parse(value, "dd/MM/yyyy", new Date())) - ? parse(value, "dd/MM/yyyy", new Date()) - : value; + const baseDate = new Date(); + const dateFormat = "dd/MM/yyyy"; + + for (let i = 0; i < worksheet.length; i++) { + const ws = worksheet[i]; + + for (let j = 0; j < elements.length; j++) { + const element = elements[j]; + const value = ws[element]; + + if (!value) continue; + + try { + const parsedDate = parse(value as string, dateFormat, baseDate); + ws[element] = + parsedDate.toString() !== "Invalid Date" ? parsedDate : value; + } catch { + continue; } - }); - }); + } + } };