From 86d923806de3e84f7c3a212a96158422b7c2c224 Mon Sep 17 00:00:00 2001 From: "Yassine R." Date: Mon, 20 Jan 2025 19:10:04 +0100 Subject: [PATCH] fix(backend): free memory and useless data for stats & exports --- .../1737390477602-manual-migration.ts | 20 ++++ .../controllers/stats.private.controller.ts | 55 ++++++----- .../dto/structure-stats-reporting.dto.ts | 11 ++- .../export-structure-usagers.controller.ts | 14 +-- .../src/usagers/services/usagers.service.ts | 21 ++-- .../reporting-form.component.html | 2 +- .../reporting-form.component.ts | 2 +- .../structure-stats.component.html | 16 ++-- .../structure-stats.component.ts | 96 ++++++++++--------- packages/frontend/src/assets/files/news.json | 38 ++++++++ 10 files changed, 175 insertions(+), 100 deletions(-) create mode 100644 packages/backend/src/_migrations/1737390477602-manual-migration.ts diff --git a/packages/backend/src/_migrations/1737390477602-manual-migration.ts b/packages/backend/src/_migrations/1737390477602-manual-migration.ts new file mode 100644 index 0000000000..38624ea425 --- /dev/null +++ b/packages/backend/src/_migrations/1737390477602-manual-migration.ts @@ -0,0 +1,20 @@ +import { IsNull, MigrationInterface } from "typeorm"; +import { structureStatsReportingQuestionsRepository } from "../database"; +import { domifaConfig } from "../config"; + +export class ManualMigration1737390477602 implements MigrationInterface { + public async up(): Promise { + if ( + domifaConfig().envId === "prod" || + domifaConfig().envId === "preprod" || + domifaConfig().envId === "local" + ) { + console.log("Delete useless data in questions"); + await structureStatsReportingQuestionsRepository.delete({ + confirmationDate: IsNull(), + }); + } + } + + public async down(): Promise {} +} diff --git a/packages/backend/src/stats/controllers/stats.private.controller.ts b/packages/backend/src/stats/controllers/stats.private.controller.ts index a94e2a0772..2316d815c9 100644 --- a/packages/backend/src/stats/controllers/stats.private.controller.ts +++ b/packages/backend/src/stats/controllers/stats.private.controller.ts @@ -27,9 +27,15 @@ import { structureStatsInPeriodGenerator } from "../services"; import { statsQuestionsCoreBuilder } from "../services/statsQuestionsCoreBuilder.service"; import { addDays, format } from "date-fns"; -import { StructureStatsFull } from "@domifa/common"; +import { + StructureStatsFull, + StructureStatsReportingQuestions, +} from "@domifa/common"; import { StructureStatsReportingDto } from "../dto"; -import { structureStatsReportingQuestionsRepository } from "../../database"; +import { + structureStatsReportingQuestionsRepository, + StructureStatsReportingQuestionsTable, +} from "../../database"; import { AppLogsService } from "../../modules/app-logs/app-logs.service"; @Controller("stats") @@ -52,30 +58,31 @@ export class StatsPrivateController { }, }); - if (!stats) { - throw new Error("CANNOT_UPDATE_REPORT"); - } - - await structureStatsReportingQuestionsRepository.update( - { - structureId: user.structureId, - year: reportingDto.year, + const questionsToAdd: StructureStatsReportingQuestions = { + ...reportingDto, + completedBy: { + id: user.id, + nom: user.nom, + prenom: user.prenom, }, - { - waitingList: reportingDto.waitingList, - waitingTime: reportingDto.waitingTime, - workers: reportingDto.workers, - volunteers: reportingDto.volunteers, - humanCosts: reportingDto.humanCosts, - totalCosts: reportingDto.totalCosts, - completedBy: { - id: user.id, - nom: user.nom, - prenom: user.prenom, + structureId: user.structureId, + confirmationDate: new Date(), + }; + + if (!stats) { + await structureStatsReportingQuestionsRepository.save( + new StructureStatsReportingQuestionsTable(questionsToAdd) + ); + } else { + await structureStatsReportingQuestionsRepository.update( + { + structureId: user.structureId, + year: reportingDto.year, }, - confirmationDate: new Date(), - } - ); + questionsToAdd + ); + } + return; } @AllowUserStructureRoles("responsable", "admin", "simple") diff --git a/packages/backend/src/stats/dto/structure-stats-reporting.dto.ts b/packages/backend/src/stats/dto/structure-stats-reporting.dto.ts index 85417dcb79..d2c6a9ad55 100644 --- a/packages/backend/src/stats/dto/structure-stats-reporting.dto.ts +++ b/packages/backend/src/stats/dto/structure-stats-reporting.dto.ts @@ -19,19 +19,26 @@ export class StructureStatsReportingDto { @IsNumber() @IsOptional() + @Max(1000) workers: number | null; @IsNumber() + @Min(0) + @Max(10000) volunteers: number | null; @IsNumber() + @Max(1000000) + @Min(0) humanCosts: number | null; @IsNumber() + @Max(1000000) + @Min(0) totalCosts: number | null; @IsNumber() - @Min(2020) - @Max(2035) + @Min(new Date().getFullYear() - 5) // 5 years ago + @Max(new Date().getFullYear()) // current year year: number; } diff --git a/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts b/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts index 1c37b089cc..3ba47a90d3 100644 --- a/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts +++ b/packages/backend/src/usagers/controllers/export-structure-usagers.controller.ts @@ -89,11 +89,6 @@ export class ExportStructureUsagersController { let currentRowEntretiens = 2; const processChunk = async (chunk: StructureUsagerExport[]) => { - console.log({ - date: new Date(), - structureId: user.structureId, - currentRowUsagers, - }); const { firstSheetUsagers, secondSheetEntretiens } = renderStructureUsagersRows(chunk, user.structure); @@ -108,14 +103,15 @@ export class ExportStructureUsagersController { "DATE_DERNIER_PASSAGE", ]); - // Ajoute les lignes à la première feuille XLSX.utils.sheet_add_json(wsUsagers, firstSheetUsagers, { skipHeader: true, origin: currentRowUsagers, cellDates: true, dateNF: "DD/MM/YYYY", }); + currentRowUsagers += firstSheetUsagers.length; + firstSheetUsagers.length = 0; applyDateFormat(secondSheetEntretiens, ["USAGER_DATE_NAISSANCE"]); @@ -125,13 +121,13 @@ export class ExportStructureUsagersController { cellDates: true, dateNF: "DD/MM/YYYY", }); - currentRowEntretiens += secondSheetEntretiens.length; + secondSheetEntretiens.length = 0; }; await this.usagersService.exportByChunks( - user.structureId, - 5000, + user, + 2000, statut, processChunk ); diff --git a/packages/backend/src/usagers/services/usagers.service.ts b/packages/backend/src/usagers/services/usagers.service.ts index 1cba17f810..cb0dfabef0 100644 --- a/packages/backend/src/usagers/services/usagers.service.ts +++ b/packages/backend/src/usagers/services/usagers.service.ts @@ -249,7 +249,7 @@ export class UsagersService { } public async exportByChunks( - structureId: number, + user: Pick, chunkSize: number = 5000, statut: UsagersFilterCriteriaStatut, processChunk: (chunk: StructureUsagerExport[]) => Promise @@ -259,7 +259,7 @@ export class UsagersService { let whereClause = 'WHERE u."structureId" = $1'; let countWhereClause = `${whereClause}`; - const countParams: any[] = [structureId]; + const countParams: any[] = [user.structureId]; if (statut !== UsagersFilterCriteriaStatut.TOUS) { countWhereClause += ` AND u.statut = $2`; @@ -329,8 +329,8 @@ export class UsagersService { LIMIT $2 OFFSET $3 `; - while (true) { - const queryParams: any[] = [structureId, chunkSize, skip]; + while (total < count) { + const queryParams: any[] = [user.structureId, chunkSize, skip]; if (statut !== UsagersFilterCriteriaStatut.TOUS) { queryParams.push(statut); } @@ -345,17 +345,22 @@ export class UsagersService { total += chunk.length; skip += chunk.length; - console.log({ + console.table({ + timestamp: new Date().toISOString(), processedCount: total, - totalCount: count, + totalCount: parseInt(count, 10), + progression: `${Math.round((total / parseInt(count, 10)) * 100)}%`, chunkSize: chunk.length, skip, - structureId, + userId: user.id, + structureId: user.structureId, }); + + chunk.length = 0; } if (total !== parseInt(count)) { - console.warn( + console.log( `⚠️ Différence détectée - Exportés: ${total}, Total attendu: ${count}` ); } diff --git a/packages/frontend/src/app/modules/structure-stats/components/reporting-form/reporting-form.component.html b/packages/frontend/src/app/modules/structure-stats/components/reporting-form/reporting-form.component.html index 9b26b018fe..6ca38565c1 100644 --- a/packages/frontend/src/app/modules/structure-stats/components/reporting-form/reporting-form.component.html +++ b/packages/frontend/src/app/modules/structure-stats/components/reporting-form/reporting-form.component.html @@ -150,7 +150,7 @@

- +

(); public readonly REPORTNG_QUESTIONS_LABELS = REPORTNG_QUESTIONS_LABELS; diff --git a/packages/frontend/src/app/modules/structure-stats/components/structure-stats/structure-stats.component.html b/packages/frontend/src/app/modules/structure-stats/components/structure-stats/structure-stats.component.html index 24b76221a9..b9c98dd090 100644 --- a/packages/frontend/src/app/modules/structure-stats/components/structure-stats/structure-stats.component.html +++ b/packages/frontend/src/app/modules/structure-stats/components/structure-stats/structure-stats.component.html @@ -40,15 +40,15 @@