ci(tests): run backend unit tests on gitlab ci (#1052)

.env.test.travis.env → .env.test.ci.env

@@ -1,4 +1,4 @@
-# fichier .env utilisé pour les tests travis (docker-compose.travis.yml)
+# fichier .env utilisé pour les tests ci (docker-compose.ci.yml)
 DOMIFA_ENV_ID=test
 
 # MONGO DB

.gitlab-ci.yml

@@ -15,14 +15,83 @@ variables:
   PYTHON_VERSION: "3.6-alpine"
 
 stages:
-  - "Install"
+  - "Tests"
+  - "Build"
   - "Registration"
 
-#
+.tests_setup:
+  stage: Tests
+  image: node:14.15.0-alpine3.12
+  cache:
+    key:
+      files:
+        - yarn.lock
+      prefix: ${CI_JOB_NAME}
+    paths:
+      - packages/frontend/node_modules/.cache
+  services:
+    - name: mongo:4.0.3
+    - name: postgres:10.14
+  variables:
+    MONGO_INITDB_ROOT_USERNAME: root
+    MONGO_INITDB_ROOT_PASSWORD: root
+    DB_USER: root
+    DB_PASS: root
+    DB_NAME: domifa_test
+    POSTGRES_DB: nice_marmot
+    POSTGRES_USER: runner
+    POSTGRES_PASSWORD: abcd
+    POSTGRES_HOST_AUTH_METHOD: trust
+    DOMIFA_ENV_ID: test
+    SECRET: efgh
+    DOMIFA_ENV_ID: test
+    POSTGRES_USERNAME: runner
+    POSTGRES_DATABASE: domifa_test
+    #DOMIFA_PRINT_CONFIG: "true"
+  before_script:
+    - apk add mongodb-tools
+    - apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.8/main/ postgresql=10.12-r0
+    - nc -zvv mongo 27017
+    - pg_isready -d postgres -h postgres -p 5432 -U runner
+    - |
+      > ./.env.test.ci.env
+      > ./packages/backend/.env.backend.test.ci.env
+    - ./_scripts/db/restore-mongo.sh --db=test
+    - ./_scripts/db/restore-postgres.sh --db=test --ci
+    - yarn --frozen-lockfile
+
+Backend unit tests:
+  extends: .tests_setup
+  script:
+    - yarn --cwd packages/backend build
+    - yarn workspace @domifa/backend test:ci --coverage --detectOpenHandles --forceExit
+  needs: []
 
-Build:
+
+.Frontend unit tests:  # disabled until it is fixed
+  extends: .tests_setup
+  script:
+    - yarn build
+    - yarn workspace @domifa/backend start:tests-ci &
+    - yarn workspace @domifa/frontend test --coverage --detectOpenHandles --forceExit
+  needs: []
+
+
+.E2E tests: # disabled until it is fixed
+  extends: .tests_setup
+  script:
+    - yarn workspace @domifa/backend start:tests-ci &
+    - npx serve packages/frontend/dist/domifa -l 4200 &
+    - cd optional/e2e/runners/puppeteer/
+    - yarn --frozen-lockfile
+    - yarn test --debug
+  needs: []
+
+
+
+Build frontend image:
   extends: .base_yarn_script
-  stage: Install
+  stage: Build
   image: node:13
   interruptible: true
   cache:
@@ -32,21 +101,44 @@ Build:
       prefix: ${CI_JOB_NAME}
     paths:
       - packages/frontend/node_modules/.cache
-  services:
-    - name: mongo:4.0.3
   before_script:
     # NOTE(douglasduteil): yarn caching
     - yarn config set cache-folder $CI_PROJECT_DIR/.cache/yarn
   script:
     - yarn --frozen-lockfile --prefer-offline
-    - yarn --cwd packages/backend build
     - yarn --cwd packages/frontend build --prod
   artifacts:
     expire_in: 1 day
     paths:
-      - packages/backend/dist
       - packages/frontend/dist
-#
+  needs: []
+
+
+Build backend image:
+  extends: .base_yarn_script
+  stage: Build
+  image: node:13
+  interruptible: true
+  cache:
+    key:
+      files:
+        - yarn.lock
+      prefix: ${CI_JOB_NAME}
+    paths:
+      - packages/frontend/node_modules/.cache
+  before_script:
+    # NOTE(douglasduteil): yarn caching
+    - yarn config set cache-folder $CI_PROJECT_DIR/.cache/yarn
+  script:
+    - yarn --frozen-lockfile --prefer-offline
+    - yarn --cwd packages/backend build
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - packages/backend/dist
+  needs: []
+
+
 
 Register frontend image:
   extends: .base_register_stage
@@ -56,6 +148,11 @@ Register frontend image:
     IMAGE_NAME: $CI_REGISTRY_IMAGE/frontend
     DOCKER_BUILD_ARGS: >-
       -f packages/frontend/Dockerfile
+  needs: ["Build frontend image"]
+  # needs: ["Build frontend image", "Frontend unit tests"]
+  only: 
+    - master # to deploy to preprod only
+    - tags # prod + preprod
 
 Register backend image:
   extends: .base_register_stage
@@ -66,3 +163,7 @@ Register backend image:
     DOCKER_BUILD_ARGS: >-
       --shm-size 768M
       -f packages/backend/Dockerfile
+  needs: ["Build backend image", "Backend unit tests"]
+  only: 
+    - master # to deploy to preprod only
+    - tags # prod + preprod

.travis.yml → OLD.travis.yml

@@ -63,7 +63,7 @@ jobs:
         - yarn build
         #
         - cp -r -i ./packages/backend/src/ressources ./packages/backend/dist/ressources
-        
+
         # start mongo+postgres (with initial dumps)
         - ./docker-compose.travis.run.sh
 
@@ -74,7 +74,7 @@ jobs:
         - ./docker-compose.travis.run.sh
 
         # launch backend
-        - yarn workspace @domifa/backend start:tests-travis &
+        - yarn workspace @domifa/backend start:tests-ci &
 
         # run frontend unit tests
         - yarn workspace @domifa/frontend test --coverage --detectOpenHandles --forceExit

README.md

@@ -1,19 +1,19 @@
 # DomiFa
 
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/f15dba30a35f44c5a62cfdea9c1e3729)](https://app.codacy.com/app/pYassine/domifa?utm_source=github.com&utm_medium=referral&utm_content=SocialGouv/domifa&utm_campaign=Badge_Grade_Dashboard)
-[![Build Status](https://travis-ci.com/SocialGouv/domifa.svg?branch=master)](https://travis-ci.com/SocialGouv/domifa)
 [![codecov](https://codecov.io/gh/SocialGouv/domifa/branch/master/graph/badge.svg)](https://codecov.io/gh/SocialGouv/domifa)
+[![Build Status](https://gitlab.factory.social.gouv.fr/SocialGouv/domifa/badges/master/pipeline.svg)](https://gitlab.factory.social.gouv.fr/SocialGouv/domifa/-/commits/master)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2FSocialGouv%2Fdomifa.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FSocialGouv%2Fdomifa?ref=badge_shield)
 
 > Faciliter l'accès aux droits pour les personnes sans domicile stable, en simplifiant la gestion de la domiciliation
 
-## Développement
+## Développement
 
-Voir [./_docs/dev](./_docs/dev).
+Voir [./\_docs/dev](./_docs/dev).
 
 ## OPS
 
-Voir [./_docs/ops.md](./_docs/ops.md).
+Voir [./\_docs/ops.md](./_docs/ops.md).
 
 ## License
 

_scripts/db/restore-mongo.sh

@@ -1,4 +1,5 @@
-#!/bin/bash
+#!/bin/sh
+CURRENT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 # echo "##############################################################################################"
 # echo "#"
@@ -42,7 +43,7 @@ case $i in
 esac
 done
 
-MONGO_DUMP_PATH="/app/_scripts/db/dumps/domifa_$DUMP_ENV.mongo.gz"
+MONGO_DUMP_PATH="$CURRENT_DIR/dumps/domifa_$DUMP_ENV.mongo.gz"
 MONGO_DUMP_FROM_DATABASE="domifa_$DUMP_ENV"
 MONGO_INITDB_DATABASE="domifa_$TARGET_DB_ENV"
 
@@ -85,6 +86,7 @@ MONGO_AUTH="-u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --auth
 
 if [ "$RECREATE_USER" == "true" ]; then
 
+    echo "--> Clean users: db.runCommand( { dropAllUsersFromDatabase: 1, writeConcern: { w: 'majority'   )"
     (set -x && mongo $MONGO_AUTH --eval "db.runCommand( { dropAllUsersFromDatabase: 1, writeConcern: { w: 'majority'   )")
     if [ $? -ne 0 ]; then
         echo ""
@@ -94,6 +96,7 @@ if [ "$RECREATE_USER" == "true" ]; then
         exit 1
     fi
 
+    echo "--> Create user: db.createUser({user:'$MONGO_INITDB_ROOT_USERNAME', pwd:'$MONGO_INITDB_ROOT_PASSWORD', roles:[{role:'readWrite', db:'$MONGO_INITDB_DATABASE'] );"
     (set -x && mongo $MONGO_AUTH --eval "db.createUser({user:'$MONGO_INITDB_ROOT_USERNAME', pwd:'$MONGO_INITDB_ROOT_PASSWORD', roles:[{role:'readWrite', db:'$MONGO_INITDB_DATABASE'] );")
     if [ $? -ne 0 ]; then
         echo ""
@@ -105,6 +108,8 @@ if [ "$RECREATE_USER" == "true" ]; then
 
 fi
 
+echo "--> Restore mongo DB"
+echo "mongorestore --nsInclude \"${MONGO_DUMP_FROM_DATABASE}.*\" --nsFrom \"${MONGO_DUMP_FROM_DATABASE}.*\" --nsTo \"${MONGO_INITDB_DATABASE}.*\" $MONGO_AUTH --drop --gzip --archive=$MONGO_DUMP_PATH"
 (set -x && mongorestore --nsInclude "${MONGO_DUMP_FROM_DATABASE}.*" --nsFrom "${MONGO_DUMP_FROM_DATABASE}.*" --nsTo "${MONGO_INITDB_DATABASE}.*" $MONGO_AUTH --drop --gzip --archive=$MONGO_DUMP_PATH)
 
 if [ $? -ne 0 ]; then

_scripts/db/restore-postgres.sh

@@ -1,10 +1,11 @@
-#!/bin/bash
+#!/bin/sh
+CURRENT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 # echo "##############################################################################################"
 # echo "#"
 # echo "# USAGE:"
 # echo "#"
-# echo "# $0 [--db=dev] [--dump=test] [--recreate-db]"
+# echo "# $0 [--db=dev] [--dump=test] [--recreate-db] [--ci]"
 # echo "#"
 # echo "##############################################################################################"
 
@@ -20,6 +21,9 @@ case $i in
     --recreate-db)
       RECREATE_DB="true"
     ;;
+    --ci)
+      CI="true"
+    ;;
     *)
     # unknown option
     echo ""
@@ -43,7 +47,7 @@ then
   DUMP_ENV=test
 fi
 
-POSTGRES_DUMP_PATH="/app/_scripts/db/dumps/domifa_$DUMP_ENV.postgres.dump"
+POSTGRES_DUMP_PATH="$CURRENT_DIR/dumps/domifa_$DUMP_ENV.postgres.dump"
 POSTGRES_DATABASE="domifa_${TARGET_DB_ENV}"
 
 if [ ! -f "$POSTGRES_DUMP_PATH" ]; then
@@ -72,7 +76,7 @@ fi
 if [ -z "$POSTGRES_DATABASE" ]; then
   echo ""
   echo "----------------------------------------------------------------------------------------------"
-  echo "[ERROR] env.POSTGRES_USERNAME not set"
+  echo "[ERROR] env.POSTGRES_DATABASE not set"
   echo "----------------------------------------------------------------------------------------------"
   exit 1
 fi
@@ -117,15 +121,48 @@ if [ "$RECREATE_DB" == "true" ]; then
 
 fi
 
-export POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-(set -x && pg_restore --username=${POSTGRES_USERNAME} --no-owner --role=${POSTGRES_USERNAME} --exit-on-error --verbose --dbname=${POSTGRES_DATABASE} ${POSTGRES_DUMP_PATH})
+if [ "$CI" == "true" ]; then
+
+  (set -x && psql -h postgres --username "${POSTGRES_USER}" --dbname postgres -c "CREATE DATABASE ${POSTGRES_DATABASE} WITH OWNER=${POSTGRES_USER}")
+  if [ $? -ne 0 ]; then
+    echo ""
+    echo "----------------------------------------------------------------------------------------------"
+    echo "[ERROR] UNEXPECTED ERROR RUNNING SCRIPT!"
+    echo "----------------------------------------------------------------------------------------------"
+    exit 1
+  fi
+
+  (set -x && psql -h postgres --username "${POSTGRES_USER}" --dbname "${POSTGRES_DATABASE}" -c "DROP SCHEMA IF EXISTS public")
+  if [ $? -ne 0 ]; then
+    echo ""
+    echo "----------------------------------------------------------------------------------------------"
+    echo "[ERROR] UNEXPECTED ERROR RUNNING SCRIPT!"
+    echo "----------------------------------------------------------------------------------------------"
+    exit 1
+  fi
+
+  # export POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+  (set -x && pg_restore -h postgres --username=${POSTGRES_USER} --clean --if-exists --no-acl --no-owner --exit-on-error --verbose --dbname=${POSTGRES_DATABASE} ${POSTGRES_DUMP_PATH})
+  if [ $? -ne 0 ]; then
+    echo ""
+    echo "----------------------------------------------------------------------------------------------"
+    echo "[ERROR] UNEXPECTED ERROR RUNNING SCRIPT!"
+    echo "----------------------------------------------------------------------------------------------"
+    exit 1
+  fi
+
+else
+
+  export POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+  (set -x && pg_restore --username=${POSTGRES_USERNAME} --no-owner --role=${POSTGRES_USERNAME} --exit-on-error --verbose --dbname=${POSTGRES_DATABASE} ${POSTGRES_DUMP_PATH})
+  if [ $? -ne 0 ]; then
+    echo ""
+    echo "----------------------------------------------------------------------------------------------"
+    echo "[ERROR] UNEXPECTED ERROR RUNNING SCRIPT!"
+    echo "----------------------------------------------------------------------------------------------"
+    exit 1
+  fi
 
-if [ $? -ne 0 ]; then
-  echo ""
-  echo "----------------------------------------------------------------------------------------------"
-  echo "[ERROR] UNEXPECTED ERROR RUNNING SCRIPT!"
-  echo "----------------------------------------------------------------------------------------------"
-  exit 1
 fi
 
 echo ""

backup.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CURRENT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 source $CURRENT_DIR/.env # load env
 

packages/backend/package.json

@@ -6,7 +6,9 @@
   "author": "",
   "license": "MIT",
   "scripts": {
-    "build": "rimraf dist && tsc -p tsconfig.build.json && cp -r src/excel/_templates dist/excel",
+    "prebuild": "rimraf dist",
+    "build": "tsc -p tsconfig.build.json",
+    "postbuild": "cp -r src/excel/_templates dist/excel && cp -r src/ressources dist/",
     "format": "prettier --write \"src/**/*.ts\"",
     "db:dev:migrate-up": "npx ts-node -r tsconfig-paths/register --transpile-only src/run-migrate-up.ts",
     "db:test:migrate-up": "NODE_ENV=tests-local npx ts-node -r tsconfig-paths/register --transpile-only src/run-migrate-up.ts",
@@ -24,10 +26,10 @@
     "start:tests-local": "NODE_ENV=tests-local nodemon",
     "start:debug": "nodemon --config nodemon-debug.json",
     "start:prod": "node dist/run-app.js",
-    "start:tests-travis": "NODE_ENV=tests-travis node dist/run-app.js",
+    "start:tests-ci": "node dist/run-app.js",
     "lint": "tslint -p tsconfig.json -t stylish",
     "test": "NODE_ENV=tests-local jest",
-    "test:travis": "NODE_ENV=tests-travis jest",
+    "test:ci": "NODE_ENV=tests-ci jest",
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand"