chore(domifa): clean anonymization process

SocialGouv · Jul 12, 2024 · fdede7f · fdede7f
1 parent d3c11f8
commit fdede7f
Show file tree

Hide file tree

Showing 6 changed files with 8 additions and 157 deletions.
diff --git a/.anonymizer/anonymize-prod.sh b/.anonymizer/anonymize-prod.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-exclude_tables="contact_support,contact_support_message,expired_token,message_email,monitor_batch_process,user_usager_login"
+exclude_tables="contact_support,contact_support_message,expired_token,message_email,monitor_batch_process,user_usager_login,spatial_ref_sys"
 
 mkdir -p /tmp/pg_dump
 

diff --git a/.github/workflows/reload-anonymized-preprod.yaml b/.github/workflows/reload-anonymized-preprod.yaml
@@ -12,7 +12,7 @@ jobs:
     uses: socialgouv/workflows/.github/workflows/use-ks-gh-custom.yaml@v1
     with:
       chart: jobs-restore-anonymization
-      environment: dev
+      environment: preprod
       inlineSet: |
         global.rancherNamespaceEnabled: true
         global.namespaceLabels:

diff --git a/.kontinuous/env/dev/templates/anonymous-db.sealed-secret.yaml b/.kontinuous/env/dev/templates/anonymous-db.sealed-secret.yaml
diff --git a/.kontinuous/env/dev/values.yaml b/.kontinuous/env/dev/values.yaml
@@ -34,130 +34,6 @@ jobs:
       with:
         seedPath: _scripts/db/dumps/domifa_test.postgres.truncate-restore-data-only.sql
 
-jobs-restore-anonymization:
-  ~chart: jobs
-  enabled: false
-  retry: 0
-  runs:
-    restore-anonymization:
-      memoryLimit: 4Gi
-      cpuLimit: "4"
-      memoryRequest: 4Gi
-      cpuRequest: "2"
-      checkout: true
-      image: ghcr.io/socialgouv/docker/s3-client:sha-d43374e
-      envFrom:
-        - secretRef:
-            name: domifa-dev-backups-access-key
-        - secretRef:
-            name: pg-app
-      env:
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_access_key
-        - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_secret_key
-        - name: AWS_DEFAULT_REGION
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_region
-        - name: AWS_ENDPOINT_URL
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_endpoint
-        - name: PGUSER
-          valueFrom:
-            secretKeyRef:
-              name: pg-superuser
-              key: username
-        - name: PGPASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: pg-superuser
-              key: password
-        - name: PGUSERAPP
-          valueFrom:
-            secretKeyRef:
-              name: pg-app
-              key: PGUSER
-      run: |
-        set -e
-        root_path=domifa-dev-backups/anonymizer/pg_dumps
-        last_dir=$(aws s3 ls s3://$root_path --recursive | awk '{print $4}' | cut -d "/" -f3 | tail -n 1)
-        echo "Restoring $last_dir"
-        aws s3 cp --recursive s3://$root_path/$last_dir /tmp/dump/$last_dir
-        export PGSSLMODE=disable
-        pg_restore --clean --if-exists --no-owner --role=$PGUSERAPP --no-acl --verbose /tmp/dump/$last_dir --dbname $PGDATABASE
-
-jobs-restore-anonymous-prod:
-  ~chart: jobs
-  enabled: false
-  retry: 0
-  runs:
-    restore-anonymization:
-      memoryLimit: 4Gi
-      cpuLimit: "4"
-      memoryRequest: 4Gi
-      cpuRequest: "2"
-      checkout: true
-      image: ghcr.io/socialgouv/docker/s3-client:sha-d43374e
-      envFrom:
-        - secretRef:
-            name: domifa-dev-backups-access-key
-        - secretRef:
-            name: anonymous-db-sealed-secret
-      env:
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_access_key
-        - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_secret_key
-        - name: AWS_DEFAULT_REGION
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_region
-        - name: AWS_ENDPOINT_URL
-          valueFrom:
-            secretKeyRef:
-              name: domifa-dev-backups-access-key
-              key: bucket_endpoint
-        - name: PGUSER
-          valueFrom:
-            secretKeyRef:
-              name: anonymous-db-sealed-secret
-              key: ADMIN_PG_USER
-        - name: PGPASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: anonymous-db-sealed-secret
-              key: ADMIN_PG_PASSWORD
-        - name: PGUSERAPP
-          valueFrom:
-            secretKeyRef:
-              name: anonymous-db-sealed-secret
-              key: PGUSER
-      run: |
-        set -e
-        root_path=domifa-dev-backups/anonymizer/pg_dumps
-        last_dir=$(aws s3 ls s3://$root_path --recursive | awk '{print $4}' | cut -d "/" -f3 | tail -n 1)
-        echo "Restoring $last_dir"
-        aws s3 cp --recursive s3://$root_path/$last_dir /tmp/dump/$last_dir
-        export PGSSLMODE=disable
-        pg_restore --clean --if-exists --no-owner --role=$PGUSERAPP --no-acl --verbose /tmp/dump/$last_dir --dbname defaultdb
-
 backend: &backendDev
   ~needs: [build-backend, pg]
   env: &backendEnv

diff --git a/.kontinuous/env/preprod/templates/anonymous-db.sealed-secret.yaml b/.kontinuous/env/preprod/templates/anonymous-db.sealed-secret.yaml
diff --git a/.kontinuous/env/preprod/values.yaml b/.kontinuous/env/preprod/values.yaml
@@ -110,17 +110,17 @@ jobs-restore-anonymous-prod:
         - name: PGUSER
           valueFrom:
             secretKeyRef:
-              name: anonymous-db-sealed-secret
-              key: ADMIN_PG_USER
+              name: pg-superuser
+              key: username
         - name: PGPASSWORD
           valueFrom:
             secretKeyRef:
-              name: anonymous-db-sealed-secret
-              key: ADMIN_PG_PASSWORD
+              name: pg-superuser
+              key: password
         - name: PGUSERAPP
           valueFrom:
             secretKeyRef:
-              name: anonymous-db-sealed-secret
+              name: pg-app
               key: PGUSER
       run: |
         set -e
@@ -129,4 +129,4 @@ jobs-restore-anonymous-prod:
         echo "Restoring $last_dir"
         aws s3 cp --recursive s3://$root_path/$last_dir /tmp/dump/$last_dir
         export PGSSLMODE=disable
-        pg_restore --clean --if-exists --no-owner --role=$PGUSERAPP --no-acl --verbose /tmp/dump/$last_dir --dbname defaultdb
+        pg_restore --clean --if-exists --no-owner --role=$PGUSERAPP --no-acl --verbose /tmp/dump/$last_dir --dbname $PGDATABASE
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,7 +2,7 @@ @@
     set -e
-    exclude_tables="contact_support,contact_support_message,expired_token,message_email,monitor_batch_process,user_usager_login"
+    exclude_tables="contact_support,contact_support_message,expired_token,message_email,monitor_batch_process,user_usager_login,spatial_ref_sys"
     mkdir -p /tmp/pg_dump
@@ Expand Down @@