From aec85c849067cf4878daafcdf67cc5e7b8be18f0 Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Wed, 1 Jan 2025 16:34:09 +0700 Subject: [PATCH] Add Dockerfile for notifications job and Github action workflow --- .github/workflows/gcp_notifications_job.yml | 82 +++++++++++++++++++++ backend/modal/Dockerfile.notifications_job | 20 +++++ backend/modal/job_modal.py | 34 ++++----- backend/utils/other/notifications.py | 1 + 4 files changed, 116 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/gcp_notifications_job.yml create mode 100644 backend/modal/Dockerfile.notifications_job diff --git a/.github/workflows/gcp_notifications_job.yml b/.github/workflows/gcp_notifications_job.yml new file mode 100644 index 0000000000..f07c23186a --- /dev/null +++ b/.github/workflows/gcp_notifications_job.yml @@ -0,0 +1,82 @@ +name: Deploy Notifications Job to Cloud RUN + +on: + push: + branches: [ "migrate-noti-job-to-gcp" ] + paths: + - '**/*' + workflow_dispatch: + inputs: + environment: + description: 'Select the environment to deploy to' + required: true + default: 'development' + branch: + description: 'Branch to deploy from' + required: true + default: 'main' + +env: + SERVICE: notifications-job + REGION: us-central1 + +jobs: + deploy: + # environment: ${{ github.event.inputs.environment }} + environment: development + permissions: + contents: 'read' + id-token: 'write' + + runs-on: ubuntu-latest + steps: + # - name: Validate Environment Input + # run: | + # if [[ "${{ github.event.inputs.environment }}" != "development" && "${{ github.event.inputs.environment }}" != "prod" ]]; then + # echo "Invalid environment: ${{ github.event.inputs.environment }}. Must be 'development' or 'prod'." + # exit 1 + # fi + + # To workaround "no space left on device" issue of GitHub-hosted runner + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout + uses: actions/checkout@v4 + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_CREDENTIALS }} + + - name: Login to GCR + run: gcloud auth configure-docker + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Google Service Account + run: echo "${{ secrets.GCP_SERVICE_ACCOUNT }}" | base64 -d > ./backend/google-credentials.json + + - name: Build and Push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./backend/modal/Dockerfile.notifications_job + push: true + tags: gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:latest + cache-from: type=registry,ref=gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:buildcache + cache-to: type=registry,ref=gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:buildcache,mode=max + + - name: Deploy to Cloud Run + id: deploy + uses: google-github-actions/deploy-cloudrun@v2 + with: + job: ${{ env.SERVICE }} + region: ${{ env.REGION }} + image: gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }} + + # If required, use the Cloud Run url output in later steps + - name: Show Output + run: echo ${{ steps.deploy.outputs.url }} diff --git a/backend/modal/Dockerfile.notifications_job b/backend/modal/Dockerfile.notifications_job new file mode 100644 index 0000000000..1953b47508 --- /dev/null +++ b/backend/modal/Dockerfile.notifications_job @@ -0,0 +1,20 @@ +FROM python:3.11 AS builder + +ENV PATH="/opt/venv/bin:$PATH" +RUN python -m venv /opt/venv + +COPY backend/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt + +FROM python:3.11-slim + +WORKDIR /app +ENV PATH="/opt/venv/bin:$PATH" + +RUN apt-get update && apt-get -y install ffmpeg curl unzip && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /opt/venv /opt/venv +COPY backend/ . +COPY backend/modal/ . + +CMD ["python", "job_modal.py"] diff --git a/backend/modal/job_modal.py b/backend/modal/job_modal.py index 3ae20a8d31..518a846819 100644 --- a/backend/modal/job_modal.py +++ b/backend/modal/job_modal.py @@ -1,28 +1,20 @@ import json import os -import firebase_admin +# import firebase_admin +import asyncio -from modal import Image, App, Secret, Cron from utils.other.notifications import start_cron_job -if os.environ.get('SERVICE_ACCOUNT_JSON'): - service_account_info = json.loads(os.environ["SERVICE_ACCOUNT_JSON"]) - credentials = firebase_admin.credentials.Certificate(service_account_info) - firebase_admin.initialize_app(credentials) -else: - firebase_admin.initialize_app() +# if os.environ.get('SERVICE_ACCOUNT_JSON'): +# service_account_info = json.loads(os.environ["SERVICE_ACCOUNT_JSON"]) +# credentials = firebase_admin.credentials.Certificate(service_account_info) +# firebase_admin.initialize_app(credentials) +# else: +# firebase_admin.initialize_app() -app = App( - name='job', - secrets=[Secret.from_name("gcp-credentials"), Secret.from_name('envs')], -) -image = ( - Image.debian_slim() - .apt_install('ffmpeg', 'git', 'unzip') - .pip_install_from_requirements('requirements.txt') -) - -@app.function(image=image, schedule=Cron('* * * * *')) -async def notifications_cronjob(): - await start_cron_job() +print('Starting cron job...') +try: + asyncio.run(start_cron_job()) +except Exception as e: + print(f"Error running cron job: {e}") \ No newline at end of file diff --git a/backend/utils/other/notifications.py b/backend/utils/other/notifications.py index 4f4fd21248..29e740a28a 100644 --- a/backend/utils/other/notifications.py +++ b/backend/utils/other/notifications.py @@ -16,6 +16,7 @@ async def start_cron_job(): + print('start_cron_job function debug...') if should_run_job(): print('start_cron_job') await send_daily_notification()