scrape new articles

hicsail · Feb 9, 2025 · 92a27bc · 92a27bc
1 parent 69be44e
commit 92a27bc
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 95 deletions.
diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml
@@ -51,84 +51,84 @@ jobs:
           method: POST
           url: ${{ secrets.PORTAINER_WEBHOOK_STAGING }}
           preventFailureOnResponse: true
-  translator:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.11
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
+  # translator:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.11
+  #     - uses: docker/setup-qemu-action@v3
+  #     - uses: docker/setup-buildx-action@v3
 
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r translator/requirements.txt
+  #     - name: Install Dependencies
+  #       run: |
+  #         python -m pip install --upgrade pip
+  #         pip install -r translator/requirements.txt
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+  #     - name: Login to Docker Hub
+  #       uses: docker/login-action@v3
+  #       with:
+  #         username: ${{ secrets.DOCKERHUB_USERNAME }}
+  #         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build & Push Docker Staging Build
-        uses: docker/build-push-action@v5
-        if: github.ref == 'refs/heads/main'
-        with:
-          context: ./translator
-          push: true
-          tags: hicsail/gdp-flask-translator:staging
-          build-args: |
-            NOCO_DB_URL=${{ secrets.NOCO_DB_URL }}
-            NOCO_XC_TOKEN=${{ secrets.NOCO_XC_TOKEN }}
-            DEEPL_API_KEY=${{ secrets.DEEPL_API_KEY }}
-            GOOGLE_API_KEY=${{ secrets.GOOGLE_API_KEY }}
+  #     - name: Build & Push Docker Staging Build
+  #       uses: docker/build-push-action@v5
+  #       if: github.ref == 'refs/heads/main'
+  #       with:
+  #         context: ./translator
+  #         push: true
+  #         tags: hicsail/gdp-flask-translator:staging
+  #         build-args: |
+  #           NOCO_DB_URL=${{ secrets.NOCO_DB_URL }}
+  #           NOCO_XC_TOKEN=${{ secrets.NOCO_XC_TOKEN }}
+  #           DEEPL_API_KEY=${{ secrets.DEEPL_API_KEY }}
+  #           GOOGLE_API_KEY=${{ secrets.GOOGLE_API_KEY }}
 
-      - name: Push to Staging
-        uses: fjogeleit/http-request-action@v1
-        if: github.ref == 'refs/heads/main'
-        with:
-          method: POST
-          url: ${{ secrets.PORTAINER_WEBHOOK_STAGING }}
-          preventFailureOnResponse: true
-  classifier:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.11
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
+  #     - name: Push to Staging
+  #       uses: fjogeleit/http-request-action@v1
+  #       if: github.ref == 'refs/heads/main'
+  #       with:
+  #         method: POST
+  #         url: ${{ secrets.PORTAINER_WEBHOOK_STAGING }}
+  #         preventFailureOnResponse: true
+  # classifier:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.11
+  #     - uses: docker/setup-qemu-action@v3
+  #     - uses: docker/setup-buildx-action@v3
 
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r translator/requirements.txt
+  #     - name: Install Dependencies
+  #       run: |
+  #         python -m pip install --upgrade pip
+  #         pip install -r translator/requirements.txt
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+  #     - name: Login to Docker Hub
+  #       uses: docker/login-action@v3
+  #       with:
+  #         username: ${{ secrets.DOCKERHUB_USERNAME }}
+  #         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build & Push Docker Staging Build
-        uses: docker/build-push-action@v5
-        if: github.ref == 'refs/heads/main'
-        with:
-          context: ./classifier
-          push: true
-          tags: hicsail/gdp-flask-classifier:staging
-          build-args: |
-            NOCO_DB_URL=${{ secrets.NOCO_DB_URL }}
-            NOCO_XC_TOKEN=${{ secrets.NOCO_XC_TOKEN }}
-            LLM_URL=${{ secrets.LLM_URL }}
+  #     - name: Build & Push Docker Staging Build
+  #       uses: docker/build-push-action@v5
+  #       if: github.ref == 'refs/heads/main'
+  #       with:
+  #         context: ./classifier
+  #         push: true
+  #         tags: hicsail/gdp-flask-classifier:staging
+  #         build-args: |
+  #           NOCO_DB_URL=${{ secrets.NOCO_DB_URL }}
+  #           NOCO_XC_TOKEN=${{ secrets.NOCO_XC_TOKEN }}
+  #           LLM_URL=${{ secrets.LLM_URL }}
 
-      - name: Push to Staging
-        uses: fjogeleit/http-request-action@v1
-        if: github.ref == 'refs/heads/main'
-        with:
-          method: POST
-          url: ${{ secrets.PORTAINER_WEBHOOK_STAGING }}
-          preventFailureOnResponse: true
+  #     - name: Push to Staging
+  #       uses: fjogeleit/http-request-action@v1
+  #       if: github.ref == 'refs/heads/main'
+  #       with:
+  #         method: POST
+  #         url: ${{ secrets.PORTAINER_WEBHOOK_STAGING }}
+  #         preventFailureOnResponse: true
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,22 +10,22 @@ services:
       - 5001:80
     env_file:
       - .env
-  translator:
-    container_name: gdp-flask-translator
-    build:
-      context: ./translator
-    restart: always
-    ports:
-      - 5002:80
-    env_file:
-      - .env
-  classifier:
-    container_name: gdp-flask-classifier
-    build:
-      context: ./classifier
-    image: hicsail/gdp-flask-classifier:staging
-    restart: always
-    ports:
-      - 5003:80
-    env_file:
-      - .env
+  # translator:
+  #   container_name: gdp-flask-translator
+  #   build:
+  #     context: ./translator
+  #   restart: always
+  #   ports:
+  #     - 5002:80
+  #   env_file:
+  #     - .env
+  # classifier:
+  #   container_name: gdp-flask-classifier
+  #   build:
+  #     context: ./classifier
+  #   image: hicsail/gdp-flask-classifier:staging
+  #   restart: always
+  #   ports:
+  #     - 5003:80
+  #   env_file:
+  #     - .env
diff --git a/scraper/app.py b/scraper/app.py
@@ -215,7 +215,7 @@ def scrape():
             headers = {"xc-token": os.getenv("NOCO_XC_TOKEN")}
 
             # get latest date of article in the database
-            date = ""
+            date = "2024-07-01"
             date_params = {
                 "fields": "articlePublishDateEst",
                 "sort": "-articlePublishDateEst",
@@ -259,6 +259,6 @@ def scrape():
     # initial scrape, this process will take longer
     print("[MOF Scraper] Start inital scraping")
     scrape()
-    scheduler.add_job(scrape, "cron", month="1,7", day="1", hour="0", minute="0")
-    scheduler.start()
+    # scheduler.add_job(scrape, "cron", month="1,7", day="1", hour="0", minute="0")
+    # scheduler.start()
     app.run(port=5001)