handle ubuntu 20.04 deprecation (#11354)

* update ubuntu 20.04 to 24.04 * updates to ubuntu-latest instead * try postgres update * Change owner of db creation script so postgres can run it. * Add sudos. * Add debug logging. * Set execute bit on scripts. * More debug logging. * try a service * clean up and split the integrations tests by os --------- Co-authored-by: Peter Allen Webb <[email protected]> # Conflicts: # .github/actions/setup-postgres-linux/action.yml
dbt-labs · Mar 4, 2025 · 202d80a · 202d80a
1 parent a9719a3
commit 202d80a
Show file tree

Hide file tree

Showing 7 changed files with 127 additions and 42 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -61,7 +61,7 @@ body:
       label: Environment
       description: |
         examples:
-          - **OS**: Ubuntu 20.04
+          - **OS**: Ubuntu 24.04
           - **Python**: 3.9.12 (`python3 --version`)
           - **dbt-core**: 1.1.1 (`dbt --version`)
       value: |

diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml
@@ -55,7 +55,7 @@ body:
       label: Environment
       description: |
         examples:
-          - **OS**: Ubuntu 20.04
+          - **OS**: Ubuntu 24.04
           - **Python**: 3.9.12 (`python3 --version`)
           - **dbt-core (working version)**: 1.1.1 (`dbt --version`)
           - **dbt-core (regression version)**: 1.2.0 (`dbt --version`)

diff --git a/.github/actions/setup-postgres-linux/action.yml b/.github/actions/setup-postgres-linux/action.yml
diff --git a/.github/actions/setup-postgres-linux/setup_db.sh b/.github/actions/setup-postgres-linux/setup_db.sh
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -151,7 +151,7 @@ jobs:
           echo "include=${INCLUDE_GROUPS}"
           echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
 
-  integration:
+  integration-postgres:
     name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
 
     runs-on: ${{ matrix.os }}
@@ -162,9 +162,8 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [ "3.9", "3.10", "3.11", "3.12" ]
-        os: [ubuntu-20.04]
+        os: [ubuntu-latest]
         split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
-        include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
     env:
       TOXENV: integration
       DBT_INVOCATION_ENV: github-actions
@@ -177,6 +176,24 @@ jobs:
       DD_ENV: ci
       DD_SERVICE: ${{ github.event.repository.name }}
 
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -186,9 +203,83 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Set up postgres (linux)
-        if: runner.os == 'Linux'
-        uses: ./.github/actions/setup-postgres-linux
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
+
+      - name: Install python tools
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Run integration tests
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: tox -- --ddtrace
+        env:
+          PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: |
+          CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
+          echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
+          path: ./logs
+
+      - name: Upload Integration Test Coverage to Codecov
+        if: ${{ matrix.python-version == '3.11' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: integration
+
+  integration-mac-windows:
+    name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    needs:
+      - integration-metadata
+    strategy:
+      fail-fast: false
+      matrix:
+        split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
+        # this include is where we add the mac and windows os
+        include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
+    env:
+      TOXENV: integration
+      DBT_INVOCATION_ENV: github-actions
+      DBT_TEST_USER_1: dbt_test_user_1
+      DBT_TEST_USER_2: dbt_test_user_2
+      DBT_TEST_USER_3: dbt_test_user_3
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+      DD_SITE: datadoghq.com
+      DD_ENV: ci
+      DD_SERVICE: ${{ github.event.repository.name }}
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Set up postgres (macos)
         if: runner.os == 'macOS'
@@ -238,10 +329,10 @@ jobs:
     if: ${{ always() }}
     name: Integration Test Suite
     runs-on: ubuntu-latest
-    needs: integration
+    needs: [integration-mac-windows, integration-postgres]
     steps:
       - name: "Integration Tests Failed"
-        if: ${{ contains(needs.integration.result, 'failure') || contains(needs.integration.result, 'cancelled') }}
+        if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }}
         # when this is true the next step won't execute
         run: |
           echo "::notice title='Integration test suite failed'"

diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml
@@ -45,7 +45,7 @@ jobs:
   # run the performance measurements on the current or default branch
   test-schema:
     name: Test Log Schema
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 30
     needs:
       - integration-metadata
@@ -67,6 +67,24 @@ jobs:
       DBT_TEST_USER_2: dbt_test_user_2
       DBT_TEST_USER_3: dbt_test_user_3
 
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: checkout dev
         uses: actions/checkout@v4
@@ -85,8 +103,13 @@ jobs:
           pip install tox
           tox --version
 
-      - name: Set up postgres
-        uses: ./.github/actions/setup-postgres-linux
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
 
       - name: ls
         run: ls

diff --git a/test/setup_db.sh b/test/setup_db.sh
@@ -11,24 +11,6 @@ PGPORT="${PGPORT:-5432}"
 export PGPORT
 PGHOST="${PGHOST:-localhost}"
 
-function connect_circle() {
-	# try to handle circleci/docker oddness
-	let rc=1
-	while [[ $rc -eq 1 ]]; do
-		nc -z ${PGHOST} ${PGPORT}
-		let rc=$?
-	done
-	if [[ $rc -ne 0 ]]; then
-		echo "Fatal: Could not connect to $PGHOST"
-		exit 1
-	fi
-}
-
-# appveyor doesn't have 'nc', but it also doesn't have these issues
-if [[ -n $CIRCLECI ]]; then
-	connect_circle
-fi
-
 for i in {1..10}; do
 	if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then
 		break