[backport 1.4] handle ubuntu 20.04 deprecation (#11354) (#11360)

* handle ubuntu 20.04 deprecation (#11354) * update ubuntu 20.04 to 24.04 * updates to ubuntu-latest instead * try postgres update * Change owner of db creation script so postgres can run it. * Add sudos. * Add debug logging. * Set execute bit on scripts. * More debug logging. * try a service * clean up and split the integrations tests by os --------- Co-authored-by: Peter Allen Webb <[email protected]> # Conflicts: # .github/actions/setup-postgres-linux/action.yml # .github/workflows/main.yml # .github/workflows/structured-logging-schema-check.yml * conflict fixes
dbt-labs · Mar 4, 2025 · a5b3850 · a5b3850
1 parent c3f97d8
commit a5b3850
Show file tree

Hide file tree

Showing 7 changed files with 122 additions and 43 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -61,7 +61,7 @@ body:
       label: Environment
       description: |
         examples:
-          - **OS**: Ubuntu 20.04
+          - **OS**: Ubuntu 24.04
           - **Python**: 3.9.12 (`python3 --version`)
           - **dbt-core**: 1.1.1 (`dbt --version`)
       value: |

diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml
@@ -55,7 +55,7 @@ body:
       label: Environment
       description: |
         examples:
-          - **OS**: Ubuntu 20.04
+          - **OS**: Ubuntu 24.04
           - **Python**: 3.9.12 (`python3 --version`)
           - **dbt-core (working version)**: 1.1.1 (`dbt --version`)
           - **dbt-core (regression version)**: 1.2.0 (`dbt --version`)

diff --git a/.github/actions/setup-postgres-linux/action.yml b/.github/actions/setup-postgres-linux/action.yml
diff --git a/.github/actions/setup-postgres-linux/setup_db.sh b/.github/actions/setup-postgres-linux/setup_db.sh
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -115,7 +115,7 @@ jobs:
           name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
           path: unit_results.csv
 
-  integration:
+  integration-postgres:
     name: integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
 
     runs-on: ${{ matrix.os }}
@@ -125,13 +125,102 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [ "3.9", "3.10", "3.11" ]
-        os: [ubuntu-20.04]
+        os: [ubuntu-latest]
+
+    env:
+      TOXENV: integration
+      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
+      DBT_INVOCATION_ENV: github-actions
+      DBT_TEST_USER_1: dbt_test_user_1
+      DBT_TEST_USER_2: dbt_test_user_2
+      DBT_TEST_USER_3: dbt_test_user_3
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+      DD_SITE: datadoghq.com
+      DD_ENV: ci
+      DD_SERVICE: ${{ github.event.repository.name }}
+
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
+
+      - name: Install python tools
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Run integration tests
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 90
+          max_attempts: 3
+          command: tox -- --ddtrace
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: |
+          CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
+          echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }}
+          path: ./logs
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }}.csv
+          path: integration_results.csv
+
+  integration-mac-windows:
+    name: integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
         include:
           - python-version: 3.9
             os: windows-latest
           - python-version: 3.9
             os: macos-14
-
     env:
       TOXENV: integration
       PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
@@ -154,10 +243,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Set up postgres (linux)
-        if: runner.os == 'Linux'
-        uses: ./.github/actions/setup-postgres-linux
-
       - name: Set up postgres (macos)
         if: runner.os == 'macOS'
         uses: ./.github/actions/setup-postgres-macos
@@ -176,7 +261,7 @@ jobs:
       - name: Run integration tests
         uses: nick-fields/retry@v3
         with:
-          timeout_minutes: 30
+          timeout_minutes: 90
           max_attempts: 3
           command: tox -- --ddtrace
 
@@ -190,7 +275,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
+          name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }}
           path: ./logs
 
       - uses: actions/upload-artifact@v4

diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml
@@ -22,7 +22,7 @@ jobs:
   # run the performance measurements on the current or default branch
   test-schema:
     name: Test Log Schema
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       # turns warnings into errors
       RUSTFLAGS: "-D warnings"
@@ -35,6 +35,24 @@ jobs:
       DBT_TEST_USER_2: dbt_test_user_2
       DBT_TEST_USER_3: dbt_test_user_3
 
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: checkout dev
         uses: actions/checkout@v4
@@ -53,8 +71,13 @@ jobs:
           pip install tox
           tox --version
 
-      - name: Set up postgres
-        uses: ./.github/actions/setup-postgres-linux
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
 
       - name: ls
         run: ls

diff --git a/test/setup_db.sh b/test/setup_db.sh
@@ -11,24 +11,6 @@ PGPORT="${PGPORT:-5432}"
 export PGPORT
 PGHOST="${PGHOST:-localhost}"
 
-function connect_circle() {
-	# try to handle circleci/docker oddness
-	let rc=1
-	while [[ $rc -eq 1 ]]; do
-		nc -z ${PGHOST} ${PGPORT}
-		let rc=$?
-	done
-	if [[ $rc -ne 0 ]]; then
-		echo "Fatal: Could not connect to $PGHOST"
-		exit 1
-	fi
-}
-
-# appveyor doesn't have 'nc', but it also doesn't have these issues
-if [[ -n $CIRCLECI ]]; then
-	connect_circle
-fi
-
 for i in {1..10}; do
 	if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then
 		break