From f61c0b20144542cf2e8e2cabe2e4d51c20e864c0 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 4 Mar 2025 13:38:27 -0600 Subject: [PATCH] [backport 1.5] handle ubuntu 20.04 deprecation (#11354) (#11361) * handle ubuntu 20.04 deprecation (#11354) * update ubuntu 20.04 to 24.04 * updates to ubuntu-latest instead * try postgres update * Change owner of db creation script so postgres can run it. * Add sudos. * Add debug logging. * Set execute bit on scripts. * More debug logging. * try a service * clean up and split the integrations tests by os --------- Co-authored-by: Peter Allen Webb # Conflicts: # .github/actions/setup-postgres-linux/action.yml # .github/workflows/main.yml * upgrade ubuntu --- .github/ISSUE_TEMPLATE/bug-report.yml | 2 +- .github/ISSUE_TEMPLATE/regression-report.yml | 2 +- .../actions/setup-postgres-linux/action.yml | 10 -- .../actions/setup-postgres-linux/setup_db.sh | 1 - .github/workflows/main.yml | 105 ++++++++++++++++-- .../structured-logging-schema-check.yml | 29 ++++- test/setup_db.sh | 18 --- 7 files changed, 126 insertions(+), 41 deletions(-) delete mode 100644 .github/actions/setup-postgres-linux/action.yml delete mode 120000 .github/actions/setup-postgres-linux/setup_db.sh diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 42a50b385f8..7fe4b04e870 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -61,7 +61,7 @@ body: label: Environment description: | examples: - - **OS**: Ubuntu 20.04 + - **OS**: Ubuntu 24.04 - **Python**: 3.9.12 (`python3 --version`) - **dbt-core**: 1.1.1 (`dbt --version`) value: | diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml index 5dfceedb482..e94c9d162ea 100644 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ b/.github/ISSUE_TEMPLATE/regression-report.yml @@ -55,7 +55,7 @@ body: label: Environment description: | examples: - - **OS**: Ubuntu 20.04 + - **OS**: Ubuntu 24.04 - **Python**: 3.9.12 (`python3 --version`) - **dbt-core (working version)**: 1.1.1 (`dbt --version`) - **dbt-core (regression version)**: 1.2.0 (`dbt --version`) diff --git a/.github/actions/setup-postgres-linux/action.yml b/.github/actions/setup-postgres-linux/action.yml deleted file mode 100644 index 1c8fc772a8a..00000000000 --- a/.github/actions/setup-postgres-linux/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: "Set up postgres (linux)" -description: "Set up postgres service on linux vm for dbt integration tests" -runs: - using: "composite" - steps: - - shell: bash - run: | - sudo systemctl start postgresql.service - pg_isready - sudo -u postgres bash ${{ github.action_path }}/setup_db.sh diff --git a/.github/actions/setup-postgres-linux/setup_db.sh b/.github/actions/setup-postgres-linux/setup_db.sh deleted file mode 120000 index ee75ef5048c..00000000000 --- a/.github/actions/setup-postgres-linux/setup_db.sh +++ /dev/null @@ -1 +0,0 @@ -../../../test/setup_db.sh \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aaefac43f15..6ca07ca8e4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -151,7 +151,7 @@ jobs: echo "include=${INCLUDE_GROUPS}" echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT - integration: + integration-postgres: name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -162,8 +162,103 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10", "3.11"] - os: [ubuntu-20.04] + os: [ubuntu-latest] split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} + env: + TOXENV: integration + DBT_INVOCATION_ENV: github-actions + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: password + POSTGRES_USER: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Run postgres setup script + run: | + ./test/setup_db.sh + env: + PGHOST: localhost + PGPORT: 5432 + PGPASSWORD: password + + - name: Install python tools + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: Run integration tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 30 + max_attempts: 3 + command: tox -- --ddtrace + env: + PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} + + - name: Get current date + if: always() + id: date + run: | + CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts + echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }} + path: ./logs + + - name: Upload Integration Test Coverage to Codecov + if: ${{ matrix.python-version == '3.11' }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: integration + + integration-mac-windows: + name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} + + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + needs: + - integration-metadata + strategy: + fail-fast: false + matrix: + split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} + # this include is where we add the mac and windows os include: ${{ fromJson(needs.integration-metadata.outputs.include) }} env: TOXENV: integration @@ -187,10 +282,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Set up postgres (linux) - if: runner.os == 'Linux' - uses: ./.github/actions/setup-postgres-linux - - name: Set up postgres (macos) if: runner.os == 'macOS' uses: ./.github/actions/setup-postgres-macos @@ -238,7 +329,7 @@ jobs: if: ${{ always() }} name: Integration Test Suite runs-on: ubuntu-latest - needs: integration + needs: [integration-mac-windows, integration-postgres] steps: - name: "[Notification] Integration test suite passes" run: | diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml index 0f5810f681c..4853a0f6436 100644 --- a/.github/workflows/structured-logging-schema-check.yml +++ b/.github/workflows/structured-logging-schema-check.yml @@ -45,7 +45,7 @@ jobs: # run the performance measurements on the current or default branch test-schema: name: Test Log Schema - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 30 needs: - integration-metadata @@ -67,6 +67,24 @@ jobs: DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: password + POSTGRES_USER: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: checkout dev uses: actions/checkout@v4 @@ -85,8 +103,13 @@ jobs: pip install tox tox --version - - name: Set up postgres - uses: ./.github/actions/setup-postgres-linux + - name: Run postgres setup script + run: | + ./test/setup_db.sh + env: + PGHOST: localhost + PGPORT: 5432 + PGPASSWORD: password - name: ls run: ls diff --git a/test/setup_db.sh b/test/setup_db.sh index de59bf0fac6..fff943126b4 100755 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -11,24 +11,6 @@ PGPORT="${PGPORT:-5432}" export PGPORT PGHOST="${PGHOST:-localhost}" -function connect_circle() { - # try to handle circleci/docker oddness - let rc=1 - while [[ $rc -eq 1 ]]; do - nc -z ${PGHOST} ${PGPORT} - let rc=$? - done - if [[ $rc -ne 0 ]]; then - echo "Fatal: Could not connect to $PGHOST" - exit 1 - fi -} - -# appveyor doesn't have 'nc', but it also doesn't have these issues -if [[ -n $CIRCLECI ]]; then - connect_circle -fi - for i in {1..10}; do if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then break