diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 42a50b385f8..7fe4b04e870 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -61,7 +61,7 @@ body: label: Environment description: | examples: - - **OS**: Ubuntu 20.04 + - **OS**: Ubuntu 24.04 - **Python**: 3.9.12 (`python3 --version`) - **dbt-core**: 1.1.1 (`dbt --version`) value: | diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml index 5dfceedb482..e94c9d162ea 100644 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ b/.github/ISSUE_TEMPLATE/regression-report.yml @@ -55,7 +55,7 @@ body: label: Environment description: | examples: - - **OS**: Ubuntu 20.04 + - **OS**: Ubuntu 24.04 - **Python**: 3.9.12 (`python3 --version`) - **dbt-core (working version)**: 1.1.1 (`dbt --version`) - **dbt-core (regression version)**: 1.2.0 (`dbt --version`) diff --git a/.github/actions/setup-postgres-linux/action.yml b/.github/actions/setup-postgres-linux/action.yml deleted file mode 100644 index 1c8fc772a8a..00000000000 --- a/.github/actions/setup-postgres-linux/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: "Set up postgres (linux)" -description: "Set up postgres service on linux vm for dbt integration tests" -runs: - using: "composite" - steps: - - shell: bash - run: | - sudo systemctl start postgresql.service - pg_isready - sudo -u postgres bash ${{ github.action_path }}/setup_db.sh diff --git a/.github/actions/setup-postgres-linux/setup_db.sh b/.github/actions/setup-postgres-linux/setup_db.sh deleted file mode 120000 index ee75ef5048c..00000000000 --- a/.github/actions/setup-postgres-linux/setup_db.sh +++ /dev/null @@ -1 +0,0 @@ -../../../test/setup_db.sh \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4c94d619e8..aafe07d24e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -115,7 +115,7 @@ jobs: name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv path: unit_results.csv - integration: + integration-postgres: name: integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -126,12 +126,103 @@ jobs: matrix: python-version: [ "3.9", "3.10", "3.11" ] os: [ubuntu-20.04] + + env: + TOXENV: integration + PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: password + POSTGRES_USER: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Run postgres setup script + run: | + ./test/setup_db.sh + env: + PGHOST: localhost + PGPORT: 5432 + PGPASSWORD: password + + - name: Install python tools + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: Run integration tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 90 + max_attempts: 3 + command: tox -- --ddtrace + + - name: Get current date + if: always() + id: date + run: | + CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts + echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }} + path: ./logs + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }}.csv + path: integration_results.csv + + integration-mac-windows: + name: integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} + + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + needs: + - integration-metadata + strategy: + fail-fast: false + matrix: include: - python-version: 3.9 os: windows-latest - python-version: 3.9 os: macos-14 - env: TOXENV: integration PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" @@ -154,10 +245,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Set up postgres (linux) - if: runner.os == 'Linux' - uses: ./.github/actions/setup-postgres-linux - - name: Set up postgres (macos) if: runner.os == 'macOS' uses: ./.github/actions/setup-postgres-macos @@ -176,7 +263,7 @@ jobs: - name: Run integration tests uses: nick-fields/retry@v3 with: - timeout_minutes: 30 + timeout_minutes: 90 max_attempts: 3 command: tox -- --ddtrace diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml index 41c452e9c60..1d028c00325 100644 --- a/.github/workflows/structured-logging-schema-check.yml +++ b/.github/workflows/structured-logging-schema-check.yml @@ -22,7 +22,7 @@ jobs: # run the performance measurements on the current or default branch test-schema: name: Test Log Schema - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest env: # turns warnings into errors RUSTFLAGS: "-D warnings" @@ -35,6 +35,24 @@ jobs: DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: password + POSTGRES_USER: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: checkout dev uses: actions/checkout@v4 @@ -53,8 +71,13 @@ jobs: pip install tox tox --version - - name: Set up postgres - uses: ./.github/actions/setup-postgres-linux + - name: Run postgres setup script + run: | + ./test/setup_db.sh + env: + PGHOST: localhost + PGPORT: 5432 + PGPASSWORD: password - name: ls run: ls diff --git a/test/setup_db.sh b/test/setup_db.sh index de59bf0fac6..fff943126b4 100755 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -11,24 +11,6 @@ PGPORT="${PGPORT:-5432}" export PGPORT PGHOST="${PGHOST:-localhost}" -function connect_circle() { - # try to handle circleci/docker oddness - let rc=1 - while [[ $rc -eq 1 ]]; do - nc -z ${PGHOST} ${PGPORT} - let rc=$? - done - if [[ $rc -ne 0 ]]; then - echo "Fatal: Could not connect to $PGHOST" - exit 1 - fi -} - -# appveyor doesn't have 'nc', but it also doesn't have these issues -if [[ -n $CIRCLECI ]]; then - connect_circle -fi - for i in {1..10}; do if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then break