clean up and split the integrations tests by os

.github/workflows/main.yml

@@ -152,7 +152,7 @@ jobs:
           echo "include=${INCLUDE_GROUPS}"
           echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
 
-  integration:
+  integration-postgres:
     name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
 
     runs-on: ${{ matrix.os }}
@@ -165,8 +165,6 @@ jobs:
         python-version: [ "3.9", "3.10", "3.11", "3.12" ]
         os: [ubuntu-latest]
         split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
-        # this include is where we add the mac and windows os
-        include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
     env:
       TOXENV: integration
       DBT_INVOCATION_ENV: github-actions
@@ -188,14 +186,14 @@ jobs:
         env:
           POSTGRES_PASSWORD: password
           POSTGRES_USER: postgres
-          POSTGRES_HOST: localhost
-          POSTGRES_PORT: 5432
         # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+        ports:
+          - 5432:5432
 
     steps:
       - name: Check out the repository
@@ -206,9 +204,83 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      # - name: Set up postgres (linux)
-      #   if: runner.os == 'Linux'
-      #   uses: ./.github/actions/setup-postgres-linux
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
+
+      - name: Install python tools
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Run integration tests
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: tox -- --ddtrace
+        env:
+          PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: |
+          CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
+          echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
+          path: ./logs
+
+      - name: Upload Integration Test Coverage to Codecov
+        if: ${{ matrix.python-version == '3.11' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: integration
+
+  integration-mac-windows:
+    name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    needs:
+      - integration-metadata
+    strategy:
+      fail-fast: false
+      matrix:
+        split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
+        # this include is where we add the mac and windows os
+        include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
+    env:
+      TOXENV: integration
+      DBT_INVOCATION_ENV: github-actions
+      DBT_TEST_USER_1: dbt_test_user_1
+      DBT_TEST_USER_2: dbt_test_user_2
+      DBT_TEST_USER_3: dbt_test_user_3
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+      DD_SITE: datadoghq.com
+      DD_ENV: ci
+      DD_SERVICE: ${{ github.event.repository.name }}
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Set up postgres (macos)
         if: runner.os == 'macOS'
@@ -258,10 +330,10 @@ jobs:
     if: ${{ always() }}
     name: Integration Test Suite
     runs-on: ubuntu-latest
-    needs: integration
+    needs: [integration-mac-windows, integration-postgres]
     steps:
       - name: "Integration Tests Failed"
-        if: ${{ contains(needs.integration.result, 'failure') || contains(needs.integration.result, 'cancelled') }}
+        if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }}
         # when this is true the next step won't execute
         run: |
           echo "::notice title='Integration test suite failed'"

.github/workflows/structured-logging-schema-check.yml

@@ -67,6 +67,24 @@ jobs:
       DBT_TEST_USER_2: dbt_test_user_2
       DBT_TEST_USER_3: dbt_test_user_3
 
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: checkout dev
         uses: actions/checkout@v4
@@ -85,8 +103,13 @@ jobs:
           pip install tox
           tox --version
 
-      - name: Set up postgres
-        uses: ./.github/actions/setup-postgres-linux
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
 
       - name: ls
         run: ls

test/setup_db.sh

@@ -1 +1,39 @@
-../../../test/setup_db.sh
+#!/bin/bash
+set -x
+env | grep '^PG'
+
+# If you want to run this script for your own postgresql (run with
+# docker-compose) it will look like this:
+# PGHOST=127.0.0.1 PGUSER=root PGPASSWORD=password PGDATABASE=postgres \
+PGUSER="${PGUSER:-postgres}"
+export PGUSER
+PGPORT="${PGPORT:-5432}"
+export PGPORT
+PGHOST="${PGHOST:-localhost}"
+
+for i in {1..10}; do
+	if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then
+		break
+	fi
+
+    echo "Waiting for postgres to be ready..."
+    sleep 2;
+done;
+
+createdb dbt
+psql -c "SELECT version();"
+psql -c "CREATE ROLE root WITH PASSWORD 'password';"
+psql -c "ALTER ROLE root WITH LOGIN;"
+psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
+
+psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
+psql -c "ALTER ROLE noaccess WITH LOGIN;"
+psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
+psql -c "CREATE ROLE dbt_test_user_1;"
+psql -c "CREATE ROLE dbt_test_user_2;"
+psql -c "CREATE ROLE dbt_test_user_3;"
+
+psql -c 'CREATE DATABASE "dbtMixedCase";'
+psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;'
+
+set +x