From a3d60003a6213fee0b48741f6097ef62a597f94e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 10 Feb 2025 11:54:54 -0500 Subject: [PATCH] Remove some other tests --- .github/workflows/dependencies.yml | 55 --- .github/workflows/docs_pr.yaml | 54 --- .github/workflows/rust.yml | 603 ----------------------------- 3 files changed, 712 deletions(-) delete mode 100644 .github/workflows/dependencies.yml delete mode 100644 .github/workflows/docs_pr.yaml delete mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml deleted file mode 100644 index a577725fed4b..000000000000 --- a/.github/workflows/dependencies.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Dependencies - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - push: - paths: - - "**/Cargo.toml" - - "**/Cargo.lock" - pull_request: - paths: - - "**/Cargo.toml" - - "**/Cargo.lock" - # manual trigger - # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -jobs: - depcheck: - name: circular dependency check - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Check dependencies - run: | - cd dev/depcheck - cargo run diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml deleted file mode 100644 index 3fad08643aa2..000000000000 --- a/.github/workflows/docs_pr.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Docs - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - push: - paths: - - "docs/**" - pull_request: - paths: - - "docs/**" - # manual trigger - # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -jobs: - # Run doc tests - linux-test-doc: - name: cargo doctest (amd64) - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run doctests - run: cargo test --doc --features avro,json - - name: Verify Working Directory Clean - run: git diff --exit-code diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index c023faa9b168..000000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,603 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Rust - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - push: - paths-ignore: - - "docs/**" - - "**.md" - - ".github/ISSUE_TEMPLATE/**" - - ".github/pull_request_template.md" - pull_request: - paths-ignore: - - "docs/**" - - "**.md" - - ".github/ISSUE_TEMPLATE/**" - - ".github/pull_request_template.md" - # manual trigger - # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -jobs: - # Check license header - license-header-check: - runs-on: ubuntu-20.04 - name: Check License Header - steps: - - uses: actions/checkout@v4 - - uses: korandoru/hawkeye@v6 - - # Check crate compiles and base cargo check passes - linux-build-lib: - name: linux build test - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Prepare cargo build - run: | - # Adding `--locked` here to assert that the `Cargo.lock` file is up to - # date with the manifest. When this fails, please make sure to commit - # the changes to `Cargo.lock` after building with the updated manifest. - cargo check --profile ci --workspace --all-targets --features integration-tests --locked - - # cargo check common, functions and substrait with no default features - linux-cargo-check-no-default-features: - name: cargo check no default features - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Check datafusion without default features - # Some of the test binaries require the parquet feature still - #run: cargo check --all-targets --no-default-features -p datafusion - run: cargo check --profile ci --no-default-features -p datafusion - - - name: Check datafusion-common without default features - run: cargo check --profile ci --all-targets --no-default-features -p datafusion-common - - - name: Check datafusion-functions without default features - run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions - - - name: Check datafusion-substrait without default features - run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait - - - name: Check workspace in debug mode - run: cargo check --profile ci --all-targets --workspace - - - name: Check workspace with additional features - run: cargo check --profile ci --workspace --benches --features avro,json,integration-tests - - # cargo check datafusion to ensure that the datafusion crate can be built with only a - # subset of the function packages enabled. - linux-cargo-check-datafusion: - name: cargo check datafusion - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Check datafusion (nested_expressions) - run: cargo check --profile ci --no-default-features --features=nested_expressions -p datafusion - - - name: Check datafusion (crypto) - run: cargo check --profile ci --no-default-features --features=crypto_expressions -p datafusion - - - name: Check datafusion (datetime_expressions) - run: cargo check --profile ci --no-default-features --features=datetime_expressions -p datafusion - - - name: Check datafusion (encoding_expressions) - run: cargo check --profile ci --no-default-features --features=encoding_expressions -p datafusion - - - name: Check datafusion (math_expressions) - run: cargo check --profile ci --no-default-features --features=math_expressions -p datafusion - - - name: Check datafusion (regex_expressions) - run: cargo check --profile ci --no-default-features --features=regex_expressions -p datafusion - - - name: Check datafusion (string_expressions) - run: cargo check --profile ci --no-default-features --features=string_expressions -p datafusion - - # cargo check datafusion-functions to ensure that the datafusion-functions crate can be built with - # only a subset of the function packages enabled. - linux-cargo-check-datafusion-functions: - name: cargo check functions - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Check datafusion-functions (crypto) - run: cargo check --profile ci --all-targets --no-default-features --features=crypto_expressions -p datafusion-functions - - - name: Check datafusion-functions (datetime_expressions) - run: cargo check --profile ci --all-targets --no-default-features --features=datetime_expressions -p datafusion-functions - - - name: Check datafusion-functions (encoding_expressions) - run: cargo check --profile ci --all-targets --no-default-features --features=encoding_expressions -p datafusion-functions - - - name: Check datafusion-functions (math_expressions) - run: cargo check --profile ci --all-targets --no-default-features --features=math_expressions -p datafusion-functions - - - name: Check datafusion-functions (regex_expressions) - run: cargo check --profile ci --all-targets --no-default-features --features=regex_expressions -p datafusion-functions - - - name: Check datafusion-functions (string_expressions) - run: cargo check --profile ci --all-targets --no-default-features --features=string_expressions -p datafusion-functions - - # Run tests - linux-test: - name: cargo test (amd64) - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run tests (excluding doctests) - run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests - - name: Verify Working Directory Clean - run: git diff --exit-code - - linux-test-example: - name: cargo examples (amd64) - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run examples - run: | - # test datafusion-sql examples - cargo run --profile ci --example sql - # test datafusion-examples - ci/scripts/rust_example.sh - - name: Verify Working Directory Clean - run: git diff --exit-code - - # Run `cargo test doc` (test documentation examples) - linux-test-doc: - name: cargo test doc (amd64) - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run doctests - run: cargo test --profile ci --doc --features avro,json - - name: Verify Working Directory Clean - run: git diff --exit-code - - # Run `cargo doc` to ensure the rustdoc is clean - linux-rustdoc: - name: cargo doc - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run cargo doc - run: ci/scripts/rust_docs.sh - - linux-wasm-pack: - name: build with wasm-pack - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install wasm-pack - run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - name: Build with wasm-pack - working-directory: ./datafusion/wasmtest - run: wasm-pack build --dev - - # verify that the benchmark queries return the correct results - verify-benchmark-results: - name: verify benchmark results (amd64) - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Generate benchmark data and expected query results - run: | - mkdir -p datafusion/sqllogictest/test_files/tpch/data - git clone https://github.com/databricks/tpch-dbgen.git - cd tpch-dbgen - make - ./dbgen -f -s 0.1 - mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data - - name: Verify that benchmark queries return expected results - run: | - # increase stack size to fix stack overflow - export RUST_MIN_STACK=20971520 - export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` - cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1 - INCLUDE_TPCH=true cargo test --profile ci --package datafusion-sqllogictest --test sqllogictests - - name: Verify Working Directory Clean - run: git diff --exit-code - - sqllogictest-postgres: - name: "Run sqllogictest with Postgres runner" - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: db_test - POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run sqllogictest - run: | - cd datafusion/sqllogictest - PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --profile ci --features=postgres --test sqllogictests - env: - # use postgres for the host here because we have specified a container for the job - POSTGRES_HOST: postgres - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} - - # Temporarily commenting out the Windows flow, the reason is enormously slow running build - # Waiting for new Windows 2025 github runner - # Details: https://github.com/apache/datafusion/issues/13726 - # - # windows: - # name: cargo test (win64) - # runs-on: windows-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-windows-builder - # - name: Run tests (excluding doctests) - # shell: bash - # run: | - # export PATH=$PATH:$HOME/d/protoc/bin - # cargo test --lib --tests --bins --features avro,json,backtrace - - # Commenting out intel mac build as so few users would ever use it - # Details: https://github.com/apache/datafusion/issues/13846 - # macos: - # name: cargo test (macos) - # runs-on: macos-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # fetch-depth: 1 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-macos-builder - # - name: Run tests (excluding doctests) - # shell: bash - # run: cargo test run --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace - - macos-aarch64: - name: cargo test (macos-aarch64) - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-macos-aarch64-builder - - name: Run tests (excluding doctests) - shell: bash - run: cargo test --profile ci --lib --tests --bins --features avro,json,backtrace,integration-tests - - test-datafusion-pyarrow: - name: cargo test pyarrow (amd64) - needs: linux-build-lib - runs-on: ubuntu-20.04 - container: - image: amd64/rust:bullseye # Workaround https://github.com/actions/setup-python/issues/721 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - name: Install PyArrow - run: | - echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV - python -m pip install pyarrow - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run datafusion-common tests - run: cargo test --profile ci -p datafusion-common --features=pyarrow - - vendor: - name: Verify Vendored Code - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run gen - run: ./regen.sh - working-directory: ./datafusion/proto - - name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results) - run: git diff --exit-code - - check-fmt: - name: Check cargo fmt - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run - run: | - echo '' > datafusion/proto/src/generated/datafusion.rs - ci/scripts/rust_fmt.sh - - # Coverage job disabled due to - # https://github.com/apache/datafusion/issues/3678 - - # coverage: - # name: coverage - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Install protobuf compiler - # shell: bash - # run: | - # mkdir -p $HOME/d/protoc - # cd $HOME/d/protoc - # export PROTO_ZIP="protoc-21.4-linux-x86_64.zip" - # curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP - # unzip $PROTO_ZIP - # export PATH=$PATH:$HOME/d/protoc/bin - # protoc --version - # - name: Setup Rust toolchain - # run: | - # rustup toolchain install stable - # rustup default stable - # rustup component add rustfmt clippy - # - name: Cache Cargo - # uses: actions/cache@v4 - # with: - # path: /home/runner/.cargo - # # this key is not equal because the user is different than on a container (runner vs github) - # key: cargo-coverage-cache3- - # - name: Run coverage - # run: | - # export PATH=$PATH:$HOME/d/protoc/bin - # rustup toolchain install stable - # rustup default stable - # cargo install --version 0.20.1 cargo-tarpaulin - # cargo tarpaulin --all --out Xml - # - name: Report coverage - # continue-on-error: true - # run: bash <(curl -s https://codecov.io/bash) - - clippy: - name: clippy - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install Clippy - run: rustup component add clippy - - name: Run clippy - run: ci/scripts/rust_clippy.sh - - cargo-toml-formatting-checks: - name: check Cargo.toml formatting - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install taplo - run: cargo +stable install taplo-cli --version ^0.9 --locked - # if you encounter an error, try running 'taplo format' to fix the formatting automatically. - - name: Check Cargo.toml formatting - run: taplo format --check - - config-docs-check: - name: check configs.md and ***_functions.md is up-to-date - needs: linux-build-lib - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - uses: actions/setup-node@v4 - with: - node-version: "20" - - name: Check if configs.md has been modified - run: | - # If you encounter an error, run './dev/update_config_docs.sh' and commit - ./dev/update_config_docs.sh - git diff --exit-code - - name: Check if any of the ***_functions.md has been modified - run: | - # If you encounter an error, run './dev/update_function_docs.sh' and commit - ./dev/update_function_docs.sh - git diff --exit-code - - # Verify MSRV for the crates which are directly used by other projects: - # - datafusion - # - datafusion-substrait - # - datafusion-proto - # - datafusion-cli - msrv: - name: Verify MSRV (Min Supported Rust Version) - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - - name: Install cargo-msrv - run: cargo install cargo-msrv - - name: Check datafusion - working-directory: datafusion/core - run: | - # If you encounter an error with any of the commands below it means - # your code or some crate in the dependency tree has a higher MSRV - # (Min Supported Rust Version) than the one specified in the - # `rust-version` key of `Cargo.toml`. - # - # To reproduce: - # 1. Install the version of Rust that is failing. Example: - # rustup install 1.80.1 - # 2. Run the command that failed with that version. Example: - # cargo +1.80.1 check -p datafusion - # - # To resolve, either: - # 1. Change your code to use older Rust features, - # 2. Revert dependency update - # 3. Update the MSRV version in `Cargo.toml` - # - # Please see the DataFusion Rust Version Compatibility Policy before - # updating Cargo.toml. You may have to update the code instead. - # https://github.com/apache/datafusion/blob/main/README.md#rust-version-compatibility-policy - cargo msrv --output-format json --log-target stdout verify - - name: Check datafusion-substrait - working-directory: datafusion/substrait - run: cargo msrv --output-format json --log-target stdout verify - - name: Check datafusion-proto - working-directory: datafusion/proto - run: cargo msrv --output-format json --log-target stdout verify