Merge remote-tracking branch 'apache/main' into alamb/upgrade_sqlparser

.github/pull_request_template.md

@@ -4,7 +4,7 @@
 We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
 -->
 
-Closes #.
+- Closes #.
 
 ## Rationale for this change
 

.github/workflows/dev.yml

@@ -28,7 +28,7 @@ jobs:
     name: Check License Header
     steps:
       - uses: actions/checkout@v4
-      - uses: korandoru/hawkeye@v5
+      - uses: korandoru/hawkeye@v6
 
   prettier:
     name: Use prettier to check formatting of documents

.github/workflows/rust.yml

@@ -45,7 +45,7 @@ jobs:
     name: Check License Header
     steps:
       - uses: actions/checkout@v4
-      - uses: korandoru/hawkeye@v5
+      - uses: korandoru/hawkeye@v6
 
   # Check crate compiles and base cargo check passes
   linux-build-lib:
@@ -60,7 +60,7 @@ jobs:
         with:
           rust-version: stable
       - name: Prepare cargo build
-        run: cargo check --profile ci --all-targets
+        run: cargo check --profile ci --all-targets --features integration-tests
 
   # cargo check common, functions and substrait with no default features
   linux-cargo-check-no-default-features:
@@ -92,8 +92,8 @@ jobs:
       - name: Check workspace in debug mode
         run: cargo check --profile ci --all-targets --workspace
 
-      - name: Check workspace with avro,json features
-        run: cargo check --profile ci --workspace --benches --features avro,json
+      - name: Check workspace with additional features
+        run: cargo check --profile ci --workspace --benches --features avro,json,integration-tests
 
       - name: Check Cargo.lock for datafusion-cli
         run: |
@@ -185,7 +185,7 @@ jobs:
         with:
           rust-version: stable
       - name: Run tests (excluding doctests)
-        run: cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace
+        run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -417,7 +417,7 @@ jobs:
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
-          cargo test --profile ci --lib --tests --bins --features avro,json,backtrace
+          cargo test --profile ci --lib --tests --bins --features avro,json,backtrace,integration-tests
           cd datafusion-cli
           cargo test --profile ci --lib --tests --bins --all-features
 

Cargo.toml

@@ -22,6 +22,7 @@ members = [
     "datafusion/common",
     "datafusion/common-runtime",
     "datafusion/catalog",
+    "datafusion/catalog-listing",
     "datafusion/core",
     "datafusion/expr",
     "datafusion/expr-common",
@@ -66,7 +67,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/datafusion"
 rust-version = "1.81.0"
-version = "44.0.0"
+version = "45.0.0"
 
 [workspace.dependencies]
 # We turn off default-features for some dependencies here so the workspaces which inherit them can
@@ -77,52 +78,53 @@ version = "44.0.0"
 ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
-arrow = { version = "54.0.0", features = [
+arrow = { version = "54.1.0", features = [
     "prettyprint",
 ] }
-arrow-array = { version = "54.0.0", default-features = false, features = [
+arrow-array = { version = "54.1.0", default-features = false, features = [
     "chrono-tz",
 ] }
-arrow-buffer = { version = "54.0.0", default-features = false }
-arrow-flight = { version = "54.0.0", features = [
+arrow-buffer = { version = "54.1.0", default-features = false }
+arrow-flight = { version = "54.1.0", features = [
     "flight-sql-experimental",
 ] }
-arrow-ipc = { version = "54.0.0", default-features = false, features = [
+arrow-ipc = { version = "54.1.0", default-features = false, features = [
     "lz4",
 ] }
-arrow-ord = { version = "54.0.0", default-features = false }
-arrow-schema = { version = "54.0.0", default-features = false }
+arrow-ord = { version = "54.1.0", default-features = false }
+arrow-schema = { version = "54.1.0", default-features = false }
 async-trait = "0.1.73"
 bigdecimal = "0.4.7"
-bytes = "1.4"
+bytes = "1.10"
 chrono = { version = "0.4.38", default-features = false }
 ctor = "0.2.9"
 dashmap = "6.0.1"
-datafusion = { path = "datafusion/core", version = "44.0.0", default-features = false }
-datafusion-catalog = { path = "datafusion/catalog", version = "44.0.0" }
-datafusion-common = { path = "datafusion/common", version = "44.0.0", default-features = false }
-datafusion-common-runtime = { path = "datafusion/common-runtime", version = "44.0.0" }
-datafusion-doc = { path = "datafusion/doc", version = "44.0.0" }
-datafusion-execution = { path = "datafusion/execution", version = "44.0.0" }
-datafusion-expr = { path = "datafusion/expr", version = "44.0.0" }
-datafusion-expr-common = { path = "datafusion/expr-common", version = "44.0.0" }
-datafusion-ffi = { path = "datafusion/ffi", version = "44.0.0" }
-datafusion-functions = { path = "datafusion/functions", version = "44.0.0" }
-datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "44.0.0" }
-datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "44.0.0" }
-datafusion-functions-nested = { path = "datafusion/functions-nested", version = "44.0.0" }
-datafusion-functions-table = { path = "datafusion/functions-table", version = "44.0.0" }
-datafusion-functions-window = { path = "datafusion/functions-window", version = "44.0.0" }
-datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "44.0.0" }
-datafusion-macros = { path = "datafusion/macros", version = "44.0.0" }
-datafusion-optimizer = { path = "datafusion/optimizer", version = "44.0.0", default-features = false }
-datafusion-physical-expr = { path = "datafusion/physical-expr", version = "44.0.0", default-features = false }
-datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "44.0.0", default-features = false }
-datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "44.0.0" }
-datafusion-physical-plan = { path = "datafusion/physical-plan", version = "44.0.0" }
-datafusion-proto = { path = "datafusion/proto", version = "44.0.0" }
-datafusion-proto-common = { path = "datafusion/proto-common", version = "44.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "44.0.0" }
+datafusion = { path = "datafusion/core", version = "45.0.0", default-features = false }
+datafusion-catalog = { path = "datafusion/catalog", version = "45.0.0" }
+datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "45.0.0" }
+datafusion-common = { path = "datafusion/common", version = "45.0.0", default-features = false }
+datafusion-common-runtime = { path = "datafusion/common-runtime", version = "45.0.0" }
+datafusion-doc = { path = "datafusion/doc", version = "45.0.0" }
+datafusion-execution = { path = "datafusion/execution", version = "45.0.0" }
+datafusion-expr = { path = "datafusion/expr", version = "45.0.0" }
+datafusion-expr-common = { path = "datafusion/expr-common", version = "45.0.0" }
+datafusion-ffi = { path = "datafusion/ffi", version = "45.0.0" }
+datafusion-functions = { path = "datafusion/functions", version = "45.0.0" }
+datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "45.0.0" }
+datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "45.0.0" }
+datafusion-functions-nested = { path = "datafusion/functions-nested", version = "45.0.0" }
+datafusion-functions-table = { path = "datafusion/functions-table", version = "45.0.0" }
+datafusion-functions-window = { path = "datafusion/functions-window", version = "45.0.0" }
+datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "45.0.0" }
+datafusion-macros = { path = "datafusion/macros", version = "45.0.0" }
+datafusion-optimizer = { path = "datafusion/optimizer", version = "45.0.0", default-features = false }
+datafusion-physical-expr = { path = "datafusion/physical-expr", version = "45.0.0", default-features = false }
+datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "45.0.0", default-features = false }
+datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "45.0.0" }
+datafusion-physical-plan = { path = "datafusion/physical-plan", version = "45.0.0" }
+datafusion-proto = { path = "datafusion/proto", version = "45.0.0" }
+datafusion-proto-common = { path = "datafusion/proto-common", version = "45.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "45.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"
@@ -133,12 +135,13 @@ itertools = "0.14"
 log = "^0.4"
 object_store = { version = "0.11.0", default-features = false }
 parking_lot = "0.12"
-parquet = { version = "54.0.0", default-features = false, features = [
+parquet = { version = "54.1.0", default-features = false, features = [
     "arrow",
     "async",
     "object_store",
 ] }
 pbjson = { version = "0.7.0" }
+pbjson-types = "0.7"
 # Should match arrow-flight's version of prost.
 prost = "0.13.1"
 prost-derive = "0.13.1"
@@ -149,7 +152,7 @@ rstest = "0.24.0"
 serde_json = "1"
 sqlparser = { version = "0.54.0", features = ["visitor"] }
 tempfile = "3"
-tokio = { version = "1.36", features = ["macros", "rt", "sync"] }
+tokio = { version = "1.43", features = ["macros", "rt", "sync"] }
 url = "2.5.4"
 
 [profile.release]

README.md

@@ -22,6 +22,8 @@
 [![Crates.io][crates-badge]][crates-url]
 [![Apache licensed][license-badge]][license-url]
 [![Build Status][actions-badge]][actions-url]
+![Commit Activity][commit-activity-badge]
+[![Open Issues][open-issues-badge]][open-issues-url]
 [![Discord chat][discord-badge]][discord-url]
 
 [crates-badge]: https://img.shields.io/crates/v/datafusion.svg
@@ -32,6 +34,9 @@
 [actions-url]: https://github.com/apache/datafusion/actions?query=branch%3Amain
 [discord-badge]: https://img.shields.io/discord/885562378132000778.svg?logo=discord&style=flat-square
 [discord-url]: https://discord.com/invite/Qw5gKqHxUM
+[commit-activity-badge]: https://img.shields.io/github/commit-activity/m/apache/datafusion
+[open-issues-badge]: https://img.shields.io/github/issues-raw/apache/datafusion
+[open-issues-url]: https://github.com/apache/datafusion/issues
 
 [Website](https://datafusion.apache.org/) |
 [API Docs](https://docs.rs/datafusion/latest/datafusion/) |

ci/scripts/rust_clippy.sh

@@ -18,6 +18,6 @@
 # under the License.
 
 set -ex
-cargo clippy --all-targets --workspace --features avro,pyarrow -- -D warnings
+cargo clippy --all-targets --workspace --features avro,pyarrow,integration-tests -- -D warnings
 cd datafusion-cli
 cargo clippy --all-targets --all-features -- -D warnings