[Spark] Update to Spark 3.5.4 (#237)

.github/workflows/main-workflow.yml

@@ -103,7 +103,7 @@ jobs:
       base_image: python-minimal
       python_version_1: 3.12.7
       python_version_2: 3.11.10
-      spark_version: 3.5.3
+      spark_version: 3.5.4
       build_gpu: false
     secrets: inherit
   jupyter-pyspark:
@@ -115,7 +115,7 @@ jobs:
       base_image: pyspark
       python_version_1: 3.12.7
       python_version_2: 3.11.10
-      spark_version: 3.5.3
+      spark_version: 3.5.4
       build_gpu: false
     secrets: inherit
   vscode-pyspark:
@@ -127,7 +127,7 @@ jobs:
       base_image: pyspark
       python_version_1: 3.12.7
       python_version_2: 3.11.10
-      spark_version: 3.5.3
+      spark_version: 3.5.4
       build_gpu: false
     secrets: inherit
   jupyter-pytorch:
@@ -189,7 +189,7 @@ jobs:
       base_image: r-minimal
       r_version_1: 4.4.2
       r_version_2: 4.3.3
-      spark_version: 3.5.3
+      spark_version: 3.5.4
       build_gpu: false
     secrets: inherit
   rstudio-sparkr:
@@ -201,7 +201,7 @@ jobs:
       base_image: sparkr
       r_version_1: 4.4.2
       r_version_2: 4.3.3
-      spark_version: 3.5.3
+      spark_version: 3.5.4
       build_gpu: false
     secrets: inherit
   jupyter-r:

scripts/install-spark-hadoop-hive.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-HADOOP_VERSION="3.4.0"
+HADOOP_VERSION="3.4.1"
 HIVE_VERSION="2.3.9"
 HIVE_LISTENER_VERSION="0.0.3"
 
@@ -48,4 +48,4 @@ mv jline-2.14.6.jar ${HIVE_HOME}/lib/
 rm ${HIVE_HOME}/lib/jline-2.12.jar
 
 # Fix multiple bindings
-rm ${HADOOP_HOME}/share/hadoop/tools/lib/bundle-2.23.19.jar
+rm ${HADOOP_HOME}/share/hadoop/tools/lib/bundle-2.24.6.jar

spark/Dockerfile

@@ -5,7 +5,7 @@ LABEL maintainer="InseeFrLab <[email protected]>"
 
 ARG BASE_IMAGE
 
-ARG SPARK_VERSION="3.5.3"
+ARG SPARK_VERSION="3.5.4"
 ENV SPARK_VERSION=${SPARK_VERSION}
 ENV JAVA_VERSION="17"