[FLINK-25002][build] Add java 17 add-opens/add-exports JVM arguments

love-star · Jun 16, 2023 · 2940c02 · 2940c02
1 parent e323a89
commit 2940c02
Show file tree

Hide file tree

Showing 42 changed files with 327 additions and 3 deletions.
diff --git a/.mvn/jvm.config b/.mvn/jvm.config
@@ -1 +1,7 @@
 -XX:+IgnoreUnrecognizedVMOptions
+--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
@@ -34,6 +34,13 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- core dependencies -->

diff --git a/flink-connectors/flink-connector-base/pom.xml b/flink-connectors/flink-connector-base/pom.xml
@@ -33,6 +33,13 @@
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<!-- Allow users to pass custom connector versions -->
 	<dependencies>
 		<dependency>

diff --git a/flink-connectors/flink-connector-files/pom.xml b/flink-connectors/flink-connector-files/pom.xml
@@ -34,6 +34,13 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- core dependencies -->

diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml
@@ -36,6 +36,12 @@ under the License.
 
 	<properties>
 		<kafka.version>3.2.3</kafka.version>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED <!--
+			Kryo
+			-->--add-opens=java.base/java.lang=ALL-UNNAMED
+		</surefire.module.config>
 	</properties>
 
 	<dependencies>

diff --git a/flink-core/pom.xml b/flink-core/pom.xml
@@ -33,6 +33,24 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config> <!--
+			required by JmxServer
+			-->--add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED <!--
+			PluginConfigTest (CommonTestUtils#setEnv)
+			-->--add-opens=java.base/java.util=ALL-UNNAMED <!--
+			ExceptionUtilsTest
+			-->--add-opens=java.base/java.lang=ALL-UNNAMED <!--
+			StateDescriptorTest (Kryo serialization of File)
+			-->--add-opens=java.base/java.io=ALL-UNNAMED <!--
+			SerializersTest (Kryo serialization of Nested1->Path->URI)
+			-->--add-opens=java.base/java.net=ALL-UNNAMED <!--
+			InitOutputPathTest
+			-->--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED <!--
+			-->--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>

diff --git a/flink-dist/src/main/resources/flink-conf.yaml b/flink-dist/src/main/resources/flink-conf.yaml
@@ -16,6 +16,9 @@
 #  limitations under the License.
 ################################################################################
 
+# These parameters are required for Java 17 support.
+# They can be safely removed when using Java 8/11.
+env.java.opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
 
 #==============================================================================
 # Common

diff --git a/flink-end-to-end-tests/flink-queryable-state-test/pom.xml b/flink-end-to-end-tests/flink-queryable-state-test/pom.xml
@@ -107,6 +107,9 @@
 							<transformers>
 								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
 									<mainClass>org.apache.flink.streaming.tests.queryablestate.QsStateClient</mainClass>
+									<manifestEntries>
+										<Add-Opens>java.base/java.util</Add-Opens>
+									</manifestEntries>
 								</transformer>
 							</transformers>
 						</configuration>

diff --git a/flink-end-to-end-tests/test-scripts/test_local_recovery_and_scheduling.sh b/flink-end-to-end-tests/test-scripts/test_local_recovery_and_scheduling.sh
@@ -73,6 +73,8 @@ function run_local_recovery_test {
     TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-local-recovery-and-allocation-test/target/StickyAllocationAndLocalRecoveryTestJob.jar
     # configure for HA
     create_ha_config
+    # required for PID business in StickyAllocationAndLocalRecoveryTestJob
+    set_config_key env.java.opts.taskmanager "--add-opens=java.management/sun.management=ALL-UNNAMED"
 
     # Enable debug logging
     sed -i -e 's/rootLogger.level = .*/rootLogger.level = DEBUG/' "$FLINK_DIR/conf/log4j.properties"

diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml
@@ -32,6 +32,13 @@ under the License.
 	<artifactId>flink-examples-table_${scala.binary.version}</artifactId>
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 		<!-- Flink core -->
 		<dependency>

diff --git a/flink-filesystems/flink-hadoop-fs/pom.xml b/flink-filesystems/flink-hadoop-fs/pom.xml
@@ -31,6 +31,17 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			HadoopConfigLoadingTest
+			-->--add-opens=java.base/java.util=ALL-UNNAMED <!--
+			HadoopUtilsTest
+			-->--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED <!--
+			HadoopDataInputStreamTest
+			-->--add-opens=java.base/java.io=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>

diff --git a/flink-formats/flink-avro/pom.xml b/flink-formats/flink-avro/pom.xml
@@ -33,6 +33,19 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			Kryo ByteBuffer
+			-->--add-opens=java.base/java.nio=ALL-UNNAMED <!--
+			Kryo LocalDate
+			-->--add-opens=java.base/java.time=ALL-UNNAMED <!--
+			Kryo
+			-->--add-opens=java.base/java.lang=ALL-UNNAMED <!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- Core -->

diff --git a/flink-formats/flink-csv/pom.xml b/flink-formats/flink-csv/pom.xml
@@ -33,6 +33,13 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<dependency>

diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml
@@ -33,6 +33,13 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<dependency>

diff --git a/flink-formats/flink-orc/pom.xml b/flink-formats/flink-orc/pom.xml
@@ -33,6 +33,15 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			Kryo buffer
+			-->--add-opens=java.base/java.nio=ALL-UNNAMED <!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- Core -->

diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml
@@ -33,6 +33,13 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- Core -->

diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml
@@ -31,6 +31,13 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<!--
 		This is a Hadoop2 only flink module.
 	-->

diff --git a/flink-java/pom.xml b/flink-java/pom.xml
@@ -33,6 +33,15 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config> <!--
+			JoinOperatorTest / chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED <!--
+			CollectionInputFormatTest
+			-->--add-opens=java.base/java.lang=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>

diff --git a/flink-kubernetes/pom.xml b/flink-kubernetes/pom.xml
@@ -32,6 +32,10 @@ under the License.
 
 	<properties>
 		<kubernetes.client.version>6.6.2</kubernetes.client.version>
+		<surefire.module.config><!--
+			CommonTestUtils#setEnv
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
 	</properties>
 
 	<dependencies>

diff --git a/flink-libraries/flink-cep/pom.xml b/flink-libraries/flink-cep/pom.xml
@@ -33,6 +33,13 @@ under the License.
     <name>Flink : Libraries : CEP</name>
     <packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
     <dependencies>
 
         <!-- core dependencies -->

diff --git a/flink-libraries/flink-state-processing-api/pom.xml b/flink-libraries/flink-state-processing-api/pom.xml
@@ -34,6 +34,15 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED <!--
+			Kryo
+			-->--add-opens=java.base/java.lang=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- core dependencies -->

diff --git a/flink-metrics/flink-metrics-jmx/pom.xml b/flink-metrics/flink-metrics-jmx/pom.xml
@@ -31,6 +31,13 @@ under the License.
 	<artifactId>flink-metrics-jmx</artifactId>
 	<name>Flink : Metrics : JMX</name>
 
+	<properties>
+		<surefire.module.config> <!--
+			required by JmxServer
+			-->--add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>

diff --git a/flink-python/pom.xml b/flink-python/pom.xml
@@ -35,6 +35,15 @@ under the License.
 
 	<properties>
 		<arrow.version>5.0.0</arrow.version>
+		<kafka.version>3.2.3</kafka.version>
+		<surefire.module.config><!--
+			CommonTestUtils#setEnv
+			-->--add-opens=java.base/java.util=ALL-UNNAMED <!--
+			Kryo AtomicBoolean
+			-->--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED <!--
+			Arrow MemoryUtil
+			-->--add-opens=java.base/java.nio=ALL-UNNAMED
+		</surefire.module.config>
 	</properties>
 
 	<dependencies>

diff --git a/flink-python/pyflink/pyflink_gateway_server.py b/flink-python/pyflink/pyflink_gateway_server.py
@@ -254,8 +254,10 @@ def launch_gateway_server_process(env, args):
             [construct_flink_classpath(env), construct_hadoop_classpath(env)])
         if "FLINK_TESTING" in env:
             classpath = os.pathsep.join([classpath, construct_test_classpath()])
-        command = [java_executable, jvm_args, "-XX:+IgnoreUnrecognizedVMOptions"] + jvm_opts \
-            + log_settings + ["-cp", classpath, program_args.main_class] + program_args.other_args
+        command = [java_executable, jvm_args, "-XX:+IgnoreUnrecognizedVMOptions",
+                   "--add-opens=jdk.proxy2/jdk.proxy2=ALL-UNNAMED"] \
+            + jvm_opts + log_settings \
+            + ["-cp", classpath, program_args.main_class] + program_args.other_args
     else:
         command = [os.path.join(env["FLINK_BIN_DIR"], "flink"), "run"] + program_args.other_args \
             + ["-c", program_args.main_class]

diff --git a/flink-queryable-state/flink-queryable-state-runtime/pom.xml b/flink-queryable-state/flink-queryable-state-runtime/pom.xml
@@ -33,6 +33,13 @@ under the License.
 	<name>Flink : Queryable state : Runtime</name>
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			chill ArraysAsListSerializer
+			-->--add-opens=java.base/java.util=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- core dependencies -->

diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
@@ -33,6 +33,20 @@ under the License.
 
 	<packaging>jar</packaging>
 
+	<properties>
+		<surefire.module.config><!--
+			JobManagerProcessUtilsTest
+			-->--add-opens=java.base/java.util=ALL-UNNAMED <!--
+			ConnectionUtilsTest
+			-->--add-opens=java.base/java.lang=ALL-UNNAMED <!--
+			-->--add-opens=java.base/java.net=ALL-UNNAMED <!--
+			OperatorStateBackendTest
+			-->--add-opens=java.base/java.io=ALL-UNNAMED <!--
+			AsynchronousFileIOChannelTest
+			-->--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+		</surefire.module.config>
+	</properties>
+
 	<dependencies>
 
 		<!-- core dependencies -->

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestJvmProcess.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestJvmProcess.java
@@ -150,6 +150,11 @@ public void startProcess() throws IOException {
             cmd = ArrayUtils.addAll(cmd, jvmArgs);
         }
 
+        final String moduleConfig = System.getProperty("surefire.module.config");
+        if (moduleConfig != null) {
+            cmd = ArrayUtils.addAll(cmd, moduleConfig.split(" "));
+        }
+
         synchronized (createDestroyLock) {
             checkState(process == null, "process already started");