Upgrade Hive client to 2.3.9 and HIVE platform JDK toolchain to 17

transportable-udfs-hive/build.gradle

@@ -5,14 +5,16 @@ dependencies {
   implementation project(':transportable-udfs-api')
   implementation project(':transportable-udfs-utils')
   implementation 'org.apache.hadoop:hadoop-common:2.7.4'
-  compileOnly('org.apache.hive:hive-exec:1.2.2') {
+  compileOnly('org.apache.hive:hive-exec:2.3.9') {
     exclude group: 'org.apache.avro'
     exclude group: 'org.apache.calcite'
+    exclude group: 'org.pentaho'
   }
   testImplementation project(path: ':transportable-udfs-type-system', configuration: 'tests')
-  testImplementation('org.apache.hive:hive-exec:1.2.2') {
+  testImplementation('org.apache.hive:hive-exec:2.3.9') {
     exclude group: 'org.apache.avro'
     exclude group: 'org.apache.calcite'
+    exclude group: 'org.pentaho'
   }
 }
 
@@ -30,3 +32,20 @@ configurations {
 artifacts {
   tests jarTests
 }
+
+// Hive 2.3.9's ObjectInspectorFactory uses reflection on java.* internals that JDK 17 modularizes.
+// Open the modules it touches when this subproject's tests run on JDK 17+. Limited to this subproject
+// because no other transport subproject loads Hive 2.3.9 classes in its tests.
+test {
+  if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) {
+    jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED',
+            '--add-opens=java.base/java.lang.invoke=ALL-UNNAMED',
+            '--add-opens=java.base/java.io=ALL-UNNAMED',
+            '--add-opens=java.base/java.net=ALL-UNNAMED',
+            '--add-opens=java.base/java.nio=ALL-UNNAMED',
+            '--add-opens=java.base/java.util=ALL-UNNAMED',
+            '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED',
+            '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED',
+            '--add-opens=java.base/sun.security.action=ALL-UNNAMED'
+  }
+}

transportable-udfs-plugin/build.gradle

@@ -22,7 +22,7 @@ dependencies {
 def writeVersionInfo = { file ->
   ant.propertyfile(file: file) {
     entry(key: "transport-version", value: version)
-    entry(key: "hive-version", value: '1.2.2')
+    entry(key: "hive-version", value: '2.3.9')
     entry(key: "trino-version", value: '406')
     entry(key: "spark_2.11-version", value: '2.3.0')
     entry(key: "spark_2.12-version", value: '3.1.1')

transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java

@@ -90,10 +90,11 @@ private static final String getVersion(final String platform) {
       new Platform(HIVE,
           Language.JAVA,
           HiveWrapperGenerator.class,
-          JavaLanguageVersion.of(8),
+          JavaLanguageVersion.of(17),
           ImmutableList.of(
               DependencyConfiguration.builder(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-hive", TRANSPORT_VERSION).build(),
-              DependencyConfiguration.builder(COMPILE_ONLY, "org.apache.hive:hive-exec", HIVE_VERSION).exclude("org.apache.calcite").build()
+              DependencyConfiguration.builder(COMPILE_ONLY, "org.apache.hive:hive-exec", HIVE_VERSION)
+                  .exclude("org.apache.calcite").exclude("org.pentaho").build()
           ),
           ImmutableList.of(
               DependencyConfiguration.builder(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-hive", TRANSPORT_VERSION).build()

transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java

@@ -204,6 +204,17 @@ private TaskProvider<GenerateWrappersTask> configureGenerateWrappersTask(Project
             task.getJavaCompiler().set(javaToolchains.compilerFor(toolChainSpec -> {
               toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion());
             }));
+            // For Hive/Spark: pin bytecode to Java 8 (release=8) so UDF jars stay runnable on
+            // Java 8 runtimes. Trino is excluded because Trino 406+ requires Java 17 bytecode
+            // (uses sealed classes, records, etc.).
+            // Some build environments already configure the bytecode target by passing a
+            // "--release" compiler argument. Gradle forbids specifying --release through both
+            // compilerArgs and JavaCompile.release, so only set the release here when it has
+            // not already been configured externally.
+            if (!"trino".equals(platform.getName())
+                && !task.getOptions().getCompilerArgs().contains("--release")) {
+              task.getOptions().getRelease().set(8);
+            }
           });
     }
 
@@ -278,6 +289,22 @@ task trinoTest(type: Test, dependsOn: test) {
       task.getJavaLauncher().set(javaToolchains.launcherFor(toolChainSpec -> {
         toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion());
       }));
+
+      // When the test JVM is JDK 17 (e.g. hiveTest once its toolchain bumps from 8 to 17),
+      // Hive 2.3.9's embedded HiveServer2 + DataNucleus + Derby need extra --add-opens to access
+      // internals that JDK 17 modularizes.
+      if (platform.getJavaLanguageVersion().asInt() >= 17 && !"trino".equals(platform.getName())) {
+        task.jvmArgs(
+            "--add-opens=java.base/java.lang=ALL-UNNAMED",
+            "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
+            "--add-opens=java.base/java.io=ALL-UNNAMED",
+            "--add-opens=java.base/java.net=ALL-UNNAMED",
+            "--add-opens=java.base/java.nio=ALL-UNNAMED",
+            "--add-opens=java.base/java.util=ALL-UNNAMED",
+            "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
+            "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
+            "--add-opens=java.base/sun.security.action=ALL-UNNAMED");
+      }
     });
   }
 

transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java

@@ -56,9 +56,9 @@ public List<TaskProvider<? extends Task>> configurePackagingTasks(Project projec
 
     // Explicitly set classifiers for the created distributions or else leads to Maven packaging issues due to multiple
     // artifacts with the same classifier
-    project.getTasks().named(platform.getName() + "DistTar", Tar.class, tar -> tar.setClassifier(platform.getName()));
+    project.getTasks().named(platform.getName() + "DistTar", Tar.class, tar -> tar.getArchiveClassifier().set(platform.getName()));
     project.getArtifacts().add(ShadowBasePlugin.CONFIGURATION_NAME, project.getTasks().named(platform.getName() + "DistTar", Tar.class));
-    project.getTasks().named(platform.getName() + "DistZip", Zip.class, zip -> zip.setClassifier(platform.getName()));
+    project.getTasks().named(platform.getName() + "DistZip", Zip.class, zip -> zip.getArchiveClassifier().set(platform.getName()));
     project.getArtifacts().add(ShadowBasePlugin.CONFIGURATION_NAME, project.getTasks().named(platform.getName() + "DistZip", Zip.class));
     return ImmutableList.of(project.getTasks().named(platform.getName() + "DistTar", Tar.class),
         project.getTasks().named(platform.getName() + "DistZip", Zip.class));
@@ -80,7 +80,7 @@ private TaskProvider<Jar> createThinJarTask(Project project, SourceSet sourceSet
       task.dependsOn(project.getTasks().named(sourceSet.getClassesTaskName()));
       task.setDescription("Assembles a thin jar archive containing the " + platformName
           + " classes to be included in the distribution");
-      task.setClassifier(platformName + "-dist-thin");
+      task.getArchiveClassifier().set(platformName + "-dist-thin");
       task.from(sourceSet.getOutput());
       task.from(sourceSet.getResources());
     });

transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ShadedJarPackaging.java

@@ -71,7 +71,7 @@ private TaskProvider<ShadeTask> createShadeTask(Project project, Platform platfo
         project.getTasks().register(sourceSet.getTaskName("shade", "Jar"), ShadeTask.class, task -> {
           task.setGroup(ShadowJavaPlugin.SHADOW_GROUP);
           task.setDescription("Create a combined JAR of " + platform.getName() + " output and runtime dependencies");
-          task.setClassifier(platform.getName());
+          task.getArchiveClassifier().set(platform.getName());
           task.getManifest()
               .inheritFrom(project.getTasks().named(mainSourceSet.getJarTaskName(), Jar.class).get().getManifest());
           task.from(sourceSet.getOutput());

transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java

@@ -38,7 +38,7 @@ public List<TaskProvider<? extends Task>> configurePackagingTasks(Project projec
           task.dependsOn(project.getTasks().named(platformSourceSet.getClassesTaskName()));
           task.setDescription("Assembles a thin jar archive containing the " + platform.getName()
               + " classes to be included in the distribution");
-          task.setClassifier(platform.getName() + "-thin");
+          task.getArchiveClassifier().set(platform.getName() + "-thin");
           task.from(platformSourceSet.getOutput());
           task.from(platformSourceSet.getResources());
         });

transportable-udfs-test/transportable-udfs-test-hive/build.gradle

@@ -7,11 +7,15 @@ dependencies {
   implementation ('org.apache.calcite:calcite-core:1.2.0-incubating') {
     exclude group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm'
   }
-  implementation ('org.apache.hive:hive-exec:1.2.2') {
+  implementation ('org.apache.hive:hive-exec:2.3.9') {
     exclude group: 'org.apache.calcite'
+    exclude group: 'org.pentaho'
   }
-  implementation ('org.apache.hive:hive-service:1.2.2') {
+  implementation ('org.apache.hive:hive-service:2.3.9') {
     exclude group: 'org.apache.hive', module: 'hive-exec'
+    exclude group: 'org.pentaho'
+    exclude group: 'org.apache.twill'
+    exclude group: 'co.cask.tephra'
   }
   runtimeOnly 'org.apache.hadoop:hadoop-common:2.7.4'
   runtimeOnly 'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.4'

transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/HiveTester.java

@@ -57,7 +57,10 @@ public HiveTester() {
 
   private void createHiveServer() {
     HiveServer2 server = new HiveServer2();
-    server.init(new HiveConf());
+    HiveConf conf = new HiveConf();
+    conf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false);
+    conf.set("datanucleus.schema.autoCreateAll", "true");
+    server.init(conf);
     for (Service service : server.getServices()) {
       if (service instanceof CLIService) {
         _client = (CLIService) service;
@@ -91,7 +94,7 @@ public void setup(
         String functionName =
             ((TopLevelStdUDF) stdUDFImplementations.get(0).getConstructor().newInstance()).getFunctionName();
         _functionRegistryAddFunctionMethod.invoke(_functionRegistry, functionName,
-            new FunctionInfo(false, functionName, wrapper));
+            new FunctionInfo(FunctionInfo.FunctionType.PERSISTENT, functionName, wrapper));
       } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException | InstantiationException e) {
         throw new RuntimeException("Error registering UDF " + topLevelStdUDF.getName() + " with Hive Server", e);
       }