diff --git a/transportable-udfs-hive/build.gradle b/transportable-udfs-hive/build.gradle index 2ab8f7f0..6b663d79 100644 --- a/transportable-udfs-hive/build.gradle +++ b/transportable-udfs-hive/build.gradle @@ -5,14 +5,16 @@ dependencies { implementation project(':transportable-udfs-api') implementation project(':transportable-udfs-utils') implementation 'org.apache.hadoop:hadoop-common:2.7.4' - compileOnly('org.apache.hive:hive-exec:1.2.2') { + compileOnly('org.apache.hive:hive-exec:2.3.9') { exclude group: 'org.apache.avro' exclude group: 'org.apache.calcite' + exclude group: 'org.pentaho' } testImplementation project(path: ':transportable-udfs-type-system', configuration: 'tests') - testImplementation('org.apache.hive:hive-exec:1.2.2') { + testImplementation('org.apache.hive:hive-exec:2.3.9') { exclude group: 'org.apache.avro' exclude group: 'org.apache.calcite' + exclude group: 'org.pentaho' } } @@ -30,3 +32,20 @@ configurations { artifacts { tests jarTests } + +// Hive 2.3.9's ObjectInspectorFactory uses reflection on java.* internals that JDK 17 modularizes. +// Open the modules it touches when this subproject's tests run on JDK 17+. Limited to this subproject +// because no other transport subproject loads Hive 2.3.9 classes in its tests. +test { + if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) { + jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.lang.invoke=ALL-UNNAMED', + '--add-opens=java.base/java.io=ALL-UNNAMED', + '--add-opens=java.base/java.net=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED', + '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens=java.base/sun.security.action=ALL-UNNAMED' + } +} diff --git a/transportable-udfs-plugin/build.gradle b/transportable-udfs-plugin/build.gradle index 84216265..450dc936 100644 --- a/transportable-udfs-plugin/build.gradle +++ b/transportable-udfs-plugin/build.gradle @@ -22,7 +22,7 @@ dependencies { def writeVersionInfo = { file -> ant.propertyfile(file: file) { entry(key: "transport-version", value: version) - entry(key: "hive-version", value: '1.2.2') + entry(key: "hive-version", value: '2.3.9') entry(key: "trino-version", value: '406') entry(key: "spark_2.11-version", value: '2.3.0') entry(key: "spark_2.12-version", value: '3.1.1') diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java index de7c6b6a..206a576b 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java @@ -90,10 +90,11 @@ private static final String getVersion(final String platform) { new Platform(HIVE, Language.JAVA, HiveWrapperGenerator.class, - JavaLanguageVersion.of(8), + JavaLanguageVersion.of(17), ImmutableList.of( DependencyConfiguration.builder(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-hive", TRANSPORT_VERSION).build(), - DependencyConfiguration.builder(COMPILE_ONLY, "org.apache.hive:hive-exec", HIVE_VERSION).exclude("org.apache.calcite").build() + DependencyConfiguration.builder(COMPILE_ONLY, "org.apache.hive:hive-exec", HIVE_VERSION) + .exclude("org.apache.calcite").exclude("org.pentaho").build() ), ImmutableList.of( DependencyConfiguration.builder(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-hive", TRANSPORT_VERSION).build() diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 986b3afc..87c9f640 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -204,6 +204,17 @@ private TaskProvider configureGenerateWrappersTask(Project task.getJavaCompiler().set(javaToolchains.compilerFor(toolChainSpec -> { toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion()); })); + // For Hive/Spark: pin bytecode to Java 8 (release=8) so UDF jars stay runnable on + // Java 8 runtimes. Trino is excluded because Trino 406+ requires Java 17 bytecode + // (uses sealed classes, records, etc.). + // Some build environments already configure the bytecode target by passing a + // "--release" compiler argument. Gradle forbids specifying --release through both + // compilerArgs and JavaCompile.release, so only set the release here when it has + // not already been configured externally. + if (!"trino".equals(platform.getName()) + && !task.getOptions().getCompilerArgs().contains("--release")) { + task.getOptions().getRelease().set(8); + } }); } @@ -278,6 +289,22 @@ task trinoTest(type: Test, dependsOn: test) { task.getJavaLauncher().set(javaToolchains.launcherFor(toolChainSpec -> { toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion()); })); + + // When the test JVM is JDK 17 (e.g. hiveTest once its toolchain bumps from 8 to 17), + // Hive 2.3.9's embedded HiveServer2 + DataNucleus + Derby need extra --add-opens to access + // internals that JDK 17 modularizes. + if (platform.getJavaLanguageVersion().asInt() >= 17 && !"trino".equals(platform.getName())) { + task.jvmArgs( + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED"); + } }); } diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java index c5ea27d1..556af942 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java @@ -56,9 +56,9 @@ public List> configurePackagingTasks(Project projec // Explicitly set classifiers for the created distributions or else leads to Maven packaging issues due to multiple // artifacts with the same classifier - project.getTasks().named(platform.getName() + "DistTar", Tar.class, tar -> tar.setClassifier(platform.getName())); + project.getTasks().named(platform.getName() + "DistTar", Tar.class, tar -> tar.getArchiveClassifier().set(platform.getName())); project.getArtifacts().add(ShadowBasePlugin.CONFIGURATION_NAME, project.getTasks().named(platform.getName() + "DistTar", Tar.class)); - project.getTasks().named(platform.getName() + "DistZip", Zip.class, zip -> zip.setClassifier(platform.getName())); + project.getTasks().named(platform.getName() + "DistZip", Zip.class, zip -> zip.getArchiveClassifier().set(platform.getName())); project.getArtifacts().add(ShadowBasePlugin.CONFIGURATION_NAME, project.getTasks().named(platform.getName() + "DistZip", Zip.class)); return ImmutableList.of(project.getTasks().named(platform.getName() + "DistTar", Tar.class), project.getTasks().named(platform.getName() + "DistZip", Zip.class)); @@ -80,7 +80,7 @@ private TaskProvider createThinJarTask(Project project, SourceSet sourceSet task.dependsOn(project.getTasks().named(sourceSet.getClassesTaskName())); task.setDescription("Assembles a thin jar archive containing the " + platformName + " classes to be included in the distribution"); - task.setClassifier(platformName + "-dist-thin"); + task.getArchiveClassifier().set(platformName + "-dist-thin"); task.from(sourceSet.getOutput()); task.from(sourceSet.getResources()); }); diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ShadedJarPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ShadedJarPackaging.java index 92914cb6..fcc87744 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ShadedJarPackaging.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ShadedJarPackaging.java @@ -71,7 +71,7 @@ private TaskProvider createShadeTask(Project project, Platform platfo project.getTasks().register(sourceSet.getTaskName("shade", "Jar"), ShadeTask.class, task -> { task.setGroup(ShadowJavaPlugin.SHADOW_GROUP); task.setDescription("Create a combined JAR of " + platform.getName() + " output and runtime dependencies"); - task.setClassifier(platform.getName()); + task.getArchiveClassifier().set(platform.getName()); task.getManifest() .inheritFrom(project.getTasks().named(mainSourceSet.getJarTaskName(), Jar.class).get().getManifest()); task.from(sourceSet.getOutput()); diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java index 48db5f35..77d144e4 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java @@ -38,7 +38,7 @@ public List> configurePackagingTasks(Project projec task.dependsOn(project.getTasks().named(platformSourceSet.getClassesTaskName())); task.setDescription("Assembles a thin jar archive containing the " + platform.getName() + " classes to be included in the distribution"); - task.setClassifier(platform.getName() + "-thin"); + task.getArchiveClassifier().set(platform.getName() + "-thin"); task.from(platformSourceSet.getOutput()); task.from(platformSourceSet.getResources()); }); diff --git a/transportable-udfs-test/transportable-udfs-test-hive/build.gradle b/transportable-udfs-test/transportable-udfs-test-hive/build.gradle index ae6c83b3..9fa5297e 100644 --- a/transportable-udfs-test/transportable-udfs-test-hive/build.gradle +++ b/transportable-udfs-test/transportable-udfs-test-hive/build.gradle @@ -7,11 +7,15 @@ dependencies { implementation ('org.apache.calcite:calcite-core:1.2.0-incubating') { exclude group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm' } - implementation ('org.apache.hive:hive-exec:1.2.2') { + implementation ('org.apache.hive:hive-exec:2.3.9') { exclude group: 'org.apache.calcite' + exclude group: 'org.pentaho' } - implementation ('org.apache.hive:hive-service:1.2.2') { + implementation ('org.apache.hive:hive-service:2.3.9') { exclude group: 'org.apache.hive', module: 'hive-exec' + exclude group: 'org.pentaho' + exclude group: 'org.apache.twill' + exclude group: 'co.cask.tephra' } runtimeOnly 'org.apache.hadoop:hadoop-common:2.7.4' runtimeOnly 'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.4' diff --git a/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/HiveTester.java b/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/HiveTester.java index e2b447e6..bc7b4404 100644 --- a/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/HiveTester.java +++ b/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/HiveTester.java @@ -57,7 +57,10 @@ public HiveTester() { private void createHiveServer() { HiveServer2 server = new HiveServer2(); - server.init(new HiveConf()); + HiveConf conf = new HiveConf(); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false); + conf.set("datanucleus.schema.autoCreateAll", "true"); + server.init(conf); for (Service service : server.getServices()) { if (service instanceof CLIService) { _client = (CLIService) service; @@ -91,7 +94,7 @@ public void setup( String functionName = ((TopLevelStdUDF) stdUDFImplementations.get(0).getConstructor().newInstance()).getFunctionName(); _functionRegistryAddFunctionMethod.invoke(_functionRegistry, functionName, - new FunctionInfo(false, functionName, wrapper)); + new FunctionInfo(FunctionInfo.FunctionType.PERSISTENT, functionName, wrapper)); } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException | InstantiationException e) { throw new RuntimeException("Error registering UDF " + topLevelStdUDF.getName() + " with Hive Server", e); }