Skip to content

fix(hive): Failed to derive column type on CTE statement in ShiftArrayIndexTransformer #574

Description

@wForget

test sql:

  @Test
  public void testShiftArrayIndexOnCTE() {
    // String sql = "select split(b, ':')[0] as item from test.tableOne";
    String sql = "with t1 as (select b as str from test.tableOne) select split(str, ':')[0] as item from t1";
    System.out.println(relToString(sql));
  }

got an error:

Failed to derive the RelDataType for SqlNode: `split`(`str`, ':') with topSqlNode: SELECT `b` AS `str`
FROM `test`.`tableOne`
java.lang.RuntimeException: Failed to derive the RelDataType for SqlNode: `split`(`str`, ':') with topSqlNode: SELECT `b` AS `str`
FROM `test`.`tableOne`
	at com.linkedin.coral.common.utils.TypeDerivationUtil.getRelDataType(TypeDerivationUtil.java:94)
	at com.linkedin.coral.common.transformers.SqlCallTransformer.deriveRelDatatype(SqlCallTransformer.java:68)
	at com.linkedin.coral.transformers.ShiftArrayIndexTransformer.condition(ShiftArrayIndexTransformer.java:37)
	at com.linkedin.coral.common.transformers.SqlCallTransformer.apply(SqlCallTransformer.java:52)
	at com.linkedin.coral.common.transformers.SqlCallTransformers.apply(SqlCallTransformers.java:36)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:31)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:21)
	at org.apache.calcite.sql.SqlCall.accept(SqlCall.java:139)
	at org.apache.calcite.sql.util.SqlShuttle$CallCopyingArgHandler.visitChild(SqlShuttle.java:134)
	at org.apache.calcite.sql.util.SqlShuttle$CallCopyingArgHandler.visitChild(SqlShuttle.java:101)
	at org.apache.calcite.sql.SqlOperator.acceptCall(SqlOperator.java:868)
	at org.apache.calcite.sql.SqlAsOperator.acceptCall(SqlAsOperator.java:123)
	at org.apache.calcite.sql.util.SqlShuttle.visit(SqlShuttle.java:66)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:31)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:21)
	at org.apache.calcite.sql.SqlCall.accept(SqlCall.java:139)
	at org.apache.calcite.sql.util.SqlShuttle.visit(SqlShuttle.java:80)
	at org.apache.calcite.sql.util.SqlShuttle.visit(SqlShuttle.java:39)
	at org.apache.calcite.sql.SqlNodeList.accept(SqlNodeList.java:153)
	at org.apache.calcite.sql.util.SqlShuttle$CallCopyingArgHandler.visitChild(SqlShuttle.java:134)
	at org.apache.calcite.sql.util.SqlShuttle$CallCopyingArgHandler.visitChild(SqlShuttle.java:101)
	at org.apache.calcite.sql.SqlOperator.acceptCall(SqlOperator.java:868)
	at org.apache.calcite.sql.SqlSelectOperator.acceptCall(SqlSelectOperator.java:129)
	at org.apache.calcite.sql.util.SqlShuttle.visit(SqlShuttle.java:66)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:31)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:21)
	at org.apache.calcite.sql.SqlCall.accept(SqlCall.java:139)
	at org.apache.calcite.sql.util.SqlShuttle$CallCopyingArgHandler.visitChild(SqlShuttle.java:134)
	at org.apache.calcite.sql.util.SqlShuttle$CallCopyingArgHandler.visitChild(SqlShuttle.java:101)
	at org.apache.calcite.sql.SqlOperator.acceptCall(SqlOperator.java:868)
	at org.apache.calcite.sql.util.SqlShuttle.visit(SqlShuttle.java:66)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:31)
	at com.linkedin.coral.hive.hive2rel.HiveSqlNodeToCoralSqlNodeConverter.visit(HiveSqlNodeToCoralSqlNodeConverter.java:21)
	at org.apache.calcite.sql.SqlCall.accept(SqlCall.java:139)
	at com.linkedin.coral.hive.hive2rel.HiveToRelConverter.toSqlNode(HiveToRelConverter.java:112)
	at com.linkedin.coral.common.ToRelConverter.toSqlNode(ToRelConverter.java:149)
	at com.linkedin.coral.common.ToRelConverter.convertSql(ToRelConverter.java:130)
	at com.linkedin.coral.hive.hive2rel.HiveToRelConverterTest.relToString(HiveToRelConverterTest.java:701)
	at com.linkedin.coral.hive.hive2rel.HiveToRelConverterTest.testShiftArrayIndexOnCTE(HiveToRelConverterTest.java:697)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:80)
	at org.testng.internal.Invoker.invokeMethod(Invoker.java:701)
	at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:893)
	at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1218)
	at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
	at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
	at org.testng.TestRunner.privateRun(TestRunner.java:758)
	at org.testng.TestRunner.run(TestRunner.java:613)
	at org.testng.SuiteRunner.runTest(SuiteRunner.java:334)
	at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:329)
	at org.testng.SuiteRunner.privateRun(SuiteRunner.java:291)
	at org.testng.SuiteRunner.run(SuiteRunner.java:240)
	at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:53)
	at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:87)
	at org.testng.TestNG.runSuitesSequentially(TestNG.java:1170)
	at org.testng.TestNG.runSuitesLocally(TestNG.java:1095)
	at org.testng.TestNG.run(TestNG.java:1007)
	at org.gradle.api.internal.tasks.testing.testng.TestNGTestClassProcessor.runTests(TestNGTestClassProcessor.java:141)
	at org.gradle.api.internal.tasks.testing.testng.TestNGTestClassProcessor.stop(TestNGTestClassProcessor.java:90)
	at org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:61)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
	at org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
	at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
	at com.sun.proxy.$Proxy2.stop(Unknown Source)
	at org.gradle.api.internal.tasks.testing.worker.TestWorker.stop(TestWorker.java:133)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
	at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182)
	at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164)
	at org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414)
	at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
	at org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56)
	at java.lang.Thread.run(Thread.java:750)

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Fields

    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions