org.scalatest.exceptions.TestFailedException: Results do not match for query: == Parsed Logical Plan == 'ScriptTransformation ['a,'b], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], ScriptInputOutputSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false) +- 'UnresolvedRelation `test`, None == Analyzed Logical Plan == c: string, d: string ScriptTransformation [a#162569L,b#162570L], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], ScriptInputOutputSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false) +- SubqueryAlias test +- Project [id#162566L AS a#162569L,id#162566L AS b#162570L] +- Range 0, 5, 1, 1, [id#162566L] == Optimized Logical Plan == ScriptTransformation [a#162569L,b#162570L], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], ScriptInputOutputSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false) +- Project [id#162566L AS a#162569L,id#162566L AS b#162570L] +- Range 0, 5, 1, 1, [id#162566L] == Physical Plan == ScriptTransformation [a#162569L,b#162570L], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], HiveScriptIOSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false) +- WholeStageCodegen : +- Project [id#162566L AS a#162569L,id#162566L AS b#162570L] : +- Range 0, 1, 1, 5, [id#162566L] == Results == == Results == !== Correct Answer - 5 == == Spark Answer - 1 == ![0#,0#] [0#,] ![1#,1#] ![2#,2#] ![3#,3#] ![4#,4#]
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException:
Results do not match for query:
== Parsed Logical Plan ==
'ScriptTransformation ['a,'b], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], ScriptInputOutputSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false)
+- 'UnresolvedRelation `test`, None
== Analyzed Logical Plan ==
c: string, d: string
ScriptTransformation [a#162569L,b#162570L], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], ScriptInputOutputSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false)
+- SubqueryAlias test
+- Project [id#162566L AS a#162569L,id#162566L AS b#162570L]
+- Range 0, 5, 1, 1, [id#162566L]
== Optimized Logical Plan ==
ScriptTransformation [a#162569L,b#162570L], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], ScriptInputOutputSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false)
+- Project [id#162566L AS a#162569L,id#162566L AS b#162570L]
+- Range 0, 5, 1, 1, [id#162566L]
== Physical Plan ==
ScriptTransformation [a#162569L,b#162570L], python src/test/resources/data/scripts/test_transform.py "|", [c#162573,d#162574], HiveScriptIOSchema(List(),List(),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),Some(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe),ArrayBuffer((field.delim,|)),ArrayBuffer((field.delim,|)),None,None,false)
+- WholeStageCodegen
: +- Project [id#162566L AS a#162569L,id#162566L AS b#162570L]
: +- Range 0, 1, 1, 5, [id#162566L]
== Results ==
== Results ==
!== Correct Answer - 5 == == Spark Answer - 1 ==
![0#,0#] [0#,]
![1#,1#]
![2#,2#]
![3#,3#]
![4#,4#]
at org.scalatest.Assertions$class.newAssertionFailedException(Assertions.scala:495)
at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1555)
at org.scalatest.Assertions$class.fail(Assertions.scala:1328)
at org.scalatest.FunSuite.fail(FunSuite.scala:1555)
at org.apache.spark.sql.QueryTest.checkAnswer(QueryTest.scala:149)
at org.apache.spark.sql.hive.execution.SQLQuerySuite$$anonfun$58.apply$mcV$sp(SQLQuerySuite.scala:1106)
at org.apache.spark.sql.hive.execution.SQLQuerySuite$$anonfun$58.apply(SQLQuerySuite.scala:1089)
at org.apache.spark.sql.hive.execution.SQLQuerySuite$$anonfun$58.apply(SQLQuerySuite.scala:1089)
at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22)
at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166)
at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:56)
at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163)
at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175)
at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175)
at org.scalatest.FunSuite.runTest(FunSuite.scala:1555)
at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208)
at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
at scala.collection.immutable.List.foreach(List.scala:381)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483)
at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208)
at org.scalatest.FunSuite.runTests(FunSuite.scala:1555)
at org.scalatest.Suite$class.run(Suite.scala:1424)
at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555)
at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212)
at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212)
at org.scalatest.SuperEngine.runImpl(Engine.scala:545)
at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212)
at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:28)
at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257)
at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256)
at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:28)
at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:357)
at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:502)
at sbt.ForkMain$Run$2.call(ForkMain.java:296)
at sbt.ForkMain$Run$2.call(ForkMain.java:286)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)