org.scalatest.exceptions.TestFailedException: == Results == !== Correct Answer - 10 == == Spark Answer - 0 == !struct<_1:timestamp,_2:int> struct<> ![1969-12-31 16:00:00.0,0] ![1969-12-31 16:00:00.1,1] ![1969-12-31 16:00:00.2,2] ![1969-12-31 16:00:00.3,3] ![1969-12-31 16:00:00.4,4] ![1969-12-31 16:00:00.5,5] ![1969-12-31 16:00:00.6,6] ![1969-12-31 16:00:00.7,7] ![1969-12-31 16:00:00.8,8] ![1969-12-31 16:00:00.9,9] == Progress == AdvanceRateManualClock(1) => CheckLastBatch: [1969-12-31 16:00:00.0,0],[1969-12-31 16:00:00.1,1],[1969-12-31 16:00:00.2,2],[1969-12-31 16:00:00.3,3],[1969-12-31 16:00:00.4,4],[1969-12-31 16:00:00.5,5],[1969-12-31 16:00:00.6,6],[1969-12-31 16:00:00.7,7],[1969-12-31 16:00:00.8,8],[1969-12-31 16:00:00.9,9] StopStream StartStream(ProcessingTime(0),org.apache.spark.util.SystemClock@6b352232,Map(),null) AdvanceRateManualClock(2) CheckLastBatch: [1969-12-31 16:00:01.0,10],[1969-12-31 16:00:01.1,11],[1969-12-31 16:00:01.2,12],[1969-12-31 16:00:01.3,13],[1969-12-31 16:00:01.4,14],[1969-12-31 16:00:01.5,15],[1969-12-31 16:00:01.6,16],[1969-12-31 16:00:01.7,17],[1969-12-31 16:00:01.8,18],[1969-12-31 16:00:01.9,19] == Stream == Output Mode: Append Stream state: {org.apache.spark.sql.execution.streaming.sources.RateStreamMicroBatchReader@4b1641eb: {"0":{"value":-1,"runTimeMs":0}}} Thread state: alive Thread stack trace: com.fasterxml.jackson.core.sym.CharsToNameCanonicalizer$Bucket.has(CharsToNameCanonicalizer.java:784) com.fasterxml.jackson.core.sym.CharsToNameCanonicalizer.findSymbol(CharsToNameCanonicalizer.java:463) com.fasterxml.jackson.core.json.ReaderBasedJsonParser._parseName(ReaderBasedJsonParser.java:1625) com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:642) org.json4s.jackson.JValueDeserializer.deserialize(JValueDeserializer.scala:38) com.fasterxml.jackson.databind.ObjectReader._bindAndClose(ObjectReader.java:1511) com.fasterxml.jackson.databind.ObjectReader.readValue(ObjectReader.java:1136) org.json4s.jackson.JsonMethods$class.parse(JsonMethods.scala:25) org.json4s.jackson.JsonMethods$.parse(JsonMethods.scala:55) org.apache.spark.sql.streaming.SourceProgress.tryParse(progress.scala:190) org.apache.spark.sql.streaming.SourceProgress.jsonValue(progress.scala:182) org.apache.spark.sql.streaming.StreamingQueryProgress$$anonfun$jsonValue$4.apply(progress.scala:141) org.apache.spark.sql.streaming.StreamingQueryProgress$$anonfun$jsonValue$4.apply(progress.scala:141) scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186) scala.collection.TraversableLike$class.map(TraversableLike.scala:234) scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:186) org.apache.spark.sql.streaming.StreamingQueryProgress.jsonValue(progress.scala:141) org.apache.spark.sql.streaming.StreamingQueryProgress.prettyJson(progress.scala:114) org.apache.spark.sql.streaming.StreamingQueryProgress.toString(progress.scala:116) java.lang.String.valueOf(String.java:2994) java.lang.StringBuilder.append(StringBuilder.java:131) scala.StringContext.standardInterpolator(StringContext.scala:125) scala.StringContext.s(StringContext.scala:95) org.apache.spark.sql.execution.streaming.ProgressReporter$$anonfun$updateProgress$1.apply(ProgressReporter.scala:126) org.apache.spark.sql.execution.streaming.ProgressReporter$$anonfun$updateProgress$1.apply(ProgressReporter.scala:126) org.apache.spark.internal.Logging$class.logInfo(Logging.scala:54) org.apache.spark.sql.execution.streaming.StreamExecution.logInfo(StreamExecution.scala:58) org.apache.spark.sql.execution.streaming.ProgressReporter$class.updateProgress(ProgressReporter.scala:126) org.apache.spark.sql.execution.streaming.ProgressReporter$class.finishTrigger(ProgressReporter.scala:172) org.apache.spark.sql.execution.streaming.StreamExecution.finishTrigger(StreamExecution.scala:58) org.apache.spark.sql.execution.streaming.MicroBatchExecution$$anonfun$runActivatedStream$1.apply$mcZ$sp(MicroBatchExecution.scala:151) org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:56) org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:131) org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:279) org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:189) == Sink == 0: 1: [1969-12-31 16:00:00.0,0] [1969-12-31 16:00:00.1,1] [1969-12-31 16:00:00.2,2] [1969-12-31 16:00:00.3,3] [1969-12-31 16:00:00.4,4] [1969-12-31 16:00:00.5,5] [1969-12-31 16:00:00.6,6] [1969-12-31 16:00:00.7,7] [1969-12-31 16:00:00.8,8] [1969-12-31 16:00:00.9,9] == Plan == == Parsed Logical Plan == WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62 +- Project [timestamp#26485 AS timestamp#26452, value#26486L AS value#26453L] +- Streaming RelationV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true]) == Analyzed Logical Plan == WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62 +- Project [timestamp#26485 AS timestamp#26452, value#26486L AS value#26453L] +- Streaming RelationV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true]) == Optimized Logical Plan == WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62 +- Streaming RelationV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true]) == Physical Plan == WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62 +- *(1) ScanV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true])
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException:
== Results ==
!== Correct Answer - 10 == == Spark Answer - 0 ==
!struct<_1:timestamp,_2:int> struct<>
![1969-12-31 16:00:00.0,0]
![1969-12-31 16:00:00.1,1]
![1969-12-31 16:00:00.2,2]
![1969-12-31 16:00:00.3,3]
![1969-12-31 16:00:00.4,4]
![1969-12-31 16:00:00.5,5]
![1969-12-31 16:00:00.6,6]
![1969-12-31 16:00:00.7,7]
![1969-12-31 16:00:00.8,8]
![1969-12-31 16:00:00.9,9]
== Progress ==
AdvanceRateManualClock(1)
=> CheckLastBatch: [1969-12-31 16:00:00.0,0],[1969-12-31 16:00:00.1,1],[1969-12-31 16:00:00.2,2],[1969-12-31 16:00:00.3,3],[1969-12-31 16:00:00.4,4],[1969-12-31 16:00:00.5,5],[1969-12-31 16:00:00.6,6],[1969-12-31 16:00:00.7,7],[1969-12-31 16:00:00.8,8],[1969-12-31 16:00:00.9,9]
StopStream
StartStream(ProcessingTime(0),org.apache.spark.util.SystemClock@6b352232,Map(),null)
AdvanceRateManualClock(2)
CheckLastBatch: [1969-12-31 16:00:01.0,10],[1969-12-31 16:00:01.1,11],[1969-12-31 16:00:01.2,12],[1969-12-31 16:00:01.3,13],[1969-12-31 16:00:01.4,14],[1969-12-31 16:00:01.5,15],[1969-12-31 16:00:01.6,16],[1969-12-31 16:00:01.7,17],[1969-12-31 16:00:01.8,18],[1969-12-31 16:00:01.9,19]
== Stream ==
Output Mode: Append
Stream state: {org.apache.spark.sql.execution.streaming.sources.RateStreamMicroBatchReader@4b1641eb: {"0":{"value":-1,"runTimeMs":0}}}
Thread state: alive
Thread stack trace: com.fasterxml.jackson.core.sym.CharsToNameCanonicalizer$Bucket.has(CharsToNameCanonicalizer.java:784)
com.fasterxml.jackson.core.sym.CharsToNameCanonicalizer.findSymbol(CharsToNameCanonicalizer.java:463)
com.fasterxml.jackson.core.json.ReaderBasedJsonParser._parseName(ReaderBasedJsonParser.java:1625)
com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:642)
org.json4s.jackson.JValueDeserializer.deserialize(JValueDeserializer.scala:38)
com.fasterxml.jackson.databind.ObjectReader._bindAndClose(ObjectReader.java:1511)
com.fasterxml.jackson.databind.ObjectReader.readValue(ObjectReader.java:1136)
org.json4s.jackson.JsonMethods$class.parse(JsonMethods.scala:25)
org.json4s.jackson.JsonMethods$.parse(JsonMethods.scala:55)
org.apache.spark.sql.streaming.SourceProgress.tryParse(progress.scala:190)
org.apache.spark.sql.streaming.SourceProgress.jsonValue(progress.scala:182)
org.apache.spark.sql.streaming.StreamingQueryProgress$$anonfun$jsonValue$4.apply(progress.scala:141)
org.apache.spark.sql.streaming.StreamingQueryProgress$$anonfun$jsonValue$4.apply(progress.scala:141)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186)
scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:186)
org.apache.spark.sql.streaming.StreamingQueryProgress.jsonValue(progress.scala:141)
org.apache.spark.sql.streaming.StreamingQueryProgress.prettyJson(progress.scala:114)
org.apache.spark.sql.streaming.StreamingQueryProgress.toString(progress.scala:116)
java.lang.String.valueOf(String.java:2994)
java.lang.StringBuilder.append(StringBuilder.java:131)
scala.StringContext.standardInterpolator(StringContext.scala:125)
scala.StringContext.s(StringContext.scala:95)
org.apache.spark.sql.execution.streaming.ProgressReporter$$anonfun$updateProgress$1.apply(ProgressReporter.scala:126)
org.apache.spark.sql.execution.streaming.ProgressReporter$$anonfun$updateProgress$1.apply(ProgressReporter.scala:126)
org.apache.spark.internal.Logging$class.logInfo(Logging.scala:54)
org.apache.spark.sql.execution.streaming.StreamExecution.logInfo(StreamExecution.scala:58)
org.apache.spark.sql.execution.streaming.ProgressReporter$class.updateProgress(ProgressReporter.scala:126)
org.apache.spark.sql.execution.streaming.ProgressReporter$class.finishTrigger(ProgressReporter.scala:172)
org.apache.spark.sql.execution.streaming.StreamExecution.finishTrigger(StreamExecution.scala:58)
org.apache.spark.sql.execution.streaming.MicroBatchExecution$$anonfun$runActivatedStream$1.apply$mcZ$sp(MicroBatchExecution.scala:151)
org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:56)
org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:131)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:279)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:189)
== Sink ==
0:
1: [1969-12-31 16:00:00.0,0] [1969-12-31 16:00:00.1,1] [1969-12-31 16:00:00.2,2] [1969-12-31 16:00:00.3,3] [1969-12-31 16:00:00.4,4] [1969-12-31 16:00:00.5,5] [1969-12-31 16:00:00.6,6] [1969-12-31 16:00:00.7,7] [1969-12-31 16:00:00.8,8] [1969-12-31 16:00:00.9,9]
== Plan ==
== Parsed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62
+- Project [timestamp#26485 AS timestamp#26452, value#26486L AS value#26453L]
+- Streaming RelationV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true])
== Analyzed Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62
+- Project [timestamp#26485 AS timestamp#26452, value#26486L AS value#26453L]
+- Streaming RelationV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true])
== Optimized Logical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62
+- Streaming RelationV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true])
== Physical Plan ==
WriteToDataSourceV2 org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter@26f70f62
+- *(1) ScanV2 ratev2[timestamp#26485, value#26486L] (Options: [rowsPerSecond=10,numPartitions=1,useManualClock=true])
at org.scalatest.Assertions$class.newAssertionFailedException(Assertions.scala:528)
at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
at org.scalatest.Assertions$class.fail(Assertions.scala:1089)
at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
at org.apache.spark.sql.streaming.StreamTest$class.failTest$1(StreamTest.scala:430)
at org.apache.spark.sql.streaming.StreamTest$$anonfun$executeAction$1$18.apply(StreamTest.scala:669)
at org.apache.spark.sql.streaming.StreamTest$$anonfun$executeAction$1$18.apply(StreamTest.scala:669)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.sql.streaming.StreamTest$class.executeAction$1(StreamTest.scala:668)
at org.apache.spark.sql.streaming.StreamTest$$anonfun$liftedTree1$1$1.apply(StreamTest.scala:704)
at org.apache.spark.sql.streaming.StreamTest$$anonfun$liftedTree1$1$1.apply(StreamTest.scala:693)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.sql.streaming.StreamTest$class.liftedTree1$1(StreamTest.scala:693)
at org.apache.spark.sql.streaming.StreamTest$class.testStream(StreamTest.scala:692)
at org.apache.spark.sql.execution.streaming.RateSourceV2Suite.testStream(RateSourceV2Suite.scala:34)
at org.apache.spark.sql.execution.streaming.RateSourceV2Suite$$anonfun$1.apply$mcV$sp(RateSourceV2Suite.scala:66)
at org.apache.spark.sql.execution.streaming.RateSourceV2Suite$$anonfun$1.apply(RateSourceV2Suite.scala:59)
at org.apache.spark.sql.execution.streaming.RateSourceV2Suite$$anonfun$1.apply(RateSourceV2Suite.scala:59)
at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:103)
at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:183)
at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:196)
at org.apache.spark.sql.execution.streaming.RateSourceV2Suite.org$scalatest$BeforeAndAfterEach$$super$runTest(RateSourceV2Suite.scala:34)
at org.scalatest.BeforeAndAfterEach$class.runTest(BeforeAndAfterEach.scala:221)
at org.apache.spark.sql.execution.streaming.RateSourceV2Suite.runTest(RateSourceV2Suite.scala:34)
at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:396)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:384)
at scala.collection.immutable.List.foreach(List.scala:381)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:379)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:229)
at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
at org.scalatest.Suite$class.run(Suite.scala:1147)
at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:233)
at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:52)
at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:210)
at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:52)
at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314)
at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:480)
at sbt.ForkMain$Run$2.call(ForkMain.java:296)
at sbt.ForkMain$Run$2.call(ForkMain.java:286)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)