Assert on query failed: Add partitions: The code passed to eventually never returned normally. Attempted 12 times over 2.8896464649499998 minutes. Last failure message: query.lastExecution.executedPlan.collectFirst[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](({
 @SerialVersionUID(value = 0) final <synthetic> class $anonfun extends scala.runtime.AbstractPartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig] with Serializable {
 def <init>(): <$anon: org.apache.spark.sql.execution.SparkPlan => org.apache.spark.sql.kafka010.KafkaContinuousScanConfig> = {
 $anonfun.super.<init>();
 ()
 };
 final override def applyOrElse[A1 <: org.apache.spark.sql.execution.SparkPlan, B1 >: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](x1: A1, default: A1 => B1): B1 = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
 case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => scan.scanConfig.asInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig]
 case (defaultCase$ @ _) => default.apply(x1)
 };
 final def isDefinedAt(x1: org.apache.spark.sql.execution.SparkPlan): Boolean = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
 case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => true
 case (defaultCase$ @ _) => false
 }
 };
 new $anonfun()
}: PartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig])).exists(((x$1: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig) => x$1.knownPartitions.size.==(newCount))) was false query never reconfigured to 10 partitions.
org.scalatest.concurrent.Eventually$class.tryTryAgain$1(Eventually.scala:421)
 org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:439)
 org.apache.spark.sql.kafka010.KafkaSourceTest.eventually(KafkaMicroBatchSourceSuite.scala:49)
 org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:337)
 org.apache.spark.sql.kafka010.KafkaSourceTest.eventually(KafkaMicroBatchSourceSuite.scala:49)
 org.apache.spark.sql.kafka010.KafkaContinuousTest$class.setTopicPartitions(KafkaContinuousTest.scala:47)
 org.apache.spark.sql.kafka010.KafkaContinuousSourceSuite.setTopicPartitions(KafkaContinuousSourceSuite.scala:28)
 org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$anonfun$org$apache$spark$sql$kafka010$KafkaSourceSuiteBase$$testFromLatestOffsets$2.apply(KafkaMicroBatchSourceSuite.scala:1353)
 org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$anonfun$org$apache$spark$sql$kafka010$KafkaSourceSuiteBase$$testFromLatestOffsets$2.apply(KafkaMicroBatchSourceSuite.scala:1352)
 org.apache.spark.sql.streaming.StreamTest$$anonfun$executeAction$1$11.apply$mcZ$sp(StreamTest.scala:652)

 Caused by: query.lastExecution.executedPlan.collectFirst[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](({
 @SerialVersionUID(value = 0) final <synthetic> class $anonfun extends scala.runtime.AbstractPartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig] with Serializable {
 def <init>(): <$anon: org.apache.spark.sql.execution.SparkPlan => org.apache.spark.sql.kafka010.KafkaContinuousScanConfig> = {
 $anonfun.super.<init>();
 ()
 };
 final override def applyOrElse[A1 <: org.apache.spark.sql.execution.SparkPlan, B1 >: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](x1: A1, default: A1 => B1): B1 = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
 case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => scan.scanConfig.asInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig]
 case (defaultCase$ @ _) => default.apply(x1)
 };
 final def isDefinedAt(x1: org.apache.spark.sql.execution.SparkPlan): Boolean = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
 case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => true
 case (defaultCase$ @ _) => false
 }
 };
 new $anonfun()
}: PartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig])).exists(((x$1: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig) => x$1.knownPartitions.size.==(newCount))) was false query never reconfigured to 10 partitions
 org.scalatest.Assertions$class.newAssertionFailedException(Assertions.scala:528)
 org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
 org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:501)
 org.apache.spark.sql.kafka010.KafkaContinuousTest$$anonfun$setTopicPartitions$1.apply(KafkaContinuousTest.scala:48)
 org.apache.spark.sql.kafka010.KafkaContinuousTest$$anonfun$setTopicPartitions$1.apply(KafkaContinuousTest.scala:48)
 org.scalatest.concurrent.Eventually$class.makeAValiantAttempt$1(Eventually.scala:395)
 org.scalatest.concurrent.Eventually$class.tryTryAgain$1(Eventually.scala:409)
 org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:439)
 org.apache.spark.sql.kafka010.KafkaSourceTest.eventually(KafkaMicroBatchSourceSuite.scala:49)
 org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:337)


== Progress ==
 AssertOnQuery(<condition>, )
 AddKafkaData(topics = Set(topic-4), data = WrappedArray(1, 2, 3), message = )
 CheckAnswer: [2],[3],[4]
 StopStream
 StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@40901497,Map(),null)
 CheckAnswer: [2],[3],[4]
 StopStream
 AddKafkaData(topics = Set(topic-4), data = WrappedArray(4, 5, 6), message = )
 StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1843358a,Map(),null)
 CheckAnswer: [2],[3],[4],[5],[6],[7]
 AddKafkaData(topics = Set(topic-4), data = WrappedArray(7, 8), message = )
 CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
=> AssertOnQuery(<condition>, Add partitions)
 AddKafkaData(topics = Set(topic-4), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )
 CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]

== Stream ==
Output Mode: Append
Stream state: {KafkaSource[Subscribe[topic-4]]: {"topic-4":{"2":3,"4":2,"1":0,"3":2,"0":2}}}
Thread state: alive
Thread stack trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:206)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:222)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:157)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:243)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:731)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2003)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2024)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2043)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2068)
org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:959)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:364)
org.apache.spark.rdd.RDD.collect(RDD.scala:958)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:53)
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3$$anonfun$apply$1.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3$$anonfun$apply$1.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:147)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:74)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.streaming.ProgressReporter$class.reportTimeTaken(ProgressReporter.scala:323)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:59)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:263)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:90)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:280)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:190)


== Sink ==
0: 
1: [4] [3] [2]
2: 
3: 
4: 
5: [6] [7] [5]
6: 
7: [9] [8]
8: 


== Plan ==
== Parsed Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- SerializeFromObject [input[0, int, false] AS value#13094]
 +- MapElements <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#13093: int
 +- DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
 +- Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
 +- Streaming RelationV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)

== Analyzed Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- SerializeFromObject [input[0, int, false] AS value#13094]
 +- MapElements <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#13093: int
 +- DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
 +- Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
 +- Streaming RelationV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)

== Optimized Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- SerializeFromObject [input[0, int, false] AS value#13094]
 +- MapElements <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#13093: int
 +- DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
 +- Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
 +- Streaming RelationV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)

== Physical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- *(1) SerializeFromObject [input[0, int, false] AS value#13094]
 +- *(1) MapElements <function1>, obj#13093: int
 +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
 +- *(1) Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
 +- *(1) Project [key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263]
 +- *(1) ScanV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)
 

org.scalatest.exceptions.TestFailedException:
Assert on query failed: Add partitions: The code passed to eventually never returned normally. Attempted 12 times over 2.8896464649499998 minutes. Last failure message: query.lastExecution.executedPlan.collectFirst[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](({
@SerialVersionUID(value = 0) final <synthetic> class $anonfun extends scala.runtime.AbstractPartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig] with Serializable {
def <init>(): <$anon: org.apache.spark.sql.execution.SparkPlan => org.apache.spark.sql.kafka010.KafkaContinuousScanConfig> = {
$anonfun.super.<init>();
()
};
final override def applyOrElse[A1 <: org.apache.spark.sql.execution.SparkPlan, B1 >: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](x1: A1, default: A1 => B1): B1 = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => scan.scanConfig.asInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig]
case (defaultCase$ @ _) => default.apply(x1)
};
final def isDefinedAt(x1: org.apache.spark.sql.execution.SparkPlan): Boolean = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => true
case (defaultCase$ @ _) => false
}
};
new $anonfun()
}: PartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig])).exists(((x$1: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig) => x$1.knownPartitions.size.==(newCount))) was false query never reconfigured to 10 partitions.
org.scalatest.concurrent.Eventually$class.tryTryAgain$1(Eventually.scala:421)
org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:439)
org.apache.spark.sql.kafka010.KafkaSourceTest.eventually(KafkaMicroBatchSourceSuite.scala:49)
org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:337)
org.apache.spark.sql.kafka010.KafkaSourceTest.eventually(KafkaMicroBatchSourceSuite.scala:49)
org.apache.spark.sql.kafka010.KafkaContinuousTest$class.setTopicPartitions(KafkaContinuousTest.scala:47)
org.apache.spark.sql.kafka010.KafkaContinuousSourceSuite.setTopicPartitions(KafkaContinuousSourceSuite.scala:28)
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$anonfun$org$apache$spark$sql$kafka010$KafkaSourceSuiteBase$$testFromLatestOffsets$2.apply(KafkaMicroBatchSourceSuite.scala:1353)
org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$anonfun$org$apache$spark$sql$kafka010$KafkaSourceSuiteBase$$testFromLatestOffsets$2.apply(KafkaMicroBatchSourceSuite.scala:1352)
org.apache.spark.sql.streaming.StreamTest$$anonfun$executeAction$1$11.apply$mcZ$sp(StreamTest.scala:652)
Caused by: query.lastExecution.executedPlan.collectFirst[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](({
@SerialVersionUID(value = 0) final <synthetic> class $anonfun extends scala.runtime.AbstractPartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig] with Serializable {
def <init>(): <$anon: org.apache.spark.sql.execution.SparkPlan => org.apache.spark.sql.kafka010.KafkaContinuousScanConfig> = {
$anonfun.super.<init>();
()
};
final override def applyOrElse[A1 <: org.apache.spark.sql.execution.SparkPlan, B1 >: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig](x1: A1, default: A1 => B1): B1 = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => scan.scanConfig.asInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousScanConfig]
case (defaultCase$ @ _) => default.apply(x1)
};
final def isDefinedAt(x1: org.apache.spark.sql.execution.SparkPlan): Boolean = ((x1.asInstanceOf[org.apache.spark.sql.execution.SparkPlan]: org.apache.spark.sql.execution.SparkPlan): org.apache.spark.sql.execution.SparkPlan @unchecked) match {
case (scan @ (_: org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec)) if scan.readSupport.isInstanceOf[org.apache.spark.sql.kafka010.KafkaContinuousReadSupport] => true
case (defaultCase$ @ _) => false
}
};
new $anonfun()
}: PartialFunction[org.apache.spark.sql.execution.SparkPlan,org.apache.spark.sql.kafka010.KafkaContinuousScanConfig])).exists(((x$1: org.apache.spark.sql.kafka010.KafkaContinuousScanConfig) => x$1.knownPartitions.size.==(newCount))) was false query never reconfigured to 10 partitions
org.scalatest.Assertions$class.newAssertionFailedException(Assertions.scala:528)
org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:501)
org.apache.spark.sql.kafka010.KafkaContinuousTest$$anonfun$setTopicPartitions$1.apply(KafkaContinuousTest.scala:48)
org.apache.spark.sql.kafka010.KafkaContinuousTest$$anonfun$setTopicPartitions$1.apply(KafkaContinuousTest.scala:48)
org.scalatest.concurrent.Eventually$class.makeAValiantAttempt$1(Eventually.scala:395)
org.scalatest.concurrent.Eventually$class.tryTryAgain$1(Eventually.scala:409)
org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:439)
org.apache.spark.sql.kafka010.KafkaSourceTest.eventually(KafkaMicroBatchSourceSuite.scala:49)
org.scalatest.concurrent.Eventually$class.eventually(Eventually.scala:337)
== Progress ==
AssertOnQuery(<condition>, )
AddKafkaData(topics = Set(topic-4), data = WrappedArray(1, 2, 3), message = )
CheckAnswer: [2],[3],[4]
StopStream
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@40901497,Map(),null)
CheckAnswer: [2],[3],[4]
StopStream
AddKafkaData(topics = Set(topic-4), data = WrappedArray(4, 5, 6), message = )
StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1843358a,Map(),null)
CheckAnswer: [2],[3],[4],[5],[6],[7]
AddKafkaData(topics = Set(topic-4), data = WrappedArray(7, 8), message = )
CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
=> AssertOnQuery(<condition>, Add partitions)
AddKafkaData(topics = Set(topic-4), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )
CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]
== Stream ==
Output Mode: Append
Stream state: {KafkaSource[Subscribe[topic-4]]: {"topic-4":{"2":3,"4":2,"1":0,"3":2,"0":2}}}
Thread state: alive
Thread stack trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:206)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:222)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:157)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:243)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:731)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2003)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2024)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2043)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2068)
org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:959)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:364)
org.apache.spark.rdd.RDD.collect(RDD.scala:958)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:53)
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3$$anonfun$apply$1.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3$$anonfun$apply$1.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:147)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:74)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$anonfun$runContinuous$3.apply(ContinuousExecution.scala:265)
org.apache.spark.sql.execution.streaming.ProgressReporter$class.reportTimeTaken(ProgressReporter.scala:323)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:59)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:263)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:90)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:280)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:190)
== Sink ==
0:
1: [4] [3] [2]
2:
3:
4:
5: [6] [7] [5]
6:
7: [9] [8]
8:
== Plan ==
== Parsed Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- SerializeFromObject [input[0, int, false] AS value#13094]
+- MapElements <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#13093: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
+- Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
+- Streaming RelationV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)
== Analyzed Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- SerializeFromObject [input[0, int, false] AS value#13094]
+- MapElements <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#13093: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
+- Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
+- Streaming RelationV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)
== Optimized Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- SerializeFromObject [input[0, int, false] AS value#13094]
+- MapElements <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#13093: int
+- DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
+- Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
+- Streaming RelationV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)
== Physical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWriteSupport@f1a7b95
+- *(1) SerializeFromObject [input[0, int, false] AS value#13094]
+- *(1) MapElements <function1>, obj#13093: int
+- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#13092: scala.Tuple2
+- *(1) Project [cast(key#13257 as string) AS key#13082, cast(value#13258 as string) AS value#13083]
+- *(1) Project [key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263]
+- *(1) ScanV2 kafka[key#13257, value#13258, topic#13259, partition#13260, offset#13261L, timestamp#13262, timestampType#13263] (Options: [kafka.metadata.max.age.ms=1,subscribe=topic-4,failOnDataLoss=true,kafka.bootstrap.servers=127.0....)
at org.scalatest.Assertions$class.newAssertionFailedException(Assertions.scala:528)
at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
at org.scalatest.Assertions$class.fail(Assertions.scala:1089)
at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
at org.apache.spark.sql.streaming.StreamTest$class.failTest$1(StreamTest.scala:453)
at org.apache.spark.sql.streaming.StreamTest$class.executeAction$1(StreamTest.scala:655)
at org.apache.spark.sql.streaming.StreamTest$$anonfun$liftedTree1$1$1.apply(StreamTest.scala:773)
at org.apache.spark.sql.streaming.StreamTest$$anonfun$liftedTree1$1$1.apply(StreamTest.scala:760)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.sql.streaming.StreamTest$class.liftedTree1$1(StreamTest.scala:760)
at org.apache.spark.sql.streaming.StreamTest$class.testStream(StreamTest.scala:759)
at org.apache.spark.sql.kafka010.KafkaSourceTest.testStream(KafkaMicroBatchSourceSuite.scala:49)
at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.org$apache$spark$sql$kafka010$KafkaSourceSuiteBase$$testFromLatestOffsets(KafkaMicroBatchSourceSuite.scala:1339)
at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$anonfun$47$$anonfun$apply$15.apply$mcV$sp(KafkaMicroBatchSourceSuite.scala:1091)
at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$anonfun$47$$anonfun$apply$15.apply(KafkaMicroBatchSourceSuite.scala:1089)
at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$anonfun$47$$anonfun$apply$15.apply(KafkaMicroBatchSourceSuite.scala:1089)
at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:103)
at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:183)
at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:196)
at org.apache.spark.sql.kafka010.KafkaSourceTest.org$scalatest$BeforeAndAfterEach$$super$runTest(KafkaMicroBatchSourceSuite.scala:49)
at org.scalatest.BeforeAndAfterEach$class.runTest(BeforeAndAfterEach.scala:221)
at org.apache.spark.sql.kafka010.KafkaSourceTest.runTest(KafkaMicroBatchSourceSuite.scala:49)
at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:396)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:384)
at scala.collection.immutable.List.foreach(List.scala:392)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:379)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:229)
at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
at org.scalatest.Suite$class.run(Suite.scala:1147)
at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:233)
at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:52)
at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:210)
at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:52)
at org.scalatest.Suite$class.callExecuteOnSuite$1(Suite.scala:1210)
at org.scalatest.Suite$$anonfun$runNestedSuites$1.apply(Suite.scala:1257)
at org.scalatest.Suite$$anonfun$runNestedSuites$1.apply(Suite.scala:1255)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186)
at org.scalatest.Suite$class.runNestedSuites(Suite.scala:1255)
at org.scalatest.tools.DiscoverySuite.runNestedSuites(DiscoverySuite.scala:30)
at org.scalatest.Suite$class.run(Suite.scala:1144)
at org.scalatest.tools.DiscoverySuite.run(DiscoverySuite.scala:30)
at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$1.apply(Runner.scala:1340)
at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$1.apply(Runner.scala:1334)
at scala.collection.immutable.List.foreach(List.scala:392)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1334)
at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1011)
at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1010)
at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1500)
at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1010)
at org.scalatest.tools.Runner$.main(Runner.scala:827)
at org.scalatest.tools.Runner.main(Runner.scala)