Hi Team, I have been working upon Migrating Synapse Spark 3.1 to Synapse Spark 3.2.1, but I am facing weird issue with column name. The following snippet of code gives error with spark 3.2.1
So, it is failing with issue not able to find EventTime column, although it is present in DataFrame. The same code works exactly fine with Spark 3.1.2.
Here is the exact error stack trace, if you carefully observe then this column is present in resolved attribute.
org.apache.spark.sql.AnalysisException: Resolved attribute(s) EventTime#21 missing from
TenantId#192,DeviceId#193,DeviceEnrolledTimeStamp#194,DeviceRemovedTimeStamp#77,DeviceHealthStatus#78,DeviceHealthStatusLastUpdatedTS#79,DeviceUsageStatus#80,DeviceUsageStatusLastUpdatedTS#81,DeviceType#82,DeviceTypeLastUpdatedTS#83,DeviceUserType#84,DeviceUserTypeLastUpdatedTS#85,DeviceManufacture#86,DeviceManufactureLastUpdatedTS#87,DeviceModel#88,DeviceModelLastUpdatedTS#89,DeviceTags#90,DeviceTagsLastUpdatedTS#91,TenantId#240,DeviceId#241,EntityType#242,EventType#243,EventValue#244,EventTime#245,Date#34,TimeInEpochMilliSeconds#45L in operator !Project [TenantId#192, DeviceId#193, DeviceEnrolledTimeStamp#194, CASE WHEN ((isnull(DeviceRemovedTimeStamp#77) AND isnotnull(EventTime#21)) AND (UDF(EventTime#21) >= UDF(DeviceEnrolledTimeStamp#194))) THEN EventTime#21 ELSE DeviceRemovedTimeStamp#77 END AS DeviceRemovedTimeStamp#304, DeviceHealthStatus#78, DeviceHealthStatusLastUpdatedTS#79, DeviceUsageStatus#80, DeviceUsageStatusLastUpdatedTS#81, DeviceType#82, DeviceTypeLastUpdatedTS#83, DeviceUserType#84, DeviceUserTypeLastUpdatedTS#85, DeviceManufacture#86, DeviceManufactureLastUpdatedTS#87, DeviceModel#88, DeviceModelLastUpdatedTS#89, DeviceTags#90, DeviceTagsLastUpdatedTS#91]. Attribute(s) with the same name appear in the operation: EventTime. Please check if the right attribute(s) are used.;
!Project [TenantId#192, DeviceId#193, DeviceEnrolledTimeStamp#194, CASE WHEN ((isnull(DeviceRemovedTimeStamp#77) AND isnotnull(EventTime#21)) AND (UDF(EventTime#21) >= UDF(DeviceEnrolledTimeStamp#194))) THEN EventTime#21 ELSE DeviceRemovedTimeStamp#77 END AS DeviceRemovedTimeStamp#304, DeviceHealthStatus#78, DeviceHealthStatusLastUpdatedTS#79, DeviceUsageStatus#80, DeviceUsageStatusLastUpdatedTS#81, DeviceType#82, DeviceTypeLastUpdatedTS#83, DeviceUserType#84, DeviceUserTypeLastUpdatedTS#85, DeviceManufacture#86, DeviceManufactureLastUpdatedTS#87, DeviceModel#88, DeviceModelLastUpdatedTS#89, DeviceTags#90, DeviceTagsLastUpdatedTS#91]
+- Join LeftOuter, ((TenantId#192 = TenantId#240) AND (DeviceId#193 = DeviceId#241))
:- Project [coalesce(TenantId#74, TenantId#16) AS TenantId#192, coalesce(DeviceId#75, DeviceId#17) AS DeviceId#193, CASE WHEN isnotnull(DeviceEnrolledTimeStamp#76) THEN DeviceEnrolledTimeStamp#76 ELSE EventTime#21 END AS DeviceEnrolledTimeStamp#194, DeviceRemovedTimeStamp#77, DeviceHealthStatus#78, DeviceHealthStatusLastUpdatedTS#79, DeviceUsageStatus#80, DeviceUsageStatusLastUpdatedTS#81, DeviceType#82, DeviceTypeLastUpdatedTS#83, DeviceUserType#84, DeviceUserTypeLastUpdatedTS#85, DeviceManufacture#86, DeviceManufactureLastUpdatedTS#87, DeviceModel#88, DeviceModelLastUpdatedTS#89, DeviceTags#90, DeviceTagsLastUpdatedTS#91]
: +- Join FullOuter, ((TenantId#74 = TenantId#16) AND (DeviceId#75 = DeviceId#17))
: :- Relation [Date#73,TenantId#74,DeviceId#75,DeviceEnrolledTimeStamp#76,DeviceRemovedTimeStamp#77,DeviceHealthStatus#78,DeviceHealthStatusLastUpdatedTS#79,DeviceUsageStatus#80,DeviceUsageStatusLastUpdatedTS#81,DeviceType#82,DeviceTypeLastUpdatedTS#83,DeviceUserType#84,DeviceUserTypeLastUpdatedTS#85,DeviceManufacture#86,DeviceManufactureLastUpdatedTS#87,DeviceModel#88,DeviceModelLastUpdatedTS#89,DeviceTags#90,DeviceTagsLastUpdatedTS#91] csv
: +- Project [TenantId#16, DeviceId#17, EntityType#18, EventType#19, EventValue#20, EventTime#21, Date#34, TimeInEpochMilliSeconds#45L]
: +- Filter (row#120 = 1)
: +- Project [TenantId#16, DeviceId#17, EntityType#18, EventType#19, EventValue#20, EventTime#21, Date#34, TimeInEpochMilliSeconds#45L, row#120]
: +- Project [TenantId#16, DeviceId#17, EntityType#18, EventType#19, EventValue#20, EventTime#21, Date#34, TimeInEpochMilliSeconds#45L, row#120, row#120]
: +- Window [row_number() windowspecdefinition(Date#34, TenantId#16, DeviceId#17, TimeInEpochMilliSeconds#45L ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS row#120], [Date#34, TenantId#16, DeviceId#17], [TimeInEpochMilliSeconds#45L ASC NULLS FIRST]
: +- Project [TenantId#16, DeviceId#17, EntityType#18, EventType#19, EventValue#20, EventTime#21, Date#34, TimeInEpochMilliSeconds#45L]
: +- Filter ((((lower(EventType#19) = devicelifecycle) AND (lower(EventValue#20) = deviceenrolled)) AND (TimeInEpochMilliSeconds#45L >= 1633219200000)) AND (TimeInEpochMilliSeconds#45L < 1633305600000))
: +- Project [TenantId#16, DeviceId#17, EntityType#18, EventType#19, EventValue#20, EventTime#21, Date#34, TimeInEpochMilliSeconds#45L]
: +- Project [TenantId#16, DeviceId#17, EntityType#18, EventType#19, EventValue#20, EventTime#21, Date#34, UDF(EventTime#21) AS TimeInEpochMilliSeconds#45L]
: +- Project [TenantId#16, DeviceId#17, EntityType#18, EventType#19, EventValue#20, EventTime#21, UDF(EventTime#21) AS Date#34]
: +- Relation [TenantId#16,DeviceId#17,EntityType#18,EventType#19,EventValue#20,EventTime#21] csv
+- Project [TenantId#240, DeviceId#241, EntityType#242, EventType#243, EventValue#244, EventTime#245, Date#34, TimeInEpochMilliSeconds#45L]
+- Filter (row#222 = 1)
+- Project [TenantId#240, DeviceId#241, EntityType#242, EventType#243, EventValue#244, EventTime#245, Date#34, TimeInEpochMilliSeconds#45L, row#222]
+- Project [TenantId#240, DeviceId#241, EntityType#242, EventType#243, EventValue#244, EventTime#245, Date#34, TimeInEpochMilliSeconds#45L, row#222, row#222]
+- Window [row_number() windowspecdefinition(Date#34, TenantId#240, DeviceId#241, TimeInEpochMilliSeconds#45L ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS row#222], [Date#34, TenantId#240, DeviceId#241], [TimeInEpochMilliSeconds#45L ASC NULLS FIRST]
+- Project [TenantId#240, DeviceId#241, EntityType#242, EventType#243, EventValue#244, EventTime#245, Date#34, TimeInEpochMilliSeconds#45L]
+- Filter ((((lower(EventType#243) = devicelifecycle) AND (lower(EventValue#244) = deviceremoved)) AND (TimeInEpochMilliSeconds#45L >= 1633219200000)) AND (TimeInEpochMilliSeconds#45L < 1633305600000))
+- Project [TenantId#240, DeviceId#241, EntityType#242, EventType#243, EventValue#244, EventTime#245, Date#34, TimeInEpochMilliSeconds#45L]
+- Project [TenantId#240, DeviceId#241, EntityType#242, EventType#243, EventValue#244, EventTime#245, Date#34, UDF(EventTime#245) AS TimeInEpochMilliSeconds#45L]
+- Project [TenantId#240, DeviceId#241, EntityType#242, EventType#243, EventValue#244, EventTime#245, UDF(EventTime#245) AS Date#34]
+- Relation [TenantId#240,DeviceId#241,EntityType#242,EventType#243,EventValue#244,EventTime#245] csv
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis(CheckAnalysis.scala:51)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis$(CheckAnalysis.scala:50)
at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:182)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$1(CheckAnalysis.scala:471)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$1$adapted(CheckAnalysis.scala:94)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:263)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.checkAnalysis(CheckAnalysis.scala:94)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.checkAnalysis$(CheckAnalysis.scala:91)
at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:182)
at org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$executeAndCheck$1(Analyzer.scala:205)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.markInAnalyzer(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.analysis.Analyzer.executeAndCheck(Analyzer.scala:202)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$analyzed$1(QueryExecution.scala:88)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:196)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:196)
at org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:88)
at org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:86)
at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:78)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$1(Dataset.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:88)
at org.apache.spark.sql.Dataset.withPlan(Dataset.scala:3734)
at org.apache.spark.sql.Dataset.select(Dataset.scala:1454)
at com.microsoft.teams.deviceanalytics.core.processor.DeviceMetadataProcessor.applyDeviceRemovalChangesOnMasterTable(DeviceMetadataProcessor.scala:239)
at com.microsoft.teams.deviceanalytics.core.test.DeviceMetaDataProcessorTest.$anonfun$new$2(DeviceMetaDataProcessorTest.scala:65)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
at org.scalatest.TestSuite.withFixture(TestSuite.scala:196)
at org.scalatest.TestSuite.withFixture$(TestSuite.scala:195)
at org.scalatest.FunSuite.withFixture(FunSuite.scala:1560)
at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
at org.scalatest.FunSuite.runTest(FunSuite.scala:1560)
at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396)
at scala.collection.immutable.List.foreach(List.scala:431)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
at org.scalatest.Suite.run(Suite.scala:1147)
at org.scalatest.Suite.run$(Suite.scala:1129)
at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
at com.microsoft.teams.deviceanalytics.core.test.DeviceMetaDataProcessorTest.org$scalatest$BeforeAndAfterAll$$super$run(DeviceMetaDataProcessorTest.scala:23)
at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at com.microsoft.teams.deviceanalytics.core.test.DeviceMetaDataProcessorTest.run(DeviceMetaDataProcessorTest.scala:23)
at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1346)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1340)
at scala.collection.immutable.List.foreach(List.scala:431)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1340)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:1031)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:1010)
at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1506)
at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1010)
at org.scalatest.tools.Runner$.run(Runner.scala:850)
at org.scalatest.tools.Runner.run(Runner.scala)
