Dropped Spec column from snapshot

databrickslabs · Jun 11, 2024 · aeef7ff · aeef7ff
1 parent a5c8b54
commit aeef7ff
Showing 1 changed file with 18 additions and 17 deletions.
diff --git a/src/main/scala/com/databricks/labs/overwatch/pipeline/BronzeTransforms.scala b/src/main/scala/com/databricks/labs/overwatch/pipeline/BronzeTransforms.scala
@@ -561,25 +561,26 @@ trait BronzeTransforms extends SparkSessionWrapper {
             .withColumn(s"azure_attributes", SchemaTools.structToMap(scrubbedDF, s"azure_attributes"))
             .withColumn(s"gcp_attributes", SchemaTools.structToMap(scrubbedDF, s"gcp_attributes"))
             .withColumn("organization_id", lit(config.organizationId))
+            .drop("spec")
             .verifyMinimumSchema(clusterSnapMinimumSchema)
 
-          val finalDF = df.schema.fields.find(_.name == "spec") match {
-            case Some(field) =>
-              field.dataType match {
-                case structType: StructType =>
-                  val newFields = structType.fields.map { f =>
-                    f.dataType match {
-                      case _: StructType => SchemaTools.structToMap(df, s"spec.${f.name}").as(f.name)
-                      case _ => col(s"spec.${f.name}").as(f.name)
-                    }
-                  }
-                  df.withColumn("spec", struct(newFields: _*))
-                case _ => df
-              }
-            case None => df
-          }
-
-          finalDF
+//          val finalDF = df.schema.fields.find(_.name == "spec") match {
+//            case Some(field) =>
+//              field.dataType match {
+//                case structType: StructType =>
+//                  val newFields = structType.fields.map { f =>
+//                    f.dataType match {
+//                      case _: StructType => SchemaTools.structToMap(df, s"spec.${f.name}").as(f.name)
+//                      case _ => col(s"spec.${f.name}").as(f.name)
+//                    }
+//                  }
+//                  df.withColumn("spec", struct(newFields: _*))
+//                case _ => df
+//              }
+//            case None => df
+//          }
+
+          df
             .verifyMinimumSchema(clusterSnapMinimumSchema)
 
         } else {