From 615e7863eeb10157776439d0b07ecdac3556ec44 Mon Sep 17 00:00:00 2001 From: Sriram Mohanty <69749553+sriram251-code@users.noreply.github.com> Date: Fri, 5 Jan 2024 10:34:10 +0530 Subject: [PATCH] code missed in merge (#1120) --- .../scala/com/databricks/labs/overwatch/env/Workspace.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/databricks/labs/overwatch/env/Workspace.scala b/src/main/scala/com/databricks/labs/overwatch/env/Workspace.scala index 2c66badda..87824fbe1 100644 --- a/src/main/scala/com/databricks/labs/overwatch/env/Workspace.scala +++ b/src/main/scala/com/databricks/labs/overwatch/env/Workspace.scala @@ -308,7 +308,8 @@ class Workspace(config: Config) extends SparkSessionWrapper { if(Helpers.pathExists(tempWorkingDir)) { try { spark.conf.set("spark.sql.caseSensitive", "true") - val df = spark.read.json(tempWorkingDir) + val baseDF = spark.read.json(tempWorkingDir) + val df = deriveRawApiResponseDF(baseDF) .select(explode(col("runs")).alias("runs")).select(col("runs" + ".*")) .withColumn("organization_id", lit(config.organizationId)) spark.conf.set("spark.sql.caseSensitive", "false")