adjust Silver Job Runs module configuration (#1256)

enable auto-optimized shuffle for module 2011 originally implemented in commit [d751d5f](d751d5f)
databrickslabs · Jul 30, 2024 · f7460bd · f7460bd
1 parent 25671b7
commit f7460bd
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/main/scala/com/databricks/labs/overwatch/pipeline/Silver.scala b/src/main/scala/com/databricks/labs/overwatch/pipeline/Silver.scala
@@ -268,7 +268,11 @@ class Silver(_workspace: Workspace, _database: Database, _config: Config)
       )
   }
 
-  lazy private[overwatch] val jobRunsModule = Module(2011, "Silver_JobsRuns", this, Array(1004, 2010, 2014), shuffleFactor = 12.0)
+  lazy private[overwatch] val jobRunsModule =
+    Module( 2011, "Silver_JobsRuns", this, Array(1004, 2010, 2014), shuffleFactor = 12.0)
+      .withSparkOverrides( Map(
+        "spark.databricks.adaptive.autoOptimizeShuffle.enabled" -> "true"))
+
   lazy private val appendJobRunsProcess: () => ETLDefinition = {
     () =>
       ETLDefinition(