[CI] In-progress Slack notifications (elastic#74012)

# Conflicts:
#	Jenkinsfile

.ci/pipeline-library/src/test/slackNotifications.groovy

@@ -9,6 +9,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
     super.setUp()
 
     helper.registerAllowedMethod('slackSend', [Map.class], null)
+    prop('buildState', loadScript("vars/buildState.groovy"))
     slackNotifications = loadScript('vars/slackNotifications.groovy')
   }
 
@@ -25,13 +26,49 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
   }
 
   @Test
-  void 'sendFailedBuild() should call slackSend() with message'() {
+  void 'sendFailedBuild() should call slackSend() with an in-progress message'() {
     mockFailureBuild()
 
     slackNotifications.sendFailedBuild()
 
     def args = fnMock('slackSend').args[0]
 
+    def expected = [
+      channel: '#kibana-operations-alerts',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':hourglass_flowing_sand: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+
+    assertEquals(
+      ":hourglass_flowing_sand: *<http://jenkins.localhost:8080/job/elastic+kibana+master/1/|elastic / kibana # master #1>*",
+      args.blocks[0].text.text.toString()
+    )
+
+    assertEquals(
+      "*Failed Steps*\n• <http://jenkins.localhost:8080|Execute test task>",
+      args.blocks[1].text.text.toString()
+    )
+
+    assertEquals(
+      "*Test Failures*\n• <https://localhost/|x-pack/test/functional/apps/fake/test·ts.Fake test &lt;Component&gt; should &amp; pass &amp;>",
+      args.blocks[2].text.text.toString()
+    )
+  }
+
+  @Test
+  void 'sendFailedBuild() should call slackSend() with message'() {
+    mockFailureBuild()
+
+    slackNotifications.sendFailedBuild(isFinal: true)
+
+    def args = fnMock('slackSend').args[0]
+
     def expected = [
       channel: '#kibana-operations-alerts',
       username: 'Kibana Operations',
@@ -65,7 +102,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
     mockFailureBuild()
     def counter = 0
     helper.registerAllowedMethod('slackSend', [Map.class], { ++counter > 1 })
-    slackNotifications.sendFailedBuild()
+    slackNotifications.sendFailedBuild(isFinal: true)
 
     def args = fnMocks('slackSend')[1].args[0]
 
@@ -88,6 +125,29 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
     )
   }
 
+  @Test
+  void 'sendFailedBuild() should call slackSend() with a channel id and timestamp on second call'() {
+    mockFailureBuild()
+    helper.registerAllowedMethod('slackSend', [Map.class], { [ channelId: 'CHANNEL_ID', ts: 'TIMESTAMP' ] })
+    slackNotifications.sendFailedBuild(isFinal: false)
+    slackNotifications.sendFailedBuild(isFinal: true)
+
+    def args = fnMocks('slackSend')[1].args[0]
+
+    def expected = [
+      channel: 'CHANNEL_ID',
+      timestamp: 'TIMESTAMP',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':broken_heart: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+  }
+
   @Test
   void 'getTestFailures() should truncate list of failures to 10'() {
     prop('testUtils', [

Jenkinsfile

@@ -4,42 +4,43 @@ library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
 kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true) {
-  githubPr.withDefaultPrComments {
-    ciStats.trackBuild {
-      catchError {
-        retryable.enable()
-        parallel([
-          'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-          'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-          'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-            'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-            'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-            'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-            'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-            'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-            'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-            'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-            'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-            'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-            'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-            'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-            'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-          ]),
-          'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-            'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-            'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-            'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-            'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-            'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-            'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-          ]),
-        ])
+  slackNotifications.onFailure(disabled: !params.NOTIFY_ON_FAILURE) {
+    githubPr.withDefaultPrComments {
+      ciStats.trackBuild {
+        catchError {
+          retryable.enable()
+          parallel([
+            'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
+            'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
+            'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+              'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
+              'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
+              'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
+              'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
+              'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
+              'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
+              'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
+              'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
+              'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
+              'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
+              'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
+              'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
+            ]),
+            'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
+              'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
+              'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
+              'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
+              'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
+              'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
+              'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
+            ]),
+          ])
+        }
       }
     }
   }
 
   if (params.NOTIFY_ON_FAILURE) {
-    slackNotifications.onFailure()
     kibanaPipeline.sendMail()
   }
 }

vars/githubPr.groovy

@@ -15,7 +15,7 @@
 */
 def withDefaultPrComments(closure) {
   catchErrors {
-    // sendCommentOnError() needs to know if comments are enabled, so lets track it with a global
+    // kibanaPipeline.notifyOnError() needs to know if comments are enabled, so lets track it with a global
     // isPr() just ensures this functionality is skipped for non-PR builds
     buildState.set('PR_COMMENTS_ENABLED', isPr())
     catchErrors {
@@ -59,19 +59,6 @@ def sendComment(isFinal = false) {
   }
 }
 
-def sendCommentOnError(Closure closure) {
-  try {
-    closure()
-  } catch (ex) {
-    // If this is the first failed step, it's likely that the error hasn't propagated up far enough to mark the build as a failure
-    currentBuild.result = 'FAILURE'
-    catchErrors {
-      sendComment(false)
-    }
-    throw ex
-  }
-}
-
 // Checks whether or not this currently executing build was triggered via a PR in the elastic/kibana repo
 def isPr() {
   return !!(env.ghprbPullId && env.ghprbPullLink && env.ghprbPullLink =~ /\/elastic\/kibana\//)

vars/kibanaPipeline.groovy

@@ -17,6 +17,25 @@ def withPostBuildReporting(Closure closure) {
   }
 }
 
+def notifyOnError(Closure closure) {
+  try {
+    closure()
+  } catch (ex) {
+    // If this is the first failed step, it's likely that the error hasn't propagated up far enough to mark the build as a failure
+    currentBuild.result = 'FAILURE'
+    catchErrors {
+      githubPr.sendComment(false)
+    }
+    catchErrors {
+      // an empty map is a valid config, but is falsey, so let's use .has()
+      if (buildState.has('SLACK_NOTIFICATION_CONFIG')) {
+        slackNotifications.sendFailedBuild(buildState.get('SLACK_NOTIFICATION_CONFIG'))
+      }
+    }
+    throw ex
+  }
+}
+
 def functionalTestProcess(String name, Closure closure) {
   return { processNumber ->
     def kibanaPort = "61${processNumber}1"
@@ -37,7 +56,7 @@ def functionalTestProcess(String name, Closure closure) {
       "JOB=${name}",
       "KBN_NP_PLUGINS_BUILT=true",
     ]) {
-      githubPr.sendCommentOnError {
+      notifyOnError {
         closure()
       }
     }
@@ -184,7 +203,7 @@ def bash(script, label) {
 }
 
 def doSetup() {
-  githubPr.sendCommentOnError {
+  notifyOnError {
     retryWithDelay(2, 15) {
       try {
         runbld("./test/scripts/jenkins_setup.sh", "Setup Build Environment and Dependencies")
@@ -201,13 +220,13 @@ def doSetup() {
 }
 
 def buildOss() {
-  githubPr.sendCommentOnError {
+  notifyOnError {
     runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana")
   }
 }
 
 def buildXpack() {
-  githubPr.sendCommentOnError {
+  notifyOnError {
     runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana")
   }
 }

vars/slackNotifications.groovy

@@ -105,16 +105,26 @@ def getDefaultDisplayName() {
   return "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
 }
 
-def getDefaultContext() {
-  def duration = currentBuild.durationString.replace(' and counting', '')
+def getDefaultContext(config = [:]) {
+  def progressMessage = ""
+  if (config && !config.isFinal) {
+    progressMessage = "In-progress"
+  } else {
+    def duration = currentBuild.durationString.replace(' and counting', '')
+    progressMessage = "${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}"
+  }
 
   return contextBlock([
-    "${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}",
+    progressMessage,
     "<https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>",
   ].join(' · '))
 }
 
-def getStatusIcon() {
+def getStatusIcon(config = [:]) {
+  if (config && !config.isFinal) {
+    return ':hourglass_flowing_sand:'
+  }
+
   def status = buildUtils.getBuildStatus()
   if (status == 'UNSTABLE') {
     return ':yellow_heart:'
@@ -124,7 +134,7 @@ def getStatusIcon() {
 }
 
 def getBackupMessage(config) {
-  return "${getStatusIcon()} ${config.title}\n\nFirst attempt at sending this notification failed. Please check the build."
+  return "${getStatusIcon(config)} ${config.title}\n\nFirst attempt at sending this notification failed. Please check the build."
 }
 
 def sendFailedBuild(Map params = [:]) {
@@ -135,19 +145,32 @@ def sendFailedBuild(Map params = [:]) {
     color: 'danger',
     icon: ':jenkins:',
     username: 'Kibana Operations',
-    context: getDefaultContext(),
+    isFinal: false,
   ] + params
 
-  def title = "${getStatusIcon()} ${config.title}"
-  def message = "${getStatusIcon()} ${config.message}"
+  config.context = config.context ?: getDefaultContext(config)
+
+  def title = "${getStatusIcon(config)} ${config.title}"
+  def message = "${getStatusIcon(config)} ${config.message}"
 
   def blocks = [markdownBlock(title)]
   getFailedBuildBlocks().each { blocks << it }
   blocks << dividerBlock()
   blocks << config.context
 
+  def channel = config.channel
+  def timestamp = null
+
+  def previousResp = buildState.get('SLACK_NOTIFICATION_RESPONSE')
+  if (previousResp) {
+    // When using `timestamp` to update a previous message, you have to use the channel ID from the previous response
+    channel = previousResp.channelId
+    timestamp = previousResp.ts
+  }
+
   def resp = slackSend(
-    channel: config.channel,
+    channel: channel,
+    timestamp: timestamp,
     username: config.username,
     iconEmoji: config.icon,
     color: config.color,
@@ -156,7 +179,7 @@ def sendFailedBuild(Map params = [:]) {
   )
 
   if (!resp) {
-    slackSend(
+    resp = slackSend(
       channel: config.channel,
       username: config.username,
       iconEmoji: config.icon,
@@ -165,20 +188,35 @@ def sendFailedBuild(Map params = [:]) {
       blocks: [markdownBlock(getBackupMessage(config))]
     )
   }
+
+  if (resp) {
+    buildState.set('SLACK_NOTIFICATION_RESPONSE', resp)
+  }
 }
 
 def onFailure(Map options = [:]) {
   catchError {
     def status = buildUtils.getBuildStatus()
     if (status != "SUCCESS") {
       catchErrors {
+        options.isFinal = true
         sendFailedBuild(options)
       }
     }
   }
 }
 
 def onFailure(Map options = [:], Closure closure) {
+  if (options.disabled) {
+    catchError {
+      closure()
+    }
+
+    return
+  }
+
+  buildState.set('SLACK_NOTIFICATION_CONFIG', options)
+
   // try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes
   catchError {
     closure()

vars/workers.groovy

@@ -126,7 +126,7 @@ def intake(jobName, String script) {
   return {
     ci(name: jobName, size: 's-highmem', ramDisk: true) {
       withEnv(["JOB=${jobName}"]) {
-        githubPr.sendCommentOnError {
+        kibanaPipeline.notifyOnError {
           runbld(script, "Execute ${jobName}")
         }
       }