fix(probe): Resiliency Score reaches more than 100 % with Probe failu…

…re (litmuschaos#568) * chore(probe): remove ambiguous attribute phase Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io> * chore(probe): handle edge mode when probe failed in prechaos phase but passed in postchaos phase with stopOnfailure set to false Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io> * chore(probe): fixed the probeSuccessPercentage > 100 issue Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io> * add the failstep for probe failures Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io> * fixing onchaos probe to run only for chaos duration Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io> Signed-off-by: Saptarshi Sarkar <saptarshi.programmer@gmail.com>
SaptarshiSarkar12 · Sep 21, 2022 · e142937 · e142937
1 parent 3e7ea39
commit e142937
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 28 deletions.
diff --git a/pkg/probe/cmdprobe.go b/pkg/probe/cmdprobe.go
@@ -342,14 +342,12 @@ func triggerSourceOnChaosCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDet
 		duration = math.Maximum(0, duration-probe.RunProperties.InitialDelaySeconds)
 	}
 
-	var endTime <-chan time.Time
-	timeDelay := time.Duration(duration) * time.Second
+	endTime := time.After(time.Duration(duration) * time.Second)
 
 	// it trigger the cmd probe for the entire duration of chaos and it fails, if any err encounter
 	// it marked the error for the probes, if any
 loop:
 	for {
-		endTime = time.After(timeDelay)
 		select {
 		case <-endTime:
 			log.Infof("[Chaos]: Time is up for the %v probe", probe.Name)

diff --git a/pkg/probe/httpprobe.go b/pkg/probe/httpprobe.go
@@ -330,14 +330,12 @@ func triggerOnChaosHTTPProbe(probe v1alpha1.ProbeAttributes, clients clients.Cli
 		duration = math.Maximum(0, duration-probe.RunProperties.InitialDelaySeconds)
 	}
 
-	var endTime <-chan time.Time
-	timeDelay := time.Duration(duration) * time.Second
+	endTime := time.After(time.Duration(duration) * time.Second)
 
 	// it trigger the http probe for the entire duration of chaos and it fails, if any error encounter
 	// it marked the error for the probes, if any
 loop:
 	for {
-		endTime = time.After(timeDelay)
 		select {
 		case <-endTime:
 			log.Infof("[Chaos]: Time is up for the %v probe", probe.Name)

diff --git a/pkg/probe/k8sprobe.go b/pkg/probe/k8sprobe.go
@@ -295,14 +295,12 @@ func triggerOnChaosK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.Clie
 		duration = math.Maximum(0, duration-probe.RunProperties.InitialDelaySeconds)
 	}
 
-	var endTime <-chan time.Time
-	timeDelay := time.Duration(duration) * time.Second
+	endTime := time.After(time.Duration(duration) * time.Second)
 
-	// it trigger the k8s probe for the entire duration of chaos and it fails, if any error encounter
+	// it triggers the k8s probe for the entire duration of chaos and it fails, if any error encounter
 	// marked the error for the probes, if any
 loop:
 	for {
-		endTime = time.After(timeDelay)
 		select {
 		case <-endTime:
 			log.Infof("[Chaos]: Time is up for the %v probe", probe.Name)

diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go
@@ -32,11 +32,23 @@ func RunProbes(chaosDetails *types.ChaosDetails, clients clients.ClientSets, res
 	}
 
 	switch strings.ToLower(phase) {
-	//execute probes for the prechaos & duringchaos phase
-	case "prechaos", "duringchaos":
+	//execute probes for the prechaos phase
+	case "prechaos":
 		for _, probe := range probes {
-			if err := execute(probe, chaosDetails, clients, resultDetails, phase); err != nil {
-				return err
+			switch strings.ToLower(probe.Mode) {
+			case "sot", "edge", "continuous":
+				if err := execute(probe, chaosDetails, clients, resultDetails, phase); err != nil {
+					return err
+				}
+			}
+		}
+	//execute probes for the duringchaos phase
+	case "duringchaos":
+		for _, probe := range probes {
+			if strings.ToLower(probe.Mode) == "onchaos" {
+				if err := execute(probe, chaosDetails, clients, resultDetails, phase); err != nil {
+					return err
+				}
 			}
 		}
 	default:
@@ -58,8 +70,11 @@ func RunProbes(chaosDetails *types.ChaosDetails, clients clients.ClientSets, res
 		}
 		// executes the eot and edge modes
 		for _, probe := range probes {
-			if err := execute(probe, chaosDetails, clients, resultDetails, phase); err != nil {
-				return err
+			switch strings.ToLower(probe.Mode) {
+			case "eot", "edge":
+				if err := execute(probe, chaosDetails, clients, resultDetails, phase); err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -137,7 +152,6 @@ func InitializeProbesInChaosResultDetails(chaosDetails *types.ChaosDetails, clie
 		tempProbe.Name = probe.Name
 		tempProbe.Type = probe.Type
 		tempProbe.Mode = probe.Mode
-		tempProbe.Phase = "N/A"
 		tempProbe.RunCount = 0
 		tempProbe.Status = v1alpha1.ProbeStatus{
 			Verdict: "Awaited",
@@ -204,12 +218,8 @@ func markedVerdictInEnd(err error, resultDetails *types.ResultDetails, probe v1a
 		// counting the passed probes count to generate the score and mark the verdict as passed
 		// for edge, probe is marked as Passed if passed in both pre/post chaos checks
 		switch strings.ToLower(probe.Mode) {
-		case "edge", "continuous":
-			if phase != "PreChaos" {
-				resultDetails.PassedProbeCount++
-			}
-		case "onchaos":
-			if phase != "DuringChaos" {
+		case "edge":
+			if phase == "PostChaos" && getProbeVerdict(resultDetails, probe.Name, probe.Type) != v1alpha1.ProbeVerdictFailed {
 				resultDetails.PassedProbeCount++
 			}
 		default:
@@ -313,3 +323,12 @@ func execute(probe v1alpha1.ProbeAttributes, chaosDetails *types.ChaosDetails, c
 	}
 	return nil
 }
+
+func getProbeVerdict(resultDetails *types.ResultDetails, name, probeType string) v1alpha1.ProbeVerdict {
+	for _, probe := range resultDetails.ProbeDetails {
+		if probe.Name == name && probe.Type == probeType {
+			return probe.Status.Verdict
+		}
+	}
+	return v1alpha1.ProbeVerdictNA
+}
diff --git a/pkg/probe/promProbe.go b/pkg/probe/promProbe.go
@@ -264,14 +264,12 @@ func triggerOnChaosPromProbe(probe v1alpha1.ProbeAttributes, clients clients.Cli
 		duration = math.Maximum(0, duration-probe.RunProperties.InitialDelaySeconds)
 	}
 
-	var endTime <-chan time.Time
-	timeDelay := time.Duration(duration) * time.Second
+	endTime := time.After(time.Duration(duration) * time.Second)
 
 	// it trigger the prom probe for the entire duration of chaos and it fails, if any err encounter
 	// it marked the error for the probes, if any
 loop:
 	for {
-		endTime = time.After(timeDelay)
 		select {
 		case <-endTime:
 			log.Infof("[Chaos]: Time is up for the %v probe", probe.Name)

diff --git a/pkg/result/chaosresult.go b/pkg/result/chaosresult.go
@@ -136,7 +136,7 @@ func GetProbeStatus(resultDetails *types.ResultDetails) (bool, []v1alpha1.ProbeS
 		probes.Mode = probe.Mode
 		probes.Status = probe.Status
 		probeStatus = append(probeStatus, probes)
-		if probe.Phase == "Failed" {
+		if probe.Status.Verdict == v1alpha1.ProbeVerdictFailed {
 			isAllProbePassed = false
 		}
 	}
@@ -165,6 +165,7 @@ func updateResultAttributes(clients clients.ClientSets, chaosDetails *types.Chao
 		if !isAllProbePassed {
 			resultDetails.Verdict = "Fail"
 			result.Status.ExperimentStatus.Verdict = "Fail"
+			result.Status.ExperimentStatus.FailStep = "Probe execution result didn't met the passing criteria"
 		}
 		switch strings.ToLower(string(resultDetails.Verdict)) {
 		case "pass":

diff --git a/pkg/types/types.go b/pkg/types/types.go
@@ -55,7 +55,6 @@ type ProbeDetails struct {
 	Name                   string
 	Type                   string
 	Mode                   string
-	Phase                  string
 	Status                 v1alpha1.ProbeStatus
 	IsProbeFailedWithError error
 	RunID                  string