Add conditional step retry support

cmd/cmd.go

@@ -161,6 +161,8 @@ func initPackageFlags() {
 	filter.ScenariosName = scenarios
 	execution.MaxRetriesCount = maxRetriesCount
 	execution.RetryOnlyTags = retryOnlyTags
+	execution.MaxStepRetriesCount = maxStepRetriesCount
+	execution.RetryStepOn = retryStepOn
 }
 
 var exit = func(err error, additionalText string) {

cmd/run.go

@@ -43,7 +43,9 @@ const (
 	onlyDefault            = ""
 	groupDefault           = -1
 	maxRetriesCountDefault = 1
+	maxStepRetriesDefault  = 1
 	retryOnlyTagsDefault   = ""
+	retryStepOnDefault     = ""
 	failSafeDefault        = false
 	skipCommandSaveDefault = false
 
@@ -61,7 +63,9 @@ const (
 	strategyName        = "strategy"
 	groupName           = "group"
 	maxRetriesCountName = "max-retries-count"
+	maxStepRetriesName  = "max-step-retries-count"
 	retryOnlyTagsName   = "retry-only"
+	retryStepOnName     = "retry-step-on"
 	streamsName         = "n"
 	onlyName            = "only"
 	failSafeName        = "fail-safe"
@@ -113,7 +117,9 @@ var (
 	strategy                   string
 	streams                    int
 	maxRetriesCount            int
+	maxStepRetriesCount        int
 	retryOnlyTags              string
+	retryStepOn                string
 	group                      int
 	failSafe                   bool
 	skipCommandSave            bool
@@ -132,7 +138,9 @@ func init() {
 	f.BoolVarP(&parallel, parallelName, "p", parallelDefault, "Execute specs in parallel")
 	f.IntVarP(&streams, streamsName, "n", streamsDefault, "Specify number of parallel execution streams")
 	f.IntVarP(&maxRetriesCount, maxRetriesCountName, "c", maxRetriesCountDefault, "Max count of iterations for failed scenario")
+	f.IntVarP(&maxStepRetriesCount, maxStepRetriesName, "", maxStepRetriesDefault, "Max count of iterations for failed step")
 	f.StringVarP(&retryOnlyTags, retryOnlyTagsName, "", retryOnlyTagsDefault, "Retries the specs and scenarios tagged with given tags")
+	f.StringVarP(&retryStepOn, retryStepOnName, "", retryStepOnDefault, "Retries failed step only when error message or stacktrace matches this regex")
 	f.StringVarP(&tagsToFilterForParallelRun, onlyName, "o", onlyDefault, "Execute only the specs and scenarios tagged with given tags in parallel, rest will be run in serial. Applicable only if run in parallel.")
 	err := f.MarkHidden(onlyName)
 	if err != nil {
@@ -294,5 +302,8 @@ func handleConflictingParams(setFlags *pflag.FlagSet, args []string) error {
 	if maxRetriesCount == 1 && retryOnlyTags != "" {
 		return errors.New("Invalid Command. flag --retry-only can be used only with --max-retry-count")
 	}
+	if maxStepRetriesCount == 1 && retryStepOn != "" {
+		return errors.New("Invalid Command. flag --retry-step-on can be used only with --max-step-retries-count")
+	}
 	return nil
 }

cmd/run_test.go

@@ -219,6 +219,45 @@ func TestHandleConflictingParamsWithJustRepeatFlag(t *testing.T) {
 	}
 }
 
+func TestHandleConflictingParamsWithRetryStepOnWithoutStepRetries(t *testing.T) {
+	origRepeat := repeat
+	origFailed := failed
+	origMaxStepRetriesCount := maxStepRetriesCount
+	origRetryStepOn := retryStepOn
+	defer func() {
+		repeat = origRepeat
+		failed = origFailed
+		maxStepRetriesCount = origMaxStepRetriesCount
+		retryStepOn = origRetryStepOn
+	}()
+
+	args := []string{}
+
+	var flags = pflag.FlagSet{}
+	flags.Int(maxStepRetriesName, maxStepRetriesDefault, "")
+	err := flags.Set(maxStepRetriesName, "1")
+	if err != nil {
+		t.Error(err)
+	}
+	flags.String(retryStepOnName, retryStepOnDefault, "")
+	err = flags.Set(retryStepOnName, "TimeoutException")
+	if err != nil {
+		t.Error(err)
+	}
+
+	repeat = false
+	failed = false
+	maxStepRetriesCount = 1
+	retryStepOn = "TimeoutException"
+
+	expectedErrorMessage := "Invalid Command. flag --retry-step-on can be used only with --max-step-retries-count"
+	err = handleConflictingParams(&flags, args)
+
+	if !reflect.DeepEqual(err.Error(), expectedErrorMessage) {
+		t.Errorf("Expected %v  Got %v", expectedErrorMessage, err)
+	}
+}
+
 func TestHandleRerunFlagsWithVerbose(t *testing.T) {
 	if os.Getenv("TEST_EXITS") == "1" {
 		cmd := &cobra.Command{}

execution/execute.go

@@ -39,6 +39,7 @@ import (
 
 	"encoding/json"
 	"path/filepath"
+	"regexp"
 
 	"github.com/getgauge/common"
 	"github.com/getgauge/gauge/config"
@@ -58,11 +59,17 @@ const (
 )
 
 // Count of iterations
-var MaxRetriesCount int
+var MaxRetriesCount = 1
 
 // Tags to filter specs/scenarios to retry
 var RetryOnlyTags string
 
+// Count of iterations for failed steps
+var MaxStepRetriesCount = 1
+
+// Regex used to determine if failed step should be retried.
+var RetryStepOn string
+
 // NumberOfExecutionStreams shows the number of execution streams, in parallel execution.
 var NumberOfExecutionStreams int
 
@@ -212,6 +219,14 @@ func validateFlags() error {
 	if MaxRetriesCount < 1 {
 		return fmt.Errorf("invalid input(%s) to --max-retries-count flag", strconv.Itoa(MaxRetriesCount))
 	}
+	if MaxStepRetriesCount < 1 {
+		return fmt.Errorf("invalid input(%s) to --max-step-retries-count flag", strconv.Itoa(MaxStepRetriesCount))
+	}
+	if RetryStepOn != "" {
+		if _, err := regexp.Compile(RetryStepOn); err != nil {
+			return fmt.Errorf("invalid input(%s) to --retry-step-on flag", RetryStepOn)
+		}
+	}
 	if !InParallel {
 		return nil
 	}

execution/execute_test.go

@@ -69,3 +69,33 @@ func (s *MySuite) TestValidateFlagsWithInvalidStream(c *C) {
 	err := validateFlags()
 	c.Assert(err.Error(), Equals, "invalid input(-1) to --n flag")
 }
+
+func (s *MySuite) TestValidateFlagsWithInvalidMaxStepRetries(c *C) {
+	defer func() {
+		MaxRetriesCount = 1
+		MaxStepRetriesCount = 1
+		RetryStepOn = ""
+		InParallel = false
+	}()
+	InParallel = false
+	MaxRetriesCount = 1
+	MaxStepRetriesCount = 0
+	RetryStepOn = ""
+	err := validateFlags()
+	c.Assert(err.Error(), Equals, "invalid input(0) to --max-step-retries-count flag")
+}
+
+func (s *MySuite) TestValidateFlagsWithInvalidStepRetryCondition(c *C) {
+	defer func() {
+		MaxRetriesCount = 1
+		MaxStepRetriesCount = 1
+		RetryStepOn = ""
+		InParallel = false
+	}()
+	InParallel = false
+	MaxRetriesCount = 1
+	MaxStepRetriesCount = 2
+	RetryStepOn = "["
+	err := validateFlags()
+	c.Assert(err.Error(), Equals, "invalid input([) to --retry-step-on flag")
+}

execution/stepExecutor.go

@@ -7,6 +7,8 @@
 package execution
 
 import (
+	"regexp"
+
 	"github.com/getgauge/gauge-proto/go/gauge_messages"
 	"github.com/getgauge/gauge/execution/event"
 	"github.com/getgauge/gauge/execution/result"
@@ -39,7 +41,7 @@ func (e *stepExecutor) executeStep(step *gauge.Step, protoStep *gauge_messages.P
 	e.notifyBeforeStepHook(stepResult)
 	if !stepResult.GetFailed() {
 		executeStepMessage := &gauge_messages.Message{MessageType: gauge_messages.Message_ExecuteStep, ExecuteStepRequest: stepRequest}
-		stepExecutionStatus := e.runner.ExecuteAndGetStatus(executeStepMessage)
+		stepExecutionStatus := e.executeStepWithRetries(executeStepMessage)
 		stepExecutionStatus.Message = append(stepResult.ProtoStepExecResult().GetExecutionResult().Message, stepExecutionStatus.Message...)
 		if stepExecutionStatus.GetFailed() {
 			e.currentExecutionInfo.CurrentStep.ErrorMessage = stepExecutionStatus.GetErrorMessage()
@@ -59,6 +61,43 @@ func (e *stepExecutor) executeStep(step *gauge.Step, protoStep *gauge_messages.P
 	return stepResult
 }
 
+func (e *stepExecutor) executeStepWithRetries(executeStepMessage *gauge_messages.Message) *gauge_messages.ProtoExecutionResult {
+	var stepExecutionStatus *gauge_messages.ProtoExecutionResult
+	for attempt := 0; attempt < MaxStepRetriesCount; attempt++ {
+		stepExecutionStatus = e.runner.ExecuteAndGetStatus(executeStepMessage)
+		if stepExecutionStatus == nil {
+			stepExecutionStatus = &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "runner returned empty step execution result"}
+		}
+		if !shouldRetryStep(stepExecutionStatus, attempt) {
+			break
+		}
+	}
+	return stepExecutionStatus
+}
+
+func shouldRetryStep(stepExecutionStatus *gauge_messages.ProtoExecutionResult, currentAttempt int) bool {
+	if currentAttempt >= MaxStepRetriesCount-1 || !stepExecutionStatus.GetFailed() || stepExecutionStatus.GetSkipScenario() {
+		return false
+	}
+	if RetryStepOn == "" {
+		return true
+	}
+	stepRetryMatcher, err := regexp.Compile(RetryStepOn)
+	if err != nil {
+		return false
+	}
+	if stepRetryMatcher.MatchString(stepExecutionStatus.GetErrorMessage()) ||
+		stepRetryMatcher.MatchString(stepExecutionStatus.GetStackTrace()) {
+		return true
+	}
+	for _, message := range stepExecutionStatus.GetMessage() {
+		if stepRetryMatcher.MatchString(message) {
+			return true
+		}
+	}
+	return false
+}
+
 func (e *stepExecutor) createStepRequest(protoStep *gauge_messages.ProtoStep) *gauge_messages.ExecuteStepRequest {
 	stepRequest := &gauge_messages.ExecuteStepRequest{ParsedStepText: protoStep.GetParsedText(), ActualStepText: protoStep.GetActualText(), Stream: int32(e.stream)}
 	stepRequest.Parameters = getParameters(protoStep.GetFragments())
@@ -98,5 +137,3 @@ func (e *stepExecutor) notifyAfterStepHook(stepResult *result.StepResult) {
 	m.StepExecutionEndingRequest.StepResult = gauge.ConvertToProtoStepResult(stepResult)
 	e.pluginHandler.NotifyPlugins(m)
 }
-
-

execution/stepExecutor_test.go

@@ -195,3 +195,122 @@ func TestStepExecutionShouldGetScreenshotsAfterStep(t *testing.T) {
 		}
 	}
 }
+
+func TestStepExecutionShouldRetryOnFailureWhenConditionIsNotSet(t *testing.T) {
+	MaxStepRetriesCount = 3
+	RetryStepOn = ""
+	defer resetStepRetryOptions()
+
+	r := &mockRunner{}
+	attempts := 0
+	r.ExecuteAndGetStatusFunc = func(m *gauge_messages.Message) *gauge_messages.ProtoExecutionResult {
+		if m.MessageType == gauge_messages.Message_ExecuteStep {
+			attempts++
+			if attempts < MaxStepRetriesCount {
+				return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "transient failure"}
+			}
+		}
+		return &gauge_messages.ProtoExecutionResult{}
+	}
+
+	se, step, protoStep := newStepExecutorForRetryTests(r)
+	stepResult := se.executeStep(step, protoStep)
+
+	if attempts != MaxStepRetriesCount {
+		t.Errorf("Expected step to execute %d times, got %d", MaxStepRetriesCount, attempts)
+	}
+	if stepResult.GetFailed() {
+		t.Errorf("Expected successful step execution result")
+	}
+}
+
+func TestStepExecutionShouldRetryOnlyWhenConditionMatches(t *testing.T) {
+	MaxStepRetriesCount = 3
+	RetryStepOn = "TimeoutException|429"
+	defer resetStepRetryOptions()
+
+	r := &mockRunner{}
+	attempts := 0
+	r.ExecuteAndGetStatusFunc = func(m *gauge_messages.Message) *gauge_messages.ProtoExecutionResult {
+		if m.MessageType == gauge_messages.Message_ExecuteStep {
+			attempts++
+			if attempts == 1 {
+				return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "HTTP 429 TimeoutException"}
+			}
+		}
+		return &gauge_messages.ProtoExecutionResult{}
+	}
+
+	se, step, protoStep := newStepExecutorForRetryTests(r)
+	stepResult := se.executeStep(step, protoStep)
+
+	if attempts != 2 {
+		t.Errorf("Expected step to execute twice, got %d", attempts)
+	}
+	if stepResult.GetFailed() {
+		t.Errorf("Expected successful step execution result")
+	}
+}
+
+func TestStepExecutionShouldNotRetryWhenConditionDoesNotMatch(t *testing.T) {
+	MaxStepRetriesCount = 3
+	RetryStepOn = "TimeoutException"
+	defer resetStepRetryOptions()
+
+	r := &mockRunner{}
+	attempts := 0
+	r.ExecuteAndGetStatusFunc = func(m *gauge_messages.Message) *gauge_messages.ProtoExecutionResult {
+		if m.MessageType == gauge_messages.Message_ExecuteStep {
+			attempts++
+			return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "AssertionError"}
+		}
+		return &gauge_messages.ProtoExecutionResult{}
+	}
+
+	se, step, protoStep := newStepExecutorForRetryTests(r)
+	stepResult := se.executeStep(step, protoStep)
+
+	if attempts != 1 {
+		t.Errorf("Expected step to execute once, got %d", attempts)
+	}
+	if !stepResult.GetFailed() {
+		t.Errorf("Expected failed step execution result")
+	}
+}
+
+func newStepExecutorForRetryTests(r *mockRunner) (*stepExecutor, *gauge.Step, *gauge_messages.ProtoStep) {
+	h := &mockPluginHandler{NotifyPluginsfunc: func(m *gauge_messages.Message) {}, GracefullyKillPluginsfunc: func() {}}
+	ei := &gauge_messages.ExecutionInfo{
+		CurrentSpec: &gauge_messages.SpecInfo{
+			Name:     "Example Spec",
+			FileName: "example.spec",
+			IsFailed: false,
+		},
+		CurrentScenario: &gauge_messages.ScenarioInfo{
+			Name:     "Example Scenario",
+			IsFailed: false,
+		},
+		CurrentStep: &gauge_messages.StepInfo{
+			Step: &gauge_messages.ExecuteStepRequest{
+				ActualStepText:  "a simple step",
+				ParsedStepText:  "a simple step",
+				ScenarioFailing: false,
+			},
+			IsFailed: false,
+		},
+	}
+	se := &stepExecutor{runner: r, pluginHandler: h, currentExecutionInfo: ei, stream: 0}
+	step := &gauge.Step{
+		Value:     "a simple step",
+		LineText:  "a simple step",
+		Fragments: []*gauge_messages.Fragment{{FragmentType: gauge_messages.Fragment_Text, Text: "a simple step"}},
+	}
+	protoStep := gauge.ConvertToProtoItem(step).GetStep()
+	protoStep.StepExecutionResult = &gauge_messages.ProtoStepExecutionResult{}
+	return se, step, protoStep
+}
+
+func resetStepRetryOptions() {
+	MaxStepRetriesCount = 1
+	RetryStepOn = ""
+}