diff --git a/cmd/cmd.go b/cmd/cmd.go index 4c3357970..8ea99a17d 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -161,6 +161,8 @@ func initPackageFlags() { filter.ScenariosName = scenarios execution.MaxRetriesCount = maxRetriesCount execution.RetryOnlyTags = retryOnlyTags + execution.MaxStepRetriesCount = maxStepRetriesCount + execution.RetryStepOn = retryStepOn } var exit = func(err error, additionalText string) { diff --git a/cmd/run.go b/cmd/run.go index 668e9a924..9533787cb 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -43,7 +43,9 @@ const ( onlyDefault = "" groupDefault = -1 maxRetriesCountDefault = 1 + maxStepRetriesDefault = 1 retryOnlyTagsDefault = "" + retryStepOnDefault = "" failSafeDefault = false skipCommandSaveDefault = false @@ -61,7 +63,9 @@ const ( strategyName = "strategy" groupName = "group" maxRetriesCountName = "max-retries-count" + maxStepRetriesName = "max-step-retries-count" retryOnlyTagsName = "retry-only" + retryStepOnName = "retry-step-on" streamsName = "n" onlyName = "only" failSafeName = "fail-safe" @@ -113,7 +117,9 @@ var ( strategy string streams int maxRetriesCount int + maxStepRetriesCount int retryOnlyTags string + retryStepOn string group int failSafe bool skipCommandSave bool @@ -132,7 +138,9 @@ func init() { f.BoolVarP(¶llel, parallelName, "p", parallelDefault, "Execute specs in parallel") f.IntVarP(&streams, streamsName, "n", streamsDefault, "Specify number of parallel execution streams") f.IntVarP(&maxRetriesCount, maxRetriesCountName, "c", maxRetriesCountDefault, "Max count of iterations for failed scenario") + f.IntVarP(&maxStepRetriesCount, maxStepRetriesName, "", maxStepRetriesDefault, "Max count of iterations for failed step") f.StringVarP(&retryOnlyTags, retryOnlyTagsName, "", retryOnlyTagsDefault, "Retries the specs and scenarios tagged with given tags") + f.StringVarP(&retryStepOn, retryStepOnName, "", retryStepOnDefault, "Retries failed step only when error message or stacktrace matches this regex") f.StringVarP(&tagsToFilterForParallelRun, onlyName, "o", onlyDefault, "Execute only the specs and scenarios tagged with given tags in parallel, rest will be run in serial. Applicable only if run in parallel.") err := f.MarkHidden(onlyName) if err != nil { @@ -294,5 +302,8 @@ func handleConflictingParams(setFlags *pflag.FlagSet, args []string) error { if maxRetriesCount == 1 && retryOnlyTags != "" { return errors.New("Invalid Command. flag --retry-only can be used only with --max-retry-count") } + if maxStepRetriesCount == 1 && retryStepOn != "" { + return errors.New("Invalid Command. flag --retry-step-on can be used only with --max-step-retries-count") + } return nil } diff --git a/cmd/run_test.go b/cmd/run_test.go index a4e389b31..24dd8135e 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -219,6 +219,45 @@ func TestHandleConflictingParamsWithJustRepeatFlag(t *testing.T) { } } +func TestHandleConflictingParamsWithRetryStepOnWithoutStepRetries(t *testing.T) { + origRepeat := repeat + origFailed := failed + origMaxStepRetriesCount := maxStepRetriesCount + origRetryStepOn := retryStepOn + defer func() { + repeat = origRepeat + failed = origFailed + maxStepRetriesCount = origMaxStepRetriesCount + retryStepOn = origRetryStepOn + }() + + args := []string{} + + var flags = pflag.FlagSet{} + flags.Int(maxStepRetriesName, maxStepRetriesDefault, "") + err := flags.Set(maxStepRetriesName, "1") + if err != nil { + t.Error(err) + } + flags.String(retryStepOnName, retryStepOnDefault, "") + err = flags.Set(retryStepOnName, "TimeoutException") + if err != nil { + t.Error(err) + } + + repeat = false + failed = false + maxStepRetriesCount = 1 + retryStepOn = "TimeoutException" + + expectedErrorMessage := "Invalid Command. flag --retry-step-on can be used only with --max-step-retries-count" + err = handleConflictingParams(&flags, args) + + if !reflect.DeepEqual(err.Error(), expectedErrorMessage) { + t.Errorf("Expected %v Got %v", expectedErrorMessage, err) + } +} + func TestHandleRerunFlagsWithVerbose(t *testing.T) { if os.Getenv("TEST_EXITS") == "1" { cmd := &cobra.Command{} diff --git a/execution/execute.go b/execution/execute.go index 665315a01..77d08ca7f 100644 --- a/execution/execute.go +++ b/execution/execute.go @@ -39,6 +39,7 @@ import ( "encoding/json" "path/filepath" + "regexp" "github.com/getgauge/common" "github.com/getgauge/gauge/config" @@ -58,11 +59,17 @@ const ( ) // Count of iterations -var MaxRetriesCount int +var MaxRetriesCount = 1 // Tags to filter specs/scenarios to retry var RetryOnlyTags string +// Count of iterations for failed steps +var MaxStepRetriesCount = 1 + +// Regex used to determine if failed step should be retried. +var RetryStepOn string + // NumberOfExecutionStreams shows the number of execution streams, in parallel execution. var NumberOfExecutionStreams int @@ -212,6 +219,14 @@ func validateFlags() error { if MaxRetriesCount < 1 { return fmt.Errorf("invalid input(%s) to --max-retries-count flag", strconv.Itoa(MaxRetriesCount)) } + if MaxStepRetriesCount < 1 { + return fmt.Errorf("invalid input(%s) to --max-step-retries-count flag", strconv.Itoa(MaxStepRetriesCount)) + } + if RetryStepOn != "" { + if _, err := regexp.Compile(RetryStepOn); err != nil { + return fmt.Errorf("invalid input(%s) to --retry-step-on flag", RetryStepOn) + } + } if !InParallel { return nil } diff --git a/execution/execute_test.go b/execution/execute_test.go index 510311596..b516777ee 100644 --- a/execution/execute_test.go +++ b/execution/execute_test.go @@ -69,3 +69,33 @@ func (s *MySuite) TestValidateFlagsWithInvalidStream(c *C) { err := validateFlags() c.Assert(err.Error(), Equals, "invalid input(-1) to --n flag") } + +func (s *MySuite) TestValidateFlagsWithInvalidMaxStepRetries(c *C) { + defer func() { + MaxRetriesCount = 1 + MaxStepRetriesCount = 1 + RetryStepOn = "" + InParallel = false + }() + InParallel = false + MaxRetriesCount = 1 + MaxStepRetriesCount = 0 + RetryStepOn = "" + err := validateFlags() + c.Assert(err.Error(), Equals, "invalid input(0) to --max-step-retries-count flag") +} + +func (s *MySuite) TestValidateFlagsWithInvalidStepRetryCondition(c *C) { + defer func() { + MaxRetriesCount = 1 + MaxStepRetriesCount = 1 + RetryStepOn = "" + InParallel = false + }() + InParallel = false + MaxRetriesCount = 1 + MaxStepRetriesCount = 2 + RetryStepOn = "[" + err := validateFlags() + c.Assert(err.Error(), Equals, "invalid input([) to --retry-step-on flag") +} diff --git a/execution/stepExecutor.go b/execution/stepExecutor.go index 0010d472e..f6df8317e 100644 --- a/execution/stepExecutor.go +++ b/execution/stepExecutor.go @@ -7,6 +7,8 @@ package execution import ( + "regexp" + "github.com/getgauge/gauge-proto/go/gauge_messages" "github.com/getgauge/gauge/execution/event" "github.com/getgauge/gauge/execution/result" @@ -39,7 +41,7 @@ func (e *stepExecutor) executeStep(step *gauge.Step, protoStep *gauge_messages.P e.notifyBeforeStepHook(stepResult) if !stepResult.GetFailed() { executeStepMessage := &gauge_messages.Message{MessageType: gauge_messages.Message_ExecuteStep, ExecuteStepRequest: stepRequest} - stepExecutionStatus := e.runner.ExecuteAndGetStatus(executeStepMessage) + stepExecutionStatus := e.executeStepWithRetries(executeStepMessage) stepExecutionStatus.Message = append(stepResult.ProtoStepExecResult().GetExecutionResult().Message, stepExecutionStatus.Message...) if stepExecutionStatus.GetFailed() { e.currentExecutionInfo.CurrentStep.ErrorMessage = stepExecutionStatus.GetErrorMessage() @@ -59,6 +61,43 @@ func (e *stepExecutor) executeStep(step *gauge.Step, protoStep *gauge_messages.P return stepResult } +func (e *stepExecutor) executeStepWithRetries(executeStepMessage *gauge_messages.Message) *gauge_messages.ProtoExecutionResult { + var stepExecutionStatus *gauge_messages.ProtoExecutionResult + for attempt := 0; attempt < MaxStepRetriesCount; attempt++ { + stepExecutionStatus = e.runner.ExecuteAndGetStatus(executeStepMessage) + if stepExecutionStatus == nil { + stepExecutionStatus = &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "runner returned empty step execution result"} + } + if !shouldRetryStep(stepExecutionStatus, attempt) { + break + } + } + return stepExecutionStatus +} + +func shouldRetryStep(stepExecutionStatus *gauge_messages.ProtoExecutionResult, currentAttempt int) bool { + if currentAttempt >= MaxStepRetriesCount-1 || !stepExecutionStatus.GetFailed() || stepExecutionStatus.GetSkipScenario() { + return false + } + if RetryStepOn == "" { + return true + } + stepRetryMatcher, err := regexp.Compile(RetryStepOn) + if err != nil { + return false + } + if stepRetryMatcher.MatchString(stepExecutionStatus.GetErrorMessage()) || + stepRetryMatcher.MatchString(stepExecutionStatus.GetStackTrace()) { + return true + } + for _, message := range stepExecutionStatus.GetMessage() { + if stepRetryMatcher.MatchString(message) { + return true + } + } + return false +} + func (e *stepExecutor) createStepRequest(protoStep *gauge_messages.ProtoStep) *gauge_messages.ExecuteStepRequest { stepRequest := &gauge_messages.ExecuteStepRequest{ParsedStepText: protoStep.GetParsedText(), ActualStepText: protoStep.GetActualText(), Stream: int32(e.stream)} stepRequest.Parameters = getParameters(protoStep.GetFragments()) @@ -98,5 +137,3 @@ func (e *stepExecutor) notifyAfterStepHook(stepResult *result.StepResult) { m.StepExecutionEndingRequest.StepResult = gauge.ConvertToProtoStepResult(stepResult) e.pluginHandler.NotifyPlugins(m) } - - diff --git a/execution/stepExecutor_test.go b/execution/stepExecutor_test.go index 939a6d1af..0828419fb 100644 --- a/execution/stepExecutor_test.go +++ b/execution/stepExecutor_test.go @@ -195,3 +195,122 @@ func TestStepExecutionShouldGetScreenshotsAfterStep(t *testing.T) { } } } + +func TestStepExecutionShouldRetryOnFailureWhenConditionIsNotSet(t *testing.T) { + MaxStepRetriesCount = 3 + RetryStepOn = "" + defer resetStepRetryOptions() + + r := &mockRunner{} + attempts := 0 + r.ExecuteAndGetStatusFunc = func(m *gauge_messages.Message) *gauge_messages.ProtoExecutionResult { + if m.MessageType == gauge_messages.Message_ExecuteStep { + attempts++ + if attempts < MaxStepRetriesCount { + return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "transient failure"} + } + } + return &gauge_messages.ProtoExecutionResult{} + } + + se, step, protoStep := newStepExecutorForRetryTests(r) + stepResult := se.executeStep(step, protoStep) + + if attempts != MaxStepRetriesCount { + t.Errorf("Expected step to execute %d times, got %d", MaxStepRetriesCount, attempts) + } + if stepResult.GetFailed() { + t.Errorf("Expected successful step execution result") + } +} + +func TestStepExecutionShouldRetryOnlyWhenConditionMatches(t *testing.T) { + MaxStepRetriesCount = 3 + RetryStepOn = "TimeoutException|429" + defer resetStepRetryOptions() + + r := &mockRunner{} + attempts := 0 + r.ExecuteAndGetStatusFunc = func(m *gauge_messages.Message) *gauge_messages.ProtoExecutionResult { + if m.MessageType == gauge_messages.Message_ExecuteStep { + attempts++ + if attempts == 1 { + return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "HTTP 429 TimeoutException"} + } + } + return &gauge_messages.ProtoExecutionResult{} + } + + se, step, protoStep := newStepExecutorForRetryTests(r) + stepResult := se.executeStep(step, protoStep) + + if attempts != 2 { + t.Errorf("Expected step to execute twice, got %d", attempts) + } + if stepResult.GetFailed() { + t.Errorf("Expected successful step execution result") + } +} + +func TestStepExecutionShouldNotRetryWhenConditionDoesNotMatch(t *testing.T) { + MaxStepRetriesCount = 3 + RetryStepOn = "TimeoutException" + defer resetStepRetryOptions() + + r := &mockRunner{} + attempts := 0 + r.ExecuteAndGetStatusFunc = func(m *gauge_messages.Message) *gauge_messages.ProtoExecutionResult { + if m.MessageType == gauge_messages.Message_ExecuteStep { + attempts++ + return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: "AssertionError"} + } + return &gauge_messages.ProtoExecutionResult{} + } + + se, step, protoStep := newStepExecutorForRetryTests(r) + stepResult := se.executeStep(step, protoStep) + + if attempts != 1 { + t.Errorf("Expected step to execute once, got %d", attempts) + } + if !stepResult.GetFailed() { + t.Errorf("Expected failed step execution result") + } +} + +func newStepExecutorForRetryTests(r *mockRunner) (*stepExecutor, *gauge.Step, *gauge_messages.ProtoStep) { + h := &mockPluginHandler{NotifyPluginsfunc: func(m *gauge_messages.Message) {}, GracefullyKillPluginsfunc: func() {}} + ei := &gauge_messages.ExecutionInfo{ + CurrentSpec: &gauge_messages.SpecInfo{ + Name: "Example Spec", + FileName: "example.spec", + IsFailed: false, + }, + CurrentScenario: &gauge_messages.ScenarioInfo{ + Name: "Example Scenario", + IsFailed: false, + }, + CurrentStep: &gauge_messages.StepInfo{ + Step: &gauge_messages.ExecuteStepRequest{ + ActualStepText: "a simple step", + ParsedStepText: "a simple step", + ScenarioFailing: false, + }, + IsFailed: false, + }, + } + se := &stepExecutor{runner: r, pluginHandler: h, currentExecutionInfo: ei, stream: 0} + step := &gauge.Step{ + Value: "a simple step", + LineText: "a simple step", + Fragments: []*gauge_messages.Fragment{{FragmentType: gauge_messages.Fragment_Text, Text: "a simple step"}}, + } + protoStep := gauge.ConvertToProtoItem(step).GetStep() + protoStep.StepExecutionResult = &gauge_messages.ProtoStepExecutionResult{} + return se, step, protoStep +} + +func resetStepRetryOptions() { + MaxStepRetriesCount = 1 + RetryStepOn = "" +}