Engine: refactor task retry backoff logic (#123)

* Engine: refactor task retry backoff logic
* Backoff README

Author: loopfz

Diff for: README.md

@@ -298,7 +298,11 @@ steps:
 - `name`: a unique identifier
 - `description`: a human readable sentence to convey the step's intent
 - `dependencies`: a list of step names on which this step waits before running
-- `retry_pattern`: (seconds|minutes|hours) define on what temporal order of magnitude the re-runs of this step should be spread
+- `retry_pattern`: (`seconds`, `minutes`, `hours`) define on what temporal order of magnitude the re-runs of this step should be spread (default = `seconds`)
+
+<p align="center">
+<img src="./assets/img/utask_backoff.png" width="70%">
+</p>
 
 #### Action
 

Diff for: assets/img/utask_backoff.png

@@ -5,12 +5,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math"
 	"strings"
 	"time"
 
 	"github.com/cenkalti/backoff"
 	"github.com/ghodss/yaml"
+	expbk "github.com/jpillora/backoff"
 	"github.com/juju/errors"
 	"github.com/loopfz/gadgeto/zesty"
 	"github.com/ovh/configstore"
@@ -820,25 +820,54 @@ func minDuration(a, b time.Duration) time.Duration {
 	return b
 }
 
-// for retryCount:    1,  2,  3,  4,  5,  6......
-// seconds: 	      10, 17, 21, 24, 26, 28.....
-// minutes and hours: 1,  1,  7,  11, 14, 16.....
-func computeDelay(d time.Duration, rc int) time.Duration {
-	// shorter two first retries for "minutes" and "hours" retry patterns
-	if d >= time.Minute {
-		if rc <= 2 {
-			return d
-		}
-		rc--
+const maxMinutes = 30 * time.Minute
+
+// 1m36s, 3m12s, 4m48s, 6m24s, 8m0s, 9m36s, 11m12s, 12m48s, 14m24s, 16m0s
+// 17m36s, 19m12s, 20m48s, 22m24s, 24m0s, 25m36s, 27m12s, 28m48s, 30m0s
+// 30m0s, 30m0s, ...
+// cap 30m0s is reached at 20th retry
+func computeDelayMinutesLinear(d time.Duration, rc int) time.Duration {
+	ret := time.Duration((float64(1.6) * float64(rc)) * float64(d))
+	if ret >= maxMinutes {
+		ret = maxMinutes
 	}
-	ret := time.Duration(float64(d) * math.Log(float64(rc)) * 10)
-	// baseline 10s for "seconds" retry pattern
-	if d < time.Minute {
-		ret += 10 * d
+	return ret
+}
+
+const maxHours = 3 * time.Hour
+
+// 1h, 1h10, 1h20, 1h30, ... 3h, 3h, 3h, ...
+// cap 3h is reached at 13th retry
+func computeDelayHoursLinear(d time.Duration, rc int) time.Duration {
+	rc--
+	ret := d + (time.Duration(rc) * 10 * time.Minute)
+	if ret >= maxHours {
+		ret = maxHours
 	}
 	return ret
 }
 
+var expBackoff = expbk.Backoff{Min: 8 * time.Second, Max: 10 * time.Minute, Factor: 1.25}
+
+// 10s, 12.5s, 15.625s, 19.53125s, 24.4140625s, 30.517578125s, 38.146972656s, 47.68371582s
+// 59.604644775s, 1m14.505805969s, 1m33.132257461s, 1m56.415321826s, 2m25.519152283s
+// 3m1.898940354s, 3m47.373675443s, 4m44.217094304s, 5m55.27136788s, 7m24.08920985s, 9m15.111512312s
+// 10m0s, 10m0s, 10m0s, ...
+// the 10m0s cap is reached at the 20th retry
+func computeDelaySecondsExponential(d time.Duration, rc int) time.Duration {
+	return expBackoff.ForAttempt(float64(rc))
+}
+
+func computeDelay(d time.Duration, rc int) time.Duration {
+	switch d {
+	case time.Minute:
+		return computeDelayMinutesLinear(d, rc)
+	case time.Hour:
+		return computeDelayHoursLinear(d, rc)
+	}
+	return computeDelaySecondsExponential(time.Second, rc)
+}
+
 func resolutionStateSetter(res *resolution.Resolution, modifiedSteps map[string]bool) step.StateSetter {
 	return func(step, state, message string) {
 		if _, ok := res.Steps[step]; ok {

Diff for: engine/engine.go

@@ -22,6 +22,7 @@ require (
 	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/jinzhu/now v1.0.1 // indirect
+	github.com/jpillora/backoff v1.0.0
 	github.com/juju/errors v0.0.0-20190207033735-e65537c515d7
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 // indirect
@@ -57,7 +58,7 @@ require (
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.4.0
 	github.com/tjarratt/babble v0.0.0-20191126185718-ccc47f626248 // indirect
-	github.com/ugorji/go/codec v1.1.7 // indirect
+	github.com/ugorji/go v1.1.7 // indirect
 	github.com/wI2L/fizz v0.0.0-20190425144348-6274bc96d962
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392

Diff for: go.mod

@@ -108,6 +108,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=