Merge pull request #311 from andrewjjenkins/mirror

Add traffic mirroring for Istio service mesh

Author: stefanprodan

artifacts/flagger/crd.yaml

@@ -41,6 +41,10 @@ spec:
       type: string
       JSONPath: .spec.canaryAnalysis.interval
       priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
     - name: StepWeight
       type: string
       JSONPath: .spec.canaryAnalysis.stepWeight
@@ -183,6 +187,9 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
                 match:
                   description: A/B testing match conditions
                   anyOf:

charts/flagger/templates/crd.yaml

@@ -42,6 +42,10 @@ spec:
       type: string
       JSONPath: .spec.canaryAnalysis.interval
       priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
     - name: StepWeight
       type: string
       JSONPath: .spec.canaryAnalysis.stepWeight
@@ -184,6 +188,9 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
                 match:
                   description: A/B testing match conditions
                   anyOf:

docs/gitbook/faq.md

@@ -102,6 +102,50 @@ The above configuration will run an analysis for five minutes.
 Flagger starts the load test for the canary service (green version) and checks the Prometheus metrics every 30 seconds.
 If the analysis result is positive, Flagger will promote the canary (green version) to primary (blue version).
 
+**When can I use traffic mirroring?**
+
+Traffic Mirroring is a pre-stage in a Canary (progressive traffic shifting) or
+Blue/Green deployment strategy.  Traffic mirroring will copy each incoming
+request, sending one request to the primary and one to the canary service.  The
+response from the primary is sent back to the user.  The response from the canary
+is discarded.  Metrics are collected on both requests so that the deployment will
+only proceed if the canary metrics are healthy.
+
+Mirroring is supported by Istio only.
+
+In Istio, mirrored requests have `-shadow` appended to the `Host` (HTTP) or
+`Authority` (HTTP/2) header; for example requests to `podinfo.test` that are
+mirrored will be reported in telemetry with a destination host
+`podinfo.test-shadow`.
+
+Mirroring must only be used for requests that are **idempotent** or capable of
+being processed twice (once by the primary and once by the canary).  Reads are
+idempotent.  Before using mirroring on requests that may be writes, you should
+consider what will happen if a write is duplicated and handled by the primary
+and canary.
+
+To use mirroring, set `spec.canaryAnalysis.mirror` to `true`.  Example for
+traffic shifting:
+
+```yaml
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+spec:
+  provider: istio
+  canaryAnalysis:
+    interval: 30s
+    mirror: true
+    stepWeight: 20
+    maxWeight: 50
+    metrics:
+    - interval: 29s
+      name: request-success-rate
+      threshold: 99
+    - interval: 29s
+      name: request-duration
+      threshold: 500
+```
+
 ### Kubernetes services
 
 **How is an application exposed inside the cluster?**

kustomize/base/flagger/crd.yaml

@@ -41,6 +41,10 @@ spec:
       type: string
       JSONPath: .spec.canaryAnalysis.interval
       priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
     - name: StepWeight
       type: string
       JSONPath: .spec.canaryAnalysis.stepWeight
@@ -183,6 +187,9 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
                 match:
                   description: A/B testing match conditions
                   anyOf:

pkg/apis/flagger/v1alpha3/types.go

@@ -111,6 +111,7 @@ type CanaryAnalysis struct {
 	Interval   string                           `json:"interval"`
 	Threshold  int                              `json:"threshold"`
 	MaxWeight  int                              `json:"maxWeight"`
+	Mirror     bool                             `json:"mirror,omitempty"`
 	StepWeight int                              `json:"stepWeight"`
 	Metrics    []CanaryMetric                   `json:"metrics"`
 	Webhooks   []CanaryWebhook                  `json:"webhooks,omitempty"`

pkg/controller/controller_test.go

@@ -42,11 +42,9 @@ type Mocks struct {
 	router        router.Interface
 }
 
-func SetupMocks(abtest bool) Mocks {
-	// init canary
-	c := newTestCanary()
-	if abtest {
-		c = newTestCanaryAB()
+func SetupMocks(c *v1alpha3.Canary) Mocks {
+	if c == nil {
+		c = newTestCanary()
 	}
 	flaggerClient := fakeFlagger.NewSimpleClientset(c)
 
@@ -269,6 +267,12 @@ func newTestCanary() *v1alpha3.Canary {
 	return cd
 }
 
+func newTestCanaryMirror() *v1alpha3.Canary {
+	cd := newTestCanary()
+	cd.Spec.CanaryAnalysis.Mirror = true
+	return cd
+}
+
 func newTestCanaryAB() *v1alpha3.Canary {
 	cd := &v1alpha3.Canary{
 		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},

pkg/controller/scheduler.go

@@ -155,7 +155,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 
 	// check if virtual service exists
 	// and if it contains weighted destination routes to the primary and canary services
-	primaryWeight, canaryWeight, err := meshRouter.GetRoutes(cd)
+	primaryWeight, canaryWeight, mirrored, err := meshRouter.GetRoutes(cd)
 	if err != nil {
 		c.recordEventWarningf(cd, "%v", err)
 		return
@@ -176,7 +176,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		// route all traffic back to primary
 		primaryWeight = 100
 		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
@@ -218,7 +218,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if cd.Status.Phase == flaggerv1.CanaryPhasePromoting {
 		if provider != "kubernetes" {
 			c.recordEventInfof(cd, "Routing all traffic to primary")
-			if err := meshRouter.SetRoutes(cd, 100, 0); err != nil {
+			if err := meshRouter.SetRoutes(cd, 100, 0, false); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
@@ -275,7 +275,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		// route all traffic back to primary
 		primaryWeight = 100
 		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
@@ -302,8 +302,9 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	}
 
 	// check if the canary success rate is above the threshold
-	// skip check if no traffic is routed to canary
-	if canaryWeight == 0 && cd.Status.Iterations == 0 {
+	// skip check if no traffic is routed or mirrored to canary
+	if canaryWeight == 0 && cd.Status.Iterations == 0 &&
+		(cd.Spec.CanaryAnalysis.Mirror == false || mirrored == false) {
 		c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
 
 		// run pre-rollout web hooks
@@ -328,7 +329,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 {
 		// route traffic to canary and increment iterations
 		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
-			if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
+			if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
@@ -372,6 +373,15 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if cd.Spec.CanaryAnalysis.Iterations > 0 {
 		// increment iterations
 		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
+			// If in "mirror" mode, mirror requests during the entire B/G canary test
+			if provider != "kubernetes" &&
+				cd.Spec.CanaryAnalysis.Mirror == true && mirrored == false {
+				if err := meshRouter.SetRoutes(cd, 100, 0, true); err != nil {
+					c.recordEventWarningf(cd, "%v", err)
+				}
+				c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
+					Infof("Enabling mirroring for Blue/Green")
+			}
 			if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
@@ -390,7 +400,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
 			if provider != "kubernetes" {
 				c.recordEventInfof(cd, "Routing all traffic to canary")
-				if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
+				if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
 					c.recordEventWarningf(cd, "%v", err)
 					return
 				}
@@ -429,16 +439,34 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if cd.Spec.CanaryAnalysis.StepWeight > 0 {
 		// increase traffic weight
 		if canaryWeight < maxWeight {
-			primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
-			if primaryWeight < 0 {
-				primaryWeight = 0
-			}
-			canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
-			if canaryWeight > 100 {
-				canaryWeight = 100
+			// If in "mirror" mode, do one step of mirroring before shifting traffic to canary.
+			// When mirroring, all requests go to primary and canary, but only responses from
+			// primary go back to the user.
+			if cd.Spec.CanaryAnalysis.Mirror && canaryWeight == 0 {
+				if mirrored == false {
+					mirrored = true
+					primaryWeight = 100
+					canaryWeight = 0
+				} else {
+					mirrored = false
+					primaryWeight = 100 - cd.Spec.CanaryAnalysis.StepWeight
+					canaryWeight = cd.Spec.CanaryAnalysis.StepWeight
+				}
+				c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
+					Infof("Running mirror step %d/%d/%t", primaryWeight, canaryWeight, mirrored)
+			} else {
+
+				primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
+				if primaryWeight < 0 {
+					primaryWeight = 0
+				}
+				canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
+				if canaryWeight > 100 {
+					canaryWeight = 100
+				}
 			}
 
-			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, mirrored); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
@@ -489,7 +517,7 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
 	// route all traffic to primary
 	primaryWeight = 100
 	canaryWeight = 0
-	if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+	if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 		c.recordEventWarningf(cd, "%v", err)
 		return false
 	}