core (grpc) use retry strategy

Author: stephenplusplus

lib/common/grpc-service.js

@@ -25,6 +25,7 @@ var grpc = require('grpc');
 var is = require('is');
 var nodeutil = require('util');
 var path = require('path');
+var retryRequest = require('retry-request');
 
 /**
  * @type {module:common/service}
@@ -241,19 +242,47 @@ GrpcService.prototype.request = function(protoOpts, reqOpts, callback) {
     grpcOpts.deadline = new Date(Date.now() + protoOpts.timeout);
   }
 
-  service[protoOpts.method](reqOpts, function(err, resp) {
-    if (err) {
-      if (HTTP_ERROR_CODE_MAP[err.code]) {
-        var httpError = HTTP_ERROR_CODE_MAP[err.code];
-        err.code = httpError.code;
-      }
-
-      callback(err);
-      return;
+  // Retains a reference to an error from the response. If the final callback is
+  // executed with this as the "response", we return it to the user as an error.
+  var respError;
+
+  retryRequest(null, {
+    shouldRetryFn: function(resp) {
+      return [429, 500, 502, 503].indexOf(resp.code) > -1;
+    },
+
+    // retry-request determines if it should retry from the incoming HTTP
+    // response status. gRPC always returns an error proto message. We pass that
+    // "error" into retry-request to act as the HTTP response, so it can use the
+    // status code to determine if it should retry.
+    request: function(_, onResponse) {
+      respError = null;
+
+      service[protoOpts.method](reqOpts, function(err, resp) {
+        if (err) {
+          if (HTTP_ERROR_CODE_MAP[err.code]) {
+            respError = err;
+            var httpError = HTTP_ERROR_CODE_MAP[err.code];
+            respError.code = httpError.code;
+            onResponse(null, respError);
+            return;
+          }
+
+          onResponse(err);
+          return;
+        }
+
+        onResponse(null, resp);
+      }, null, grpcOpts);
+    }
+  }, function(err, resp) {
+    if (!err && resp === respError) {
+      err = respError;
+      resp = null;
     }
 
-    callback(null, resp);
-  }, null, grpcOpts);
+    callback(err, resp);
+  });
 };
 
 /**