Adding retries to AWS API calls

Adding retries to AWS API calls

Author: biacar

deps/rabbitmq_aws/include/rabbitmq_aws.hrl

@@ -43,6 +43,9 @@
 
 -define(METADATA_TOKEN, "X-aws-ec2-metadata-token").
 
+-define(LINEAR_BACK_OFF_MILLIS, 500).
+-define(MAX_RETRIES, 5).
+
 -type access_key() :: nonempty_string().
 -type secret_access_key() :: nonempty_string().
 -type expiration() :: calendar:datetime() | undefined.

deps/rabbitmq_aws/src/rabbitmq_aws.erl

@@ -543,10 +543,22 @@ ensure_credentials_valid() ->
 %% @end
 api_get_request(Service, Path) ->
   rabbit_log:debug("Invoking AWS request {Service: ~p; Path: ~p}...", [Service, Path]),
+  api_get_request_with_retries(Service, Path, ?MAX_RETRIES, ?LINEAR_BACK_OFF_MILLIS).
+
+
+  -spec api_get_request_with_retries(string(), path(), integer(), integer()) -> result().
+  %% @doc Invoke an API call to an AWS service with retries.
+  %% @end
+api_get_request_with_retries(Service, Path, Retries, WaitTimeBetweenRetries) ->
   ensure_credentials_valid(),
   case get(Service, Path) of
     {ok, {_Headers, Payload}} -> rabbit_log:debug("AWS request: ~s~nResponse: ~p", [Path, Payload]),
                                  {ok, Payload};
     {error, {credentials, _}} -> {error, credentials};
-    {error, Message, _}       -> {error, Message}
+    {error, Message, _}       -> case Retries of
+                                   0 -> {error, Message};
+                                   _ -> rabbit_log:warning("Error occurred ~s~nWill retry AWS request, remaining retries: ~b", [Message, Retries]),
+                                        timer:sleep(WaitTimeBetweenRetries),
+                                        api_get_request_with_retries(Service, Path, Retries - 1, WaitTimeBetweenRetries)
+                                 end
   end.

deps/rabbitmq_aws/test/src/rabbitmq_aws_tests.erl

@@ -479,7 +479,7 @@ api_get_request_test_() ->
           ?assertEqual({error, credentials}, Result)
         end
       },
-      {"AWS service API request failed - API error",
+      {"AWS service API request failed - API error with persistent failure",
         fun() ->
           State = #state{access_key = "ExpiredKey",
                          secret_access_key = "ExpiredAccessKey",
@@ -489,11 +489,27 @@ api_get_request_test_() ->
           {ok, Pid} = rabbitmq_aws:start_link(),
           rabbitmq_aws:set_region("us-east-1"),
           rabbitmq_aws:set_credentials(State),
-          Result = rabbitmq_aws:api_get_request("AWS", "API"),
+          Result = rabbitmq_aws:api_get_request_with_retries("AWS", "API", 3, 1),
           ok = gen_server:stop(Pid),
           ?assertEqual({error, "invalid input"}, Result),
           meck:validate(httpc)
         end
+      },
+      {"AWS service API request succeeded after a transient error",
+        fun() ->
+          State = #state{access_key = "ExpiredKey",
+                         secret_access_key = "ExpiredAccessKey",
+                         region = "us-east-1",
+                         expiration = {{3016, 4, 1}, {12, 0, 0}}},
+          meck:expect(httpc, request, 4, meck:seq([{error, "invalid input"}, {ok, {{"HTTP/1.0", 200, "OK"}, [{"content-type", "application/json"}], "{\"data\": \"value\"}"}}])),
+          {ok, Pid} = rabbitmq_aws:start_link(),
+          rabbitmq_aws:set_region("us-east-1"),
+          rabbitmq_aws:set_credentials(State),
+          Result = rabbitmq_aws:api_get_request_with_retries("AWS", "API", 3, 1),
+          ok = gen_server:stop(Pid),
+          ?assertEqual({ok, [{"data","value"}]}, Result),
+          meck:validate(httpc)
+        end
       }
     ]
   }.