Try #401:

Author: bors[bot]

src/utilities/request.jl

@@ -1,3 +1,21 @@
+# this is used to allow custom dispatches to `_http_request`
+abstract type AbstractBackend end
+
+"""
+    AWS.HTTPBackend <: AWS.AbstractBackend
+
+An `HTTPBackend` can hold default `http_options::AbstractDict{Symbol,<:Any}`
+to pass to HTTP.jl, which can be overwritten per-request by any `http_options`
+supplied there.
+"""
+struct HTTPBackend <: AbstractBackend
+    http_options::AbstractDict{Symbol,<:Any}
+end
+
+HTTPBackend(; kwargs...) = HTTPBackend(LittleDict(kwargs))
+
+const DEFAULT_BACKEND = Ref{AbstractBackend}(HTTPBackend())
+
 Base.@kwdef mutable struct Request
     service::String
     api_version::String
@@ -13,6 +31,7 @@ Base.@kwdef mutable struct Request
     http_options::AbstractDict{Symbol,<:Any}=LittleDict{Symbol,String}()
     return_raw::Bool=false
     response_dict_type::Type{<:AbstractDict}=LittleDict
+    backend::AbstractBackend=DEFAULT_BACKEND[]
 end
 
 
@@ -54,7 +73,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     @repeat 3 try
         credentials(aws) === nothing || sign!(aws, request)
 
-        response = @mock _http_request(request)
+        response = @mock _http_request(request.backend, request)
 
         if response.status in REDIRECT_ERROR_CODES
             if HTTP.header(response, "Location") != ""
@@ -141,23 +160,25 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
 end
 
 
-function _http_request(request::Request)
+function _http_request(http_backend::HTTPBackend, request::Request)
+    http_options = merge(http_backend.http_options, request.http_options)
+
     @repeat 4 try
         http_stack = HTTP.stack(redirect=false, retry=false, aws_authorization=false)
 
         if request.return_stream && request.response_stream === nothing
             request.response_stream = Base.BufferStream()
         end
 
-        return HTTP.request(
+        return @mock HTTP.request(
             http_stack,
             request.request_method,
             HTTP.URI(request.url),
             HTTP.mkheaders(request.headers),
             request.content;
             require_ssl_verification=false,
             response_stream=request.response_stream,
-            request.http_options...
+            http_options...
         )
     catch e
         # Base.IOError is needed because HTTP.jl can often have errors that aren't

src/utilities/utilities.jl

@@ -69,6 +69,7 @@ function _extract_common_kw_args(service, args)
         headers=LittleDict{String, String}(_pop!(args, "headers", [])),
         http_options=_pop!(args, "http_options", LittleDict{Symbol, String}()),
         response_dict_type=_pop!(args, "response_dict_type", LittleDict),
+        backend=_pop!(args, "backend", DEFAULT_BACKEND[]),
     )
 end
 

test/AWS.jl

@@ -351,6 +351,50 @@ end
     end
 end
 
+@testset "HTTPBackend" begin
+    request = Request(
+        service="s3",
+        api_version="api_version",
+        request_method="GET",
+        url="https://s3.us-east-1.amazonaws.com/sample-bucket",
+    )
+    apply(Patches._http_options_patch) do
+        # No default options
+        @test isempty(AWS._http_request(request.backend, request))
+
+        # We can pass options via the backend
+        custom_backend = AWS.HTTPBackend(Dict(:connection_limit => 5))
+        @test custom_backend isa AWS.AbstractBackend
+        @test AWS._http_request(custom_backend, request) == Dict(:connection_limit => 5)
+
+        # We can pass options per-request
+        request.http_options = Dict(:pipeline_limit => 20)
+        @test AWS._http_request(request.backend, request) == Dict(:pipeline_limit => 20)
+        @test AWS._http_request(custom_backend, request) == Dict(:pipeline_limit => 20, :connection_limit => 5)
+
+        # per-request options override backend options:
+        custom_backend = AWS.HTTPBackend(Dict(:pipeline_limit => 5))
+        @test AWS._http_request(custom_backend, request) == Dict(:pipeline_limit => 20)
+    end
+
+    # Let's test setting the default backend
+    try
+        AWS.DEFAULT_BACKEND[] = AWS.HTTPBackend(; connection_limit = 3)
+        request = Request(
+            service="s3",
+            api_version="api_version",
+            request_method="GET",
+            url="https://s3.us-east-1.amazonaws.com/sample-bucket",
+        )
+        @test request.backend.http_options == Dict(:connection_limit => 3)
+        apply(Patches._http_options_patch) do
+            @test AWS._http_request(request.backend, request) == Dict(:connection_limit => 3)
+        end
+    finally
+        AWS.DEFAULT_BACKEND[] = AWS.HTTPBackend()
+    end
+end
+
 @testset "_generate_rest_resource" begin
     request_uri = "/{Bucket}/{Key+}"
     args = Dict{String, Any}(

test/patch.jl

@@ -56,7 +56,7 @@ end
 
 
 function _aws_http_request_patch(response::HTTP.Messages.Response=_response())
-    return @patch AWS._http_request(request::Request) = response
+    return @patch AWS._http_request(::AWS.HTTPBackend, request::Request) = response
 end
 
 _cred_file_patch = @patch function dot_aws_credentials_file()
@@ -73,7 +73,7 @@ _web_identity_patch = function (;
     session_token="web_session_token",
     role_arn="arn:aws:sts:::assumed-role/role-name",
 )
-    @patch function AWS._http_request(request)
+    @patch function AWS._http_request(::AWS.HTTPBackend, request)
         params = Dict(split.(split(request.content, '&'), '='))
         creds = Dict(
             "AccessKeyId" => access_key,
@@ -107,4 +107,15 @@ _instance_metadata_timeout_patch = @patch function HTTP.request(method::String,
     throw(HTTP.ConnectionPool.ConnectTimeout("169.254.169.254", "80"))
 end
 
+# This patch causes `HTTP.request` to return all of its keyword arguments
+# except `require_ssl_verification` and `response_stream`. This is used to
+# test which other options are being passed to `HTTP.Request` inside of
+# `_http_request`.
+_http_options_patch = @patch function HTTP.request(args...; kwargs...)
+    options = Dict(kwargs)
+    delete!(options, :require_ssl_verification)
+    delete!(options, :response_stream)
+    return options
+end
+
 end