Land #14139, Add cookie management to HttpClient and improve standards compliance

Author: smcintyre-r7

lib/msf/core/exploit/http/client.rb

@@ -2,6 +2,7 @@
 
 require 'uri'
 require 'digest'
+
 module Msf
 
 ###
@@ -11,6 +12,7 @@ module Msf
 #
 ###
 module Exploit::Remote::HttpClient
+
   include Msf::Auxiliary::Report
 
   #
@@ -47,7 +49,6 @@ def initialize(info = {})
         OptBool.new('FingerprintCheck', [ false, 'Conduct a pre-exploit fingerprint verification', true]),
         OptString.new('DOMAIN', [ true, 'The domain to use for Windows authentication', 'WORKSTATION']),
         OptFloat.new('HttpClientTimeout', [false, 'HTTP connection and receive timeout']),
-        OptBool.new('HttpPartialResponses', [false, 'Return partial HTTP responses despite timeouts', false]),
         OptBool.new('HttpTrace', [false, 'Show the raw HTTP requests and responses', false]),
         OptBool.new('HttpTraceHeadersOnly', [false, 'Show HTTP headers only in HttpTrace', false]),
         OptString.new('HttpTraceColors', [false, 'HTTP request and response colors for HttpTrace (unset to disable)', 'red/blu'])
@@ -88,6 +89,9 @@ def initialize(info = {})
     )
     register_autofilter_ports([ 80, 8080, 443, 8000, 8888, 8880, 8008, 3000, 8443 ])
     register_autofilter_services(%W{ http https })
+
+    # Initialize an empty cookie jar to keep cookies
+    self.cookie_jar = Set.new
   end
 
   def deregister_http_client_options
@@ -168,7 +172,7 @@ def connect(opts={})
     nclient.set_config(
       'vhost' => opts['vhost'] || opts['rhost'] || self.vhost(),
       'agent' => datastore['UserAgent'],
-      'partial' => opts['partial'] || datastore['HttpPartialResponses'],
+      'partial' => opts['partial'],
       'uri_encode_mode'        => datastore['HTTP::uri_encode_mode'],
       'uri_full_url'           => datastore['HTTP::uri_full_url'],
       'pad_method_uri_count'   => datastore['HTTP::pad_method_uri_count'],
@@ -314,69 +318,80 @@ def cleanup
   #
   # Passes +opts+ through directly to Rex::Proto::Http::Client#request_raw.
   #
-  def send_request_raw(opts={}, timeout = 20, disconnect = false)
+  def send_request_raw(opts = {}, timeout = 20, disconnect = false)
     if datastore['HttpClientTimeout'] && datastore['HttpClientTimeout'] > 0
       actual_timeout = datastore['HttpClientTimeout']
     else
-      actual_timeout =  opts[:timeout] || timeout
+      actual_timeout = opts[:timeout] || timeout
     end
 
-    begin
-      c = connect(opts)
-      r = opts[:cgi] ? c.request_cgi(opts) : c.request_raw(opts)
+    c = connect(opts)
+    r = opts[:cgi] ? c.request_cgi(opts) : c.request_raw(opts)
 
-      if datastore['HttpTrace']
-        request_color, response_color =
-          (datastore['HttpTraceColors'] || '').split('/').map { |color| "%bld%#{color}" }
+    if datastore['HttpTrace']
+      request_color, response_color =
+        (datastore['HttpTraceColors'] || '').split('/').map { |color| "%bld%#{color}" }
 
-        request = r.to_s(headers_only: datastore['HttpTraceHeaders'])
+      request = r.to_s(headers_only: datastore['HttpTraceHeaders'])
 
-        print_line('#' * 20)
-        print_line('# Request:')
-        print_line('#' * 20)
-        print_line("%clr#{request_color}#{request}%clr")
-      end
+      print_line('#' * 20)
+      print_line('# Request:')
+      print_line('#' * 20)
+      print_line("%clr#{request_color}#{request}%clr")
+    end
 
-      res = c.send_recv(r, actual_timeout)
+    res = c.send_recv(r, actual_timeout)
 
-      if datastore['HttpTrace']
-        print_line('#' * 20)
-        print_line('# Response:')
-        print_line('#' * 20)
+    if datastore['HttpTrace']
+      print_line('#' * 20)
+      print_line('# Response:')
+      print_line('#' * 20)
 
-        if res
-          response = res.to_terminal_output(headers_only: datastore['HttpTraceHeadersOnly'])
+      if res
+        response = res.to_terminal_output(headers_only: datastore['HttpTraceHeadersOnly'])
 
-          print_line("%clr#{response_color}#{response}%clr")
-        else
-          print_line('No response received')
-        end
+        print_line("%clr#{response_color}#{response}%clr")
+      else
+        print_line('No response received')
       end
+    end
 
-      disconnect(c) if disconnect
+    disconnect(c) if disconnect
 
-      res
-    rescue ::Errno::EPIPE, ::Timeout::Error => e
-      print_line(e.message) if datastore['HttpTrace']
-      nil
-    rescue Rex::ConnectionError => e
-      vprint_error(e.to_s)
-      nil
-    rescue ::Exception => e
-      print_line(e.message) if datastore['HttpTrace']
-      raise e
-    end
+    res
+  rescue ::Errno::EPIPE, ::Timeout::Error => e
+    print_line(e.message) if datastore['HttpTrace']
+    nil
+  rescue Rex::ConnectionError => e
+    vprint_error(e.to_s)
+    nil
+  rescue ::Exception => e
+    print_line(e.message) if datastore['HttpTrace']
+    raise e
   end
 
-
   # Connects to the server, creates a request, sends the request,
   # reads the response
   #
   # Passes `opts` through directly to {Rex::Proto::Http::Client#request_cgi}.
+  # Set `opts['keep_cookies']` to keep cookies from responses for reuse in requests.
   #
   # @return (see Rex::Proto::Http::Client#send_recv))
-  def send_request_cgi(opts={}, timeout = 20, disconnect = true)
-    send_request_raw(opts.merge(cgi: true), timeout, disconnect)
+  def send_request_cgi(opts = {}, timeout = 20, disconnect = true)
+    if cookie_jar.any?
+      opts = { 'cookie' => cookie_jar.to_a.join(' ') }.merge(opts)
+    end
+
+    res = send_request_raw(opts.merge(cgi: true), timeout, disconnect)
+
+    return unless res
+
+    if opts['keep_cookies'] && res.headers['Set-Cookie'].present?
+      # XXX: CGI::Cookie (get_cookies_parsed) is hella broken
+      cookie_jar.merge(res.get_cookies.split(' '))
+    end
+
+    res
   end
 
   # Connects to the server, creates a request, sends the request, reads the
@@ -387,30 +402,31 @@ def send_request_cgi(opts={}, timeout = 20, disconnect = true)
   #   `opts['redirect_uri']` will contain the full URI.
   #
   # @return (see #send_request_cgi)
-  def send_request_cgi!(opts={}, timeout = 20, redirect_depth = 1)
-    if datastore['HttpClientTimeout'] && datastore['HttpClientTimeout'] > 0
-      actual_timeout = datastore['HttpClientTimeout']
-    else
-      actual_timeout =  opts[:timeout] || timeout
-    end
+  def send_request_cgi!(opts = {}, timeout = 20, redirect_depth = 1)
+    res = send_request_cgi(opts, timeout)
 
-    res = send_request_cgi(opts, actual_timeout)
-    return res unless res && res.redirect? && redirect_depth > 0
+    return unless res
+    return res unless res.redirect? && res.redirection && redirect_depth > 0
 
     redirect_depth -= 1
-    return res if res.redirection.nil?
 
     reconfig_redirect_opts!(res, opts)
-    send_request_cgi!(opts, actual_timeout, redirect_depth)
+    send_request_cgi!(opts, timeout, redirect_depth)
   end
 
-
   # Modifies the HTTP request options for a redirection.
   #
   # @param res [Rex::Proto::HTTP::Response] HTTP Response.
   # @param opts [Hash] The HTTP request options to modify.
   # @return [void]
   def reconfig_redirect_opts!(res, opts)
+    # XXX: https://github.com/rapid7/metasploit-framework/issues/12281
+    if opts['method'] == 'POST'
+      opts['method'] = 'GET'
+      opts['data'] = nil
+      opts['vars_post'] = {}
+    end
+
     location = res.redirection
 
     if location.relative?
@@ -425,7 +441,7 @@ def reconfig_redirect_opts!(res, opts)
         opts['redirect_uri'] = new_redirect_uri
         opts['uri'] = new_redirect_uri
       end
-      
+
       opts['rhost'] = datastore['RHOST']
       opts['vhost'] = opts['vhost'] || opts['rhost'] || self.vhost()
       opts['rport'] = datastore['RPORT']
@@ -446,6 +462,9 @@ def reconfig_redirect_opts!(res, opts)
         opts['SSL'] = false
       end
     end
+
+    # Don't forget any GET parameters
+    opts['query'] ||= location.query if location.query
   end
 
   #
@@ -867,10 +886,10 @@ def service_details
     }
   end
 
-protected
+  protected
 
   attr_accessor :client
+  attr_accessor :cookie_jar
 
 end
-
 end

lib/rex/proto/http/client.rb

@@ -123,38 +123,36 @@ def set_config(opts = {})
   # @option opts 'vhost'         [String] Host header value
   #
   # @return [ClientRequest]
-  def request_raw(opts={})
+  def request_raw(opts = {})
     opts = self.config.merge(opts)
 
-    opts['ssl']         = self.ssl
-    opts['cgi']         = false
-    opts['port']        = self.port
+    opts['cgi'] = false
+    opts['port'] = self.port
+    opts['ssl'] = self.ssl
 
-    req = ClientRequest.new(opts)
+    ClientRequest.new(opts)
   end
 
-
   #
   # Create a CGI compatible request
   #
   # @param (see #request_raw)
   # @option opts (see #request_raw)
-  # @option opts 'ctype'         [String] Content-Type header value, default: +application/x-www-form-urlencoded+
+  # @option opts 'ctype'         [String] Content-Type header value, default for POST requests: +application/x-www-form-urlencoded+
   # @option opts 'encode_params' [Bool]   URI encode the GET or POST variables (names and values), default: true
   # @option opts 'vars_get'      [Hash]   GET variables as a hash to be translated into a query string
   # @option opts 'vars_post'     [Hash]   POST variables as a hash to be translated into POST data
   #
   # @return [ClientRequest]
-  def request_cgi(opts={})
+  def request_cgi(opts = {})
     opts = self.config.merge(opts)
 
-    opts['ctype']       ||= 'application/x-www-form-urlencoded'
-    opts['ssl']         = self.ssl
-    opts['cgi']         = true
-    opts['port']        = self.port
+    opts['cgi'] = true
+    opts['port'] = self.port
+    opts['ssl'] = self.ssl
+    opts['ctype'] ||= 'application/x-www-form-urlencoded' if opts['method'] == 'POST'
 
-    req = ClientRequest.new(opts)
-    req
+    ClientRequest.new(opts)
   end
 
   #

lib/rex/proto/http/response.rb

@@ -219,11 +219,9 @@ def redirect?
   # @return [URI] the uri of the redirection location.
   # @return [nil] if the response hasn't a Location header or it isn't a valid uri.
   def redirection
-    begin
-      URI(headers['Location'])
-    rescue ::URI::InvalidURIError
-      nil
-    end
+    URI(headers['Location'])
+  rescue ArgumentError, ::URI::InvalidURIError
+    nil
   end
 
   #