Rename existing runner parameter from "hosts_blacklist" to

"url_hosts_blacklist" and also add support for whitelist approach using
"url_hosts_whitelist" runner parameter.

Author: Kami

contrib/runners/http_runner/http_runner/http_runner.py

@@ -57,7 +57,8 @@
 RUNNER_VERIFY_SSL_CERT = 'verify_ssl_cert'
 RUNNER_USERNAME = 'username'
 RUNNER_PASSWORD = 'password'
-RUNNER_HOSTS_BLACKLIST = 'hosts_blacklist'
+RUNNER_URL_HOSTS_BLACKLIST = 'url_hosts_blacklist'
+RUNNER_URL_HOSTS_WHITELIST = 'url_hosts_whitelist'
 
 # Lookup constants for action params
 ACTION_AUTH = 'auth'
@@ -96,11 +97,17 @@ def pre_run(self):
         self._http_proxy = self.runner_parameters.get(RUNNER_HTTP_PROXY, None)
         self._https_proxy = self.runner_parameters.get(RUNNER_HTTPS_PROXY, None)
         self._verify_ssl_cert = self.runner_parameters.get(RUNNER_VERIFY_SSL_CERT, None)
-        self._hosts_blacklist = self.runner_parameters.get(RUNNER_HOSTS_BLACKLIST, [])
+        self._url_hosts_blacklist = self.runner_parameters.get(RUNNER_URL_HOSTS_BLACKLIST, [])
+        self._url_hosts_whitelist = self.runner_parameters.get(RUNNER_URL_HOSTS_WHITELIST, [])
 
     def run(self, action_parameters):
         client = self._get_http_client(action_parameters)
 
+        if self._url_hosts_blacklist and self._url_hosts_whitelist:
+            msg = ('"url_hosts_blacklist" and "url_hosts_whitelist" parameters are mutually '
+                   'exclusive. Only one should be provided.')
+            raise ValueError(msg)
+
         try:
             result = client.run()
         except requests.exceptions.Timeout as e:
@@ -152,7 +159,8 @@ def _get_http_client(self, action_parameters):
                           timeout=timeout, allow_redirects=self._allow_redirects,
                           proxies=proxies, files=files, verify=self._verify_ssl_cert,
                           username=self._username, password=self._password,
-                          hosts_blacklist=self._hosts_blacklist)
+                          url_hosts_blacklist=self._url_hosts_blacklist,
+                          url_hosts_whitelist=self._url_hosts_whitelist)
 
     @staticmethod
     def _get_result_status(status_code):
@@ -164,7 +172,7 @@ class HTTPClient(object):
     def __init__(self, url=None, method=None, body='', params=None, headers=None, cookies=None,
                  auth=None, timeout=60, allow_redirects=False, proxies=None,
                  files=None, verify=False, username=None, password=None,
-                 hosts_blacklist=None):
+                 url_hosts_blacklist=None, url_hosts_whitelist=None):
         if url is None:
             raise Exception('URL must be specified.')
 
@@ -194,7 +202,8 @@ def __init__(self, url=None, method=None, body='', params=None, headers=None, co
         self.verify = verify
         self.username = username
         self.password = password
-        self.hosts_blacklist = hosts_blacklist or []
+        self.url_hosts_blacklist = url_hosts_blacklist or []
+        self.url_hosts_whitelist = url_hosts_whitelist or []
 
     def run(self):
         results = {}
@@ -207,6 +216,11 @@ def run(self):
         if is_url_blacklisted:
             raise ValueError('URL "%s" is blacklisted' % (self.url))
 
+        is_url_whitelisted = self._is_url_whitelisted(url=self.url)
+
+        if not is_url_whitelisted:
+            raise ValueError('URL "%s" is not whitelisted' % (self.url))
+
         try:
             if json_content:
                 # cast params (body) to dict
@@ -316,24 +330,46 @@ def _cast_object(self, value):
 
     def _is_url_blacklisted(self, url):
         """
-        Verify if the provided URL is blacklisted via hosts_blacklist runner parameter.
+        Verify if the provided URL is blacklisted via url_hosts_blacklist runner parameter.
         """
-        if not self.hosts_blacklist:
+        if not self.url_hosts_blacklist:
             # Blacklist is empty
             return False
 
+        host = self._get_host_from_url(url=url)
+
+        if host in self.url_hosts_blacklist:
+            return True
+
+        return False
+
+    def _is_url_whitelisted(self, url):
+        """
+        Verify if the provided URL is whitelisted via url_hosts_whitelist runner parameter.
+        """
+        if not self.url_hosts_whitelist:
+            return True
+
+        host = self._get_host_from_url(url=url)
+
+        if host in self.url_hosts_whitelist:
+            return True
+
+        return False
+
+    def _get_host_from_url(self, url):
+        """
+        Return sanitized host (netloc) value from the provided url.
+        """
         parsed = urlparse.urlparse(url)
 
-        # Remove the port and []
+        # Remove port and []
         host = parsed.netloc.replace('[', '').replace(']', '')
 
         if parsed.port is not None:
             host = host.replace(':%s' % (parsed.port), '')
 
-        if host in self.hosts_blacklist:
-            return True
-
-        return False
+        return host
 
 
 def get_runner():

contrib/runners/http_runner/http_runner/runner.yaml

@@ -36,9 +36,18 @@
         CA bundle which comes from Mozilla. Verification using a custom CA bundle
         is not yet supported. Set to False to skip verification.
       type: boolean
-    hosts_blacklist:
+    url_hosts_blacklist:
       description: Optional list of hosts (network locations) to blacklist (e.g. example.com,
-        127.0.0.1, etc.)
+        127.0.0.1, ::1, etc.). If action will try to access that endpoint, an exception will be
+        thrown and action will be marked as failed.
+      required: false
+      type: array
+      items:
+        type: string
+    url_hosts_whitelist:
+      description: Optional list of hosts (network locations) to whitelist (e.g. example.com,
+        127.0.0.1, ::1, etc.). If specified, actions will only be able to hit hosts on this
+        whitelist.
       required: false
       type: array
       items:

contrib/runners/http_runner/tests/unit/test_http_runner.py

@@ -216,7 +216,7 @@ def test_http_unicode_body_data(self, mock_requests):
         self.assertEqual(call_kwargs['data'], expected_data)
 
     @mock.patch('http_runner.http_runner.requests')
-    def test_blacklisted_url_netloc(self, mock_requests):
+    def test_blacklisted_url_url_hosts_blacklist_runner_parameter(self, mock_requests):
         # Black list is empty
         self.assertEqual(mock_requests.request.call_count, 0)
 
@@ -227,7 +227,7 @@ def test_blacklisted_url_netloc(self, mock_requests):
         self.assertEqual(mock_requests.request.call_count, 1)
 
         # Blacklist is set
-        hosts_blacklist = [
+        url_hosts_blacklist = [
             'example.com',
             '127.0.0.1',
             '::1',
@@ -250,7 +250,7 @@ def test_blacklisted_url_netloc(self, mock_requests):
 
         for url in urls:
             expected_msg = r'URL "%s" is blacklisted' % (re.escape(url))
-            client = HTTPClient(url=url, method='GET', hosts_blacklist=hosts_blacklist)
+            client = HTTPClient(url=url, method='GET', url_hosts_blacklist=url_hosts_blacklist)
             self.assertRaisesRegexp(ValueError, expected_msg, client.run)
 
         # Non blacklisted URLs
@@ -265,7 +265,62 @@ def test_blacklisted_url_netloc(self, mock_requests):
 
             self.assertEqual(mock_requests.request.call_count, 0)
 
-            client = HTTPClient(url=url, method='GET')
+            client = HTTPClient(url=url, method='GET', url_hosts_blacklist=url_hosts_blacklist)
+            client.run()
+
+            self.assertEqual(mock_requests.request.call_count, 1)
+
+    @mock.patch('http_runner.http_runner.requests')
+    def test_whitelisted_url_url_hosts_whitelist_runner_parameter(self, mock_requests):
+        # Whitelist is empty
+        self.assertEqual(mock_requests.request.call_count, 0)
+
+        url = 'http://www.example.com'
+        client = HTTPClient(url=url, method='GET')
+        client.run()
+
+        self.assertEqual(mock_requests.request.call_count, 1)
+
+        # Whitelist is set
+        url_hosts_whitelist = [
+            'example.com',
+            '127.0.0.1',
+            '::1',
+            '2001:0db8:85a3:0000:0000:8a2e:0370:7334'
+        ]
+
+        # Non whitelisted urls
+        urls = [
+            'https://www.google.com',
+            'https://www.example2.com',
+            'http://127.0.0.2'
+        ]
+
+        for url in urls:
+            expected_msg = r'URL "%s" is not whitelisted' % (re.escape(url))
+            client = HTTPClient(url=url, method='GET', url_hosts_whitelist=url_hosts_whitelist)
+            self.assertRaisesRegexp(ValueError, expected_msg, client.run)
+
+        # Whitelisted URLS
+        urls = [
+            'https://example.com',
+            'http://example.com',
+            'http://example.com:81',
+            'http://example.com:80',
+            'http://example.com:9000',
+            'http://[::1]:80/',
+            'http://[::1]',
+            'http://[::1]:9000',
+            'http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]',
+            'https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8000'
+        ]
+
+        for url in urls:
+            mock_requests.request.reset_mock()
+
+            self.assertEqual(mock_requests.request.call_count, 0)
+
+            client = HTTPClient(url=url, method='GET', url_hosts_whitelist=url_hosts_whitelist)
             client.run()
 
             self.assertEqual(mock_requests.request.call_count, 1)