Fix SSL support for MongoDB and RabbitMQ under Python 3.x

st2api/st2api/app.py

@@ -24,7 +24,6 @@
 from st2common.middleware.instrumentation import RequestInstrumentationMiddleware
 from st2common.middleware.instrumentation import ResponseInstrumentationMiddleware
 from st2common.router import Router
-from st2common.util.monkey_patch import monkey_patch
 from st2common.constants.system import VERSION_STRING
 from st2common.service_setup import setup as common_setup
 from st2common.util import spec_loader
@@ -33,16 +32,14 @@
 LOG = logging.getLogger(__name__)
 
 
-def setup_app(config={}):
+def setup_app(config=None):
+    config = config or {}
+
     LOG.info('Creating st2api: %s as OpenAPI app.', VERSION_STRING)
 
     is_gunicorn = config.get('is_gunicorn', False)
     if is_gunicorn:
-        # Note: We need to perform monkey patching in the worker. If we do it in
-        # the master process (gunicorn_config.py), it breaks tons of things
-        # including shutdown
-        monkey_patch()
-
+        # NOTE: We only want to perform this logic in the WSGI worker
         st2api_config.register_opts()
         capabilities = {
             'name': 'api',

st2api/st2api/cmd/api.py

@@ -15,14 +15,20 @@
 import os
 import sys
 
+# NOTE: It's important that we perform monkey patch as early as possible before any other modules
+# are important, otherwise SSL support for MongoDB won't work.
+# See https://github.com/StackStorm/st2/issues/4832 and https://github.com/gevent/gevent/issues/1016
+# for details.
+from st2common.util.monkey_patch import monkey_patch
+monkey_patch()
+
 import eventlet
 from oslo_config import cfg
 from eventlet import wsgi
 
 from st2common import log as logging
 from st2common.service_setup import setup as common_setup
 from st2common.service_setup import teardown as common_teardown
-from st2common.util.monkey_patch import monkey_patch
 from st2api import config
 config.register_opts()
 from st2api import app
@@ -33,8 +39,6 @@
     'main'
 ]
 
-monkey_patch()
-
 LOG = logging.getLogger(__name__)
 
 # How much time to give to the request in progress to finish in seconds before killing them

st2api/st2api/wsgi.py

@@ -12,8 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+WSGI entry point used by gunicorn.
+"""
+
 import os
 
+from st2common.util.monkey_patch import monkey_patch
+# Note: We need to perform monkey patching in the worker. If we do it in
+# the master process (gunicorn_config.py), it breaks tons of things
+# including shutdown
+# NOTE: It's important that we perform monkey patch as early as possible before any other modules
+# are important, otherwise SSL support for MongoDB won't work.
+# See https://github.com/StackStorm/st2/issues/4832 and https://github.com/gevent/gevent/issues/1016
+# for details.
+monkey_patch()
+
 from st2api import app
 
 config = {

st2auth/st2auth/app.py

@@ -22,7 +22,6 @@
 from st2common.middleware.instrumentation import RequestInstrumentationMiddleware
 from st2common.middleware.instrumentation import ResponseInstrumentationMiddleware
 from st2common.router import Router
-from st2common.util.monkey_patch import monkey_patch
 from st2common.constants.system import VERSION_STRING
 from st2common.service_setup import setup as common_setup
 from st2common.util import spec_loader
@@ -32,16 +31,14 @@
 LOG = logging.getLogger(__name__)
 
 
-def setup_app(config={}):
+def setup_app(config=None):
+    config = config or {}
+
     LOG.info('Creating st2auth: %s as OpenAPI app.', VERSION_STRING)
 
     is_gunicorn = config.get('is_gunicorn', False)
     if is_gunicorn:
-        # Note: We need to perform monkey patching in the worker. If we do it in
-        # the master process (gunicorn_config.py), it breaks tons of things
-        # including shutdown
-        monkey_patch()
-
+        # NOTE: We only want to perform this logic in the WSGI worker
         st2auth_config.register_opts()
         capabilities = {
             'name': 'auth',

st2auth/st2auth/wsgi.py

@@ -14,6 +14,16 @@
 
 import os
 
+from st2common.util.monkey_patch import monkey_patch
+# Note: We need to perform monkey patching in the worker. If we do it in
+# the master process (gunicorn_config.py), it breaks tons of things
+# including shutdown
+# NOTE: It's important that we perform monkey patch as early as possible before any other modules
+# are important, otherwise SSL support for MongoDB won't work.
+# See https://github.com/StackStorm/st2/issues/4832 and https://github.com/gevent/gevent/issues/1016
+# for details.
+monkey_patch()
+
 from st2auth import app
 
 config = {

st2common/st2common/models/db/__init__.py

@@ -25,6 +25,7 @@
 from pymongo import uri_parser
 from pymongo.errors import OperationFailure
 from pymongo.errors import ConnectionFailure
+from pymongo.errors import ServerSelectionTimeoutError
 
 from st2common import log as logging
 from st2common.util import isotime
@@ -122,9 +123,13 @@ def _db_connect(db_name, db_host, db_port, username=None, password=None,
                                  authentication_mechanism=authentication_mechanism,
                                  ssl_match_hostname=ssl_match_hostname)
 
+    # NOTE: We intentionally set "serverSelectionTimeoutMS" to 5 seconds. By default it's set to
+    # 30 seconds, which means it will block up to 30 seconds and fail if there are any SSL related
+    # errors
     connection = mongoengine.connection.connect(db_name, host=db_host,
                                                 port=db_port, tz_aware=True,
                                                 username=username, password=password,
+                                                serverSelectionTimeoutMS=5000,
                                                 **ssl_kwargs)
 
     # NOTE: Since pymongo 3.0, connect() method is lazy and not blocking (always returns success)
@@ -134,7 +139,10 @@ def _db_connect(db_name, db_host, db_port, username=None, password=None,
     try:
         # The ismaster command is cheap and does not require auth
         connection.admin.command('ismaster')
-    except ConnectionFailure as e:
+    except (ConnectionFailure, ServerSelectionTimeoutError) as e:
+        # NOTE: ServerSelectionTimeoutError can also be thrown if SSLHandShake fails in the server
+        # Sadly the client doesn't include more information about the error so in such scenarios
+        # user needs to check MongoDB server log
         LOG.error('Failed to connect to database "%s" @ "%s" as user "%s": %s' %
                   (db_name, host_string, str(username_string), six.text_type(e)))
         raise e