Get rid if automatic auto conversion / de-referencing of embedded fields

inside the base mongoengine document class.

This process is very slow and adds a tons of overhead since it calls
"to_python()" on every model field even if the field doesn't need to be
referenced.

With this approach we manually de-reference only the fields which need
to be dereferenced / converted.

The most ideal and faster approach would be to work directly with
dictionaries as returned by pymongo (this way we could avoid the whole
overhead), but this is a compromise.

Author: Kami

st2common/st2common/models/db/__init__.py

@@ -487,7 +487,8 @@ def insert(self, instance):
     def add_or_update(self, instance, validate=True, undo_dict_field_escape=True):
         instance.save(validate=validate)
         if undo_dict_field_escape:
-            return self._undo_dict_field_escape(instance)
+            instance = self._undo_dict_field_escape(instance)
+        instance.id = str(instance.id)
         return instance
 
     def update(self, instance, **kwargs):
@@ -609,8 +610,11 @@ def _process_as_pymongo_queryset(self, queryset, as_pymongo=False):
         for item in result:
             if '_id' in item:
                 item['id'] = str(item.pop('_id'))
-            # TODO: Also avoid expensive auto conversion since it's not needed in most cases
-            model_db = self.model(__auto_convert=True, **item)
+            # NOTE: Disabling auto_convert speeds it up by 50%
+            # Derefernces only need to happen where we use EmbeddedDocumentField which is only in
+            # a few places
+            model_db = self.model(__auto_convert=False, **item)
+            model_db.id = str(model_db.id)
             models_result.append(model_db)
 
         return models_result

st2common/st2common/models/db/action.py

@@ -94,6 +94,15 @@ def __init__(self, *args, **values):
         self.ref = self.get_reference().ref
         self.uid = self.get_uid()
 
+        # Manualy de-reference EmbeddedDocumentField fields to avoid overhead of de-referencing all
+        # the fields inside the base Document class constructor when __auto_convert is True.
+        # This approach means we need to update this code each time we add new
+        # EmbeddedDocumentField (which we should avoid anyway for performance reasons)
+        stormbase.TagsMixin.__init__(self)
+
+        if self.notify:
+            self.notify = self._fields['notify'].to_python(self.notify)
+
     def is_workflow(self):
         """
         Return True if this action is a workflow, False otherwise.

st2common/st2common/models/db/liveaction.py

@@ -87,6 +87,16 @@ class LiveActionDB(stormbase.StormFoundationDB):
         ]
     }
 
+    def __init__(self, *args, **kwargs):
+        super(LiveActionDB, self).__init__(*args, **kwargs)
+
+        # Manualy de-reference EmbeddedDocumentField fields to avoid overhead of de-referencing all
+        # the fields inside the base Document class constructor when __auto_convert is True.
+        # This approach means we need to update this code each time we add new
+        # EmbeddedDocumentField (which we should avoid anyway for performance reasons)
+        if self.notify:
+            self.notify = self._fields['notify'].to_python(self.notify)
+
     def mask_secrets(self, value):
         from st2common.util import action_db
 

st2common/st2common/models/db/rule.py

@@ -159,6 +159,18 @@ def __init__(self, *args, **values):
         self.ref = self.get_reference().ref
         self.uid = self.get_uid()
 
+        # Manualy de-reference EmbeddedDocumentField fields to avoid overhead of de-referencing all
+        # the fields inside the base Document class constructor when __auto_convert is True.
+        # This approach means we need to update this code each time we add new
+        # EmbeddedDocumentField (which we should avoid anyway for performance reasons)
+        stormbase.TagsMixin.__init__(self)
+
+        if self.type:
+            self.type = self._fields['type'].to_python(self.type)
+
+        if self.action:
+            self.action = self._fields['action'].to_python(self.action)
+
 
 rule_access = MongoDBAccess(RuleDB)
 rule_type_access = MongoDBAccess(RuleTypeDB)

st2common/st2common/models/db/rule_enforcement.py

@@ -80,6 +80,15 @@ class RuleEnforcementDB(stormbase.StormFoundationDB, stormbase.TagsMixin):
     def __init__(self, *args, **values):
         super(RuleEnforcementDB, self).__init__(*args, **values)
 
+        # Manualy de-reference EmbeddedDocumentField fields to avoid overhead of de-referencing all
+        # the fields inside the base Document class constructor when __auto_convert is True.
+        # This approach means we need to update this code each time we add new
+        # EmbeddedDocumentField (which we should avoid anyway for performance reasons)
+        stormbase.TagsMixin.__init__(self)
+
+        if self.rule:
+            self.rule = self._fields['rule'].to_python(self.rule)
+
         # Set status to succeeded for old / existing RuleEnforcementDB which predate status field
         status = getattr(self, 'status', None)
         failure_reason = getattr(self, 'failure_reason', None)

st2common/st2common/models/db/stormbase.py

@@ -165,6 +165,12 @@ class TagsMixin(object):
     """
     tags = me.ListField(field=me.EmbeddedDocumentField(TagField))
 
+    def __init__(self):
+        # Manualy de-reference EmbeddedDocumentField fields to avoid overhead of de-referencing all
+        # the fields inside the base Document class constructor when __auto_convert is True
+        if self.tags:
+            self.tags = self._fields['tags'].to_python(self.tags)
+
     @classmethod
     def get_indexes(cls):
         return ['tags.name', 'tags.value']

st2common/st2common/models/db/trace.py

@@ -94,6 +94,22 @@ def __init__(self, *args, **values):
         super(TraceDB, self).__init__(*args, **values)
         self.uid = self.get_uid()
 
+        # Manualy de-reference EmbeddedDocumentField fields to avoid overhead of de-referencing all
+        # the fields inside the base Document class constructor when __auto_convert is True.
+        # This approach means we need to update this code each time we add new
+        # EmbeddedDocumentField (which we should avoid anyway for performance reasons)
+        if self.trigger_instances:
+            self.trigger_instances = \
+                self._fields['trigger_instances'].to_python(self.trigger_instances)
+
+        if self.rules:
+            self.rules = \
+                self._fields['rules'].to_python(self.rules)
+
+        if self.action_executions:
+            self.action_executions = \
+                self._fields['action_executions'].to_python(self.action_executions)
+
     def get_uid(self):
         parts = []
         parts.append(self.RESOURCE_TYPE)

st2common/st2common/models/db/trigger.py

@@ -64,6 +64,12 @@ def __init__(self, *args, **values):
         # pylint: disable=no-member
         self.uid = self.get_uid()
 
+        # Manualy de-reference EmbeddedDocumentField fields to avoid overhead of de-referencing all
+        # the fields inside the base Document class constructor when __auto_convert is True.
+        # This approach means we need to update this code each time we add new
+        # EmbeddedDocumentField (which we should avoid anyway for performance reasons)
+        stormbase.TagsMixin.__init__(self)
+
 
 class TriggerDB(stormbase.StormBaseDB, stormbase.ContentPackResourceMixin,
                 stormbase.UIDFieldMixin):