feat: allow passing serializer on cache creation and implement JSONSerializer

CHANGES.rst

@@ -4,6 +4,8 @@ Version 0.15.0
 Unreleased
 
 - Make ``SimpleCache`` thread-safe using a ``threading.RLock``. :issue:`446`
+- Allow passing a custom ``serializer`` on cache creation and add ``JSONSerializer``. :pr:`462`
+
 
 Version 0.14.0
 --------------

src/cachelib/dynamodb.py

@@ -2,6 +2,7 @@
 import typing as _t
 
 from cachelib.base import BaseCache
+from cachelib.serializers import BaseSerializer
 from cachelib.serializers import DynamoDbSerializer
 
 CREATED_AT_FIELD = "created_at"
@@ -33,10 +34,12 @@ class DynamoDbCache(BaseCache):
                                   this as the TTL field, then DynamoDB will
                                   automatically delete expired entries.
     :param key_prefix: A prefix that should be added to all keys.
-
+    :param serializer: An optional serializer to use instead of the default
+                       BaseSerializer. The serializer must implement the
+                       dumps and loads methods.
     """
 
-    serializer = DynamoDbSerializer()
+    serializer: BaseSerializer = DynamoDbSerializer()
 
     def __init__(
         self,
@@ -45,6 +48,7 @@ def __init__(
         key_field: _t.Optional[str] = "cache_key",
         expiration_time_field: _t.Optional[str] = "expiration_time",
         key_prefix: _t.Optional[str] = None,
+        serializer: _t.Optional[BaseSerializer] = None,
         **kwargs: _t.Any,
     ):
         super().__init__(default_timeout)
@@ -60,6 +64,8 @@ def __init__(
         self.key_prefix = key_prefix or ""
         self._dynamo = boto3.resource("dynamodb", **kwargs)
         self._attr = boto3.dynamodb.conditions.Attr
+        if serializer is not None:
+            self.serializer = serializer
 
         try:
             self._table = self._dynamo.Table(table_name)
@@ -134,7 +140,7 @@ def get(self, key: str) -> _t.Any:
         cache_item = self._get_item(self.key_prefix + key)
         if cache_item:
             response = cache_item[RESPONSE_FIELD]
-            value = self.serializer.loads(response)
+            value = self.serializer.loads(response.value)
             return value
         return None
 

src/cachelib/file.py

@@ -13,6 +13,7 @@
 from time import time
 
 from cachelib.base import BaseCache
+from cachelib.serializers import BaseSerializer
 from cachelib.serializers import FileSystemSerializer
 
 
@@ -42,6 +43,9 @@ class FileSystemCache(BaseCache):
                         generate the filename for cached results.
                         Default is lazy loaded and can be overridden by
                         setting  `_default_hash_method`
+    :param serializer: An optional serializer to use instead of the default
+                       BaseSerializer. The serializer must implement the
+                       dump and load methods.
     """
 
     #: used for temporary files by the FileSystemCache
@@ -51,7 +55,7 @@ class FileSystemCache(BaseCache):
     #: default file name hashing method
     _default_hash_method = staticmethod(_lazy_md5)
 
-    serializer = FileSystemSerializer()
+    serializer: BaseSerializer = FileSystemSerializer()
 
     def __init__(
         self,
@@ -60,6 +64,7 @@ def __init__(
         default_timeout: int = 300,
         mode: _t.Optional[int] = None,
         hash_method: _t.Any = None,
+        serializer: _t.Optional[BaseSerializer] = None,
     ):
         BaseCache.__init__(self, default_timeout)
         self._path = cache_dir
@@ -76,6 +81,9 @@ def __init__(
         if self._mode is None:
             self._mode = self._get_compatible_platform_mode()
 
+        if serializer is not None:
+            self.serializer = serializer
+
         try:
             os.makedirs(self._path)
         except OSError as ex:

src/cachelib/memcached.py

@@ -29,6 +29,8 @@ class MemcachedCache(BaseCache):
     the keys in the same format as passed.  Furthermore all get methods
     silently ignore key errors to not cause problems when untrusted user data
     is passed to the get methods which is often the case in web applications.
+    This cache doesn't have a serializer since the underlying memcached client
+    libraries handle serialization internally."
 
     :param servers: a list or tuple of server addresses or alternatively
                     a :class:`memcache.Client` or a compatible client.

src/cachelib/mongodb.py

@@ -1,9 +1,9 @@
 import datetime
-import logging
 import typing as _t
 
 from cachelib.base import BaseCache
 from cachelib.serializers import BaseSerializer
+from cachelib.serializers import MongoDbSerializer
 
 
 class MongoDbCache(BaseCache):
@@ -19,10 +19,12 @@ class MongoDbCache(BaseCache):
     :param default_timeout: Set the timeout in seconds after which cache entries
                             expire
     :param key_prefix: A prefix that should be added to all keys.
-
+    :param serializer: An optional serializer to use instead of the default
+                       BaseSerializer. The serializer must implement the
+                       dumps and loads methods.
     """
 
-    serializer = BaseSerializer()
+    serializer: BaseSerializer = MongoDbSerializer()
 
     def __init__(
         self,
@@ -31,25 +33,29 @@ def __init__(
         collection: _t.Optional[str] = "cache-collection",
         default_timeout: int = 300,
         key_prefix: _t.Optional[str] = None,
+        serializer: _t.Optional[BaseSerializer] = None,
         **kwargs: _t.Any,
     ):
         super().__init__(default_timeout)
         try:
             import pymongo
-        except ImportError:
-            logging.warning("no pymongo module found")
+        except ImportError as err:
+            raise RuntimeError("no pymongo module found") from err
 
         if client is None or isinstance(client, str):
-            client = pymongo.MongoClient(host=client)
+            client = pymongo.MongoClient(host=client, **kwargs)
         self.client = client[db][collection]
         index_info = self.client.index_information()
         all_keys = {
             subkey[0] for value in index_info.values() for subkey in value["key"]
         }
         if "id" not in all_keys:
             self.client.create_index("id", unique=True)
+
         self.key_prefix = key_prefix or ""
         self.collection = collection
+        if serializer is not None:
+            self.serializer = serializer
 
     def _utcnow(self) -> _t.Any:
         """Return a tz-aware UTC datetime representing the current time"""

src/cachelib/redis.py

@@ -1,6 +1,7 @@
 import typing as _t
 
 from cachelib.redis_base import BaseRedisCache
+from cachelib.serializers import BaseSerializer
 from cachelib.serializers import RedisSerializer
 
 
@@ -22,6 +23,9 @@ class RedisCache(BaseRedisCache):
                             specified on :meth:`~BaseCache.set`. A timeout of
                             0 indicates that the cache never expires.
     :param key_prefix: A prefix that should be added to all keys.
+    :param serializer: An optional serializer to use instead of the default
+                       BaseSerializer. The serializer must implement the
+                       dumps and loads methods.
 
     Any additional keyword arguments will be passed to ``redis.Redis``.
     """
@@ -36,6 +40,7 @@ def __init__(
         db: int = 0,
         default_timeout: int = 300,
         key_prefix: _t.Optional[_t.Union[str, _t.Callable[[], str]]] = None,
+        serializer: _t.Optional[BaseSerializer] = None,
         **kwargs: _t.Any,
     ):
         if host is None:
@@ -52,4 +57,4 @@ def __init__(
             )
         else:
             client = host
-        super().__init__(client, default_timeout, key_prefix)
+        super().__init__(client, default_timeout, key_prefix, serializer)

src/cachelib/redis_base.py

@@ -2,6 +2,7 @@
 
 from cachelib.base import BaseCache
 from cachelib.serializers import BaseRedisSerializer
+from cachelib.serializers import BaseSerializer
 
 
 class BaseRedisCache(BaseCache):
@@ -15,21 +16,27 @@ class BaseRedisCache(BaseCache):
                             specified on :meth:`~BaseCache.set`. A timeout of
                             0 indicates that the cache never expires.
     :param key_prefix: A prefix that should be added to all keys.
+    :param serializer: An optional serializer to use instead of the default
+                       BaseSerializer. The serializer must implement the
+                       dumps and loads methods.
     """
 
     _read_client: _t.Any = None
     _write_client: _t.Any = None
-    serializer = BaseRedisSerializer()
+    serializer: BaseSerializer = BaseRedisSerializer()
 
     def __init__(
         self,
         client: _t.Any,
         default_timeout: int = 300,
         key_prefix: _t.Optional[_t.Union[str, _t.Callable[[], str]]] = None,
+        serializer: _t.Optional[BaseSerializer] = None,
     ):
         BaseCache.__init__(self, default_timeout)
         self._read_client = self._write_client = client
         self.key_prefix = key_prefix or ""
+        if serializer is not None:
+            self.serializer = serializer
 
     def _get_prefix(self) -> str:
         return (

src/cachelib/serializers.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import pickle
 import typing as _t
@@ -11,10 +12,8 @@ class BaseSerializer:
     used only by FileSystemCache which dumps/loads to/from a file stream.
     """
 
-    def _warn(self, e: pickle.PickleError) -> None:
-        logging.warning(
-            f"An exception has been raised during a pickling operation: {e}"
-        )
+    def _warn(self, e: Exception) -> None:
+        logging.warning(f"An exception has been raised during an operation: {e}")
 
     def dump(
         self, value: int, f: _t.IO[bytes], protocol: int = pickle.HIGHEST_PROTOCOL
@@ -119,9 +118,58 @@ class ValkeySerializer(BaseRedisSerializer):
 class DynamoDbSerializer(RedisSerializer):
     """Default serializer for DynamoDbCache."""
 
-    def loads(self, value: _t.Any) -> _t.Any:
-        """The reversal of :meth:`dump_object`. This might be called with
-        None.
-        """
-        value = value.value
-        return super().loads(value)
+    pass
+
+
+class MongoDbSerializer(BaseSerializer):
+    """Default serializer for MongoDbCache."""
+
+    pass
+
+
+class JSONSerializer(BaseSerializer):
+    """Generic JSON serializer for caches.
+
+    Use this serializer if you want to serialize to JSON instead of pickle.
+    Note that JSON does not support serialization of Python bytes objects.
+    If you need to cache bytes, use the default pickle-based serializer.
+    This serializer is not used by default.
+    """
+
+    def dump(
+        self, value: _t.Any, f: _t.IO[bytes], *args: _t.Any, **kwargs: _t.Any
+    ) -> None:
+        try:
+            f.write(json.dumps(value, *args, **kwargs).encode())
+        except (TypeError, ValueError) as e:
+            self._warn(e)
+
+    def load(self, f: _t.IO[bytes], *args: _t.Any, **kwargs: _t.Any) -> _t.Any:
+        try:
+            data = json.loads(f.read(), *args, **kwargs)
+        except (TypeError, ValueError) as e:
+            self._warn(e)
+            return None
+        else:
+            return data
+
+    def dumps(
+        self, value: _t.Any, *args: _t.Any, **kwargs: _t.Any
+    ) -> _t.Optional[bytes]:
+        try:
+            serialized = json.dumps(value, *args, **kwargs).encode()
+        except (TypeError, ValueError) as e:
+            self._warn(e)
+            return None
+        return serialized
+
+    def loads(
+        self, bvalue: _t.Union[str, bytes, bytearray], *args: _t.Any, **kwargs: _t.Any
+    ) -> _t.Any:
+        try:
+            data = json.loads(bvalue, *args, **kwargs)
+        except (TypeError, ValueError) as e:
+            self._warn(e)
+            return None
+        else:
+            return data

src/cachelib/simple.py

@@ -3,6 +3,7 @@
 from time import time
 
 from cachelib.base import BaseCache
+from cachelib.serializers import BaseSerializer
 from cachelib.serializers import SimpleSerializer
 
 
@@ -16,19 +17,25 @@ class SimpleCache(BaseCache):
     :param default_timeout: the default timeout that is used if no timeout is
                             specified on :meth:`~BaseCache.set`. A timeout of
                             0 indicates that the cache never expires.
+    :param serializer: An optional serializer to use instead of the default
+                       BaseSerializer. The serializer must implement the
+                       dumps and loads methods.
     """
 
-    serializer = SimpleSerializer()
+    serializer: BaseSerializer = SimpleSerializer()
 
     def __init__(
         self,
         threshold: int = 500,
         default_timeout: int = 300,
+        serializer: _t.Optional[BaseSerializer] = None,
     ):
         BaseCache.__init__(self, default_timeout)
         self._cache: _t.Dict[str, _t.Any] = {}
         self._threshold = threshold or 500  # threshold = 0
         self._lock = threading.RLock()
+        if serializer is not None:
+            self.serializer = serializer
 
     def _over_threshold(self) -> bool:
         return len(self._cache) > self._threshold