We can (now?) use value_to_string and avoid problems with custom fields.

.gitignore

@@ -1,2 +1,4 @@
 *.pyc
 docs/_build
+/.eggs
+/*.egg-info

COPYING

@@ -1,3 +1,4 @@
+Copyright © 2016 Chris Lamb <[email protected]>
 Copyright © 2010, 2011 UUMC Ltd. <[email protected]>
 All rights reserved.
 

README.md

@@ -0,0 +1,42 @@
+# django-cache-toolbox
+
+_Non-magical object caching for Django._
+
+Another caching framework for Django that does not do any magic behind your
+back, saving brain cycles when debugging as well as sticking to Django
+principles.
+
+## Installation
+
+From [PyPI](https://pypi.org/project/django-cache-toolbox/):
+```
+pip install django-cache-toolbox
+```
+
+## Basic Usage
+
+``` python
+from cache_toolbox import cache_model, cache_relation
+from django.db import models
+
+class Foo(models.Model):
+    ...
+
+class Bazz(models.Model):
+    foo = models.OneToOneField(Foo, related_name='bazz', primary_key=True)
+    ...
+
+# Prepare caching of a model
+cache_model(Foo)
+
+# Prepare caching of a relation
+cache_relation(Foo.bazz)
+
+# Fetch the cached version of a model
+foo = Foo.get_cached(pk=42)
+
+# Load a cached relation
+print(foo.bazz_cache)
+```
+
+See the module docstrings for further details.

cache_toolbox/app_settings.py

@@ -3,6 +3,6 @@
 # Default cache timeout
 CACHE_TOOLBOX_DEFAULT_TIMEOUT = getattr(
     settings,
-    'CACHE_TOOLBOX_DEFAULT_TIMEOUT',
+    "CACHE_TOOLBOX_DEFAULT_TIMEOUT",
     60 * 60 * 24 * 7,
 )

cache_toolbox/core.py

@@ -8,16 +8,80 @@
 
 """
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
 from django.core.cache import cache
-from django.db import DEFAULT_DB_ALIAS
+from django.db import DEFAULT_DB_ALIAS, transaction
 
 from . import app_settings
 
 
-def get_instance(
-    model, instance_or_pk,
-    timeout=None, using=None, create=False, defaults=None
-):
+CACHE_FORMAT_VERSION = 2
+
+
+def setattrdefault(obj, name, default):
+    try:
+        return getattr(obj, name)
+    except AttributeError:
+        setattr(obj, name, default)
+        return default
+
+
+def get_related_name(descriptor):
+    return "%s_cache" % descriptor.related.field.related_query_name()
+
+
+def get_related_cache_name(related_name: str) -> str:
+    return "_%s_cache" % related_name
+
+
+def add_always_fetch_relation(descriptor):
+    setattrdefault(
+        descriptor.related.model,
+        "_cache_fetch_related",
+        [],
+    ).append(descriptor)
+
+
+def serialise(instance):
+    data = {}
+    for field in instance._meta.fields:
+        # Harmless to save, but saves space in the dictionary - we already know
+        # the primary key when we lookup
+        if field.primary_key:
+            continue
+
+        # We also don't want to save any virtual fields.
+        if not field.concrete:
+            continue
+
+        data[field.attname] = getattr(instance, field.attname)
+
+    # Encode through Pickle, since that allows overriding and covers (most)
+    # Python types we'd want to serialise.
+    return pickle.dumps(data, protocol=-1)
+
+
+def deserialise(model, data, pk, using):
+    # Try and construct instance from dictionary
+    instance = model(pk=pk, **pickle.loads(data))
+
+    # Ensure instance knows that it already exists in the database,
+    # otherwise we will fail any uniqueness checks when saving the
+    # instance.
+    instance._state.adding = False
+
+    # Specify database so that instance is setup correctly. We don't
+    # namespace cached objects by their origin database, however.
+    instance._state.db = using or DEFAULT_DB_ALIAS
+
+    return instance
+
+
+def get_instance(model, instance_or_pk, timeout=None, using=None):
     """
     Returns the ``model`` instance with a primary key of ``instance_or_pk``.
 
@@ -27,9 +91,6 @@ def get_instance(
     If omitted, the timeout value defaults to
     ``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero).
 
-    If ``create`` is True, we are going to create the instance in case that it
-    was not found.
-
     Example::
 
         >>> get_instance(User, 1) # Cache miss
@@ -40,76 +101,114 @@ def get_instance(
         True
     """
 
-    pk = getattr(instance_or_pk, 'pk', instance_or_pk)
-    key = instance_key(model, instance_or_pk)
-    data = cache.get(key)
+    pk = getattr(instance_or_pk, "pk", instance_or_pk)
 
-    if data is not None:
-        try:
-            # Try and construct instance from dictionary
-            instance = model(pk=pk, **data)
-
-            # Ensure instance knows that it already exists in the database,
-            # otherwise we will fail any uniqueness checks when saving the
-            # instance.
-            instance._state.adding = False
+    primary_model = model
+    descriptors = getattr(primary_model, "_cache_fetch_related", ())
+    models = [model, *(d.related.field.model for d in descriptors)]
+    # Note: we're assuming that the relations are primary key foreign keys, and
+    # so all have the same primary key. This matches the assumption which
+    # `cache_relation` makes.
+    keys_to_models = {instance_key(model, instance_or_pk): model for model in models}
 
-            # Specify database so that instance is setup correctly. We don't
-            # namespace cached objects by their origin database, however.
-            instance._state.db = using or DEFAULT_DB_ALIAS
+    data_map = cache.get_many(tuple(keys_to_models.keys()))
+    instance_map = {}
 
-            return instance
+    if data_map.keys() == keys_to_models.keys():
+        try:
+            for key, data in data_map.items():
+                model = keys_to_models[key]
+                instance_map[key] = deserialise(model, data, pk, using)
         except:
             # Error when deserialising - remove from the cache; we will
             # fallback and return the underlying instance
-            cache.delete(key)
+            cache.delete_many(tuple(keys_to_models.keys()))
 
-    # Use the default manager so we are never filtered by a .get_query_set()
-    queryset = model._default_manager.using(using)
-    if create:
-        # It's possible that the related object didn't exist yet
-        instance, _ = queryset.get_or_create(pk=pk, defaults=defaults or {})
-    else:
-        instance = queryset.get(pk=pk)
+        else:
+            key = instance_key(primary_model, instance_or_pk)
+            primary_instance = instance_map[key]
 
-    data = {}
-    for field in instance._meta.fields:
-        # Harmless to save, but saves space in the dictionary - we already know
-        # the primary key when we lookup
-        if field.primary_key:
+            for descriptor in descriptors:
+                related_instance = instance_map[
+                    instance_key(
+                        descriptor.related.field.model,
+                        instance_or_pk,
+                    )
+                ]
+                related_cache_name = get_related_cache_name(
+                    get_related_name(descriptor),
+                )
+                setattr(primary_instance, related_cache_name, related_instance)
+
+            return primary_instance
+
+    related_names = [d.related.field.related_query_name() for d in descriptors]
+
+    # Use the default manager so we are never filtered by a .get_query_set()
+    queryset = primary_model._default_manager.using(using)
+    if related_names:
+        # NB: select_related without args selects all it can find, which we don't want.
+        queryset = queryset.select_related(*related_names)
+    primary_instance = queryset.get(pk=pk)
+
+    instances = [
+        primary_instance,
+        *(getattr(primary_instance, x, None) for x in related_names),
+    ]
+
+    cache_data = {}
+    for instance in instances:
+        if instance is None:
             continue
 
-        if field.get_internal_type() == 'FileField':
-            # Avoid problems with serializing FileFields
-            # by only serializing the file name
-            file = getattr(instance, field.attname)
-            data[field.attname] = file.name
-        else:
-            data[field.attname] = getattr(instance, field.attname)
+        key = instance_key(instance._meta.model, instance)
+        cache_data[key] = serialise(instance)
 
     if timeout is None:
         timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT
 
-    cache.set(key, data, timeout)
+    cache.set_many(cache_data, timeout)
 
-    return instance
+    return primary_instance
 
 
 def delete_instance(model, *instance_or_pk):
     """
     Purges the cache keys for the instances of this model.
     """
 
-    cache.delete_many([instance_key(model, x) for x in instance_or_pk])
+    # Only clear the cache when the current transaction commits.
+    # While clearing the cache earlier than that is valid, it is insufficient
+    # to ensure cache consistency. There is a possible race between two
+    # transactions as follows:
+    #
+    #   Transaction 1: modifies model (and thus clears cache)
+    #   Transaction 2: queries cache, which misses, so it populates the cache
+    #                  from the database, picking up the unmodified model
+    #   Transaction 1: commits, without further signal to the cache
+    #
+    # At this point the cache contains the _original_ value of the model, which
+    # is out of step with the database.
+    # To avoid this we delay clearing the cache until the transaction commits.
+    # While this does leave a small window after the transaction has committed
+    # but before the cache has cleared, that is better than leaving the cache
+    # incorrect until the model is next updated.
+
+    transaction.on_commit(
+        lambda: cache.delete_many(
+            [instance_key(model, x) for x in instance_or_pk],
+        )
+    )
 
 
 def instance_key(model, instance_or_pk):
     """
     Returns the cache key for this (model, instance) pair.
     """
 
-    return '%s.%s:%d' % (
+    return "cache.%d:%s.%s:%s" % (
+        CACHE_FORMAT_VERSION,
         model._meta.app_label,
-        model._meta.module_name,
-        getattr(instance_or_pk, 'pk', instance_or_pk),
+        model._meta.model_name,
+        getattr(instance_or_pk, "pk", instance_or_pk),
     )

cache_toolbox/middleware.py

@@ -79,19 +79,23 @@
 """
 
 from django.contrib.auth import SESSION_KEY
-from django.contrib.auth.models import User
+from django.contrib.auth import get_user_model
 from django.contrib.auth.middleware import AuthenticationMiddleware
 
 from .model import cache_model
 
+
 class CacheBackedAuthenticationMiddleware(AuthenticationMiddleware):
-    def __init__(self):
-        cache_model(User)
+    def __init__(self, get_response):
+        super(CacheBackedAuthenticationMiddleware, self).__init__(get_response)
+        cache_model(get_user_model())
 
     def process_request(self, request):
         try:
             # Try and construct a User instance from data stored in the cache
-            request.user = User.get_cached(request.session[SESSION_KEY])
-        except:
+            request.user = get_user_model().get_cached(
+                int(request.session[SESSION_KEY])
+            )
+        except Exception:
             # Fallback to constructing the User from the database.
             super(CacheBackedAuthenticationMiddleware, self).process_request(request)

cache_toolbox/model.py

@@ -58,8 +58,9 @@ class Foo(models.Model):
 
 from .core import get_instance, delete_instance
 
+
 def cache_model(model, timeout=None):
-    if hasattr(model, 'get_cached'):
+    if hasattr(model, "get_cached"):
         # Already patched
         return