From 71ddde7ef5b4c4f4f0c6845351bb5981a1f81a7b Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 28 Feb 2014 10:45:53 +0000 Subject: [PATCH 01/79] We can (now?) use value_to_string and avoid problems with custom fields. Signed-off-by: Chris Lamb --- cache_toolbox/core.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 8abc16c..098ebfa 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -67,13 +67,8 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): if field.primary_key: continue - if field.get_internal_type() == 'FileField': - # Avoid problems with serializing FileFields - # by only serializing the file name - file = getattr(instance, field.attname) - data[field.attname] = file.name - else: - data[field.attname] = getattr(instance, field.attname) + # Serialise the instance using the Field's own serialisation routines. + data[field.attname] = field.value_to_string(instance) if timeout is None: timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT From 7ef7d954f647f625fbc541975891a6102056273a Mon Sep 17 00:00:00 2001 From: Dan Palmer Date: Tue, 2 Jun 2015 11:33:41 +0100 Subject: [PATCH 02/79] Support Django >= 1.6 --- cache_toolbox/core.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 5f7b576..df78f7b 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -103,8 +103,15 @@ def instance_key(model, instance_or_pk): Returns the cache key for this (model, instance) pair. """ + try: + model_name = model._meta.model_name + except AttributeError: + # Django version <1.6 + model_name = model._meta.module_name + return '%s.%s:%d' % ( model._meta.app_label, model._meta.module_name, + model_name, getattr(instance_or_pk, 'pk', instance_or_pk), ) From c09df44be67077190602f9a3ddb54ea32ce21a34 Mon Sep 17 00:00:00 2001 From: Dan Palmer Date: Tue, 2 Jun 2015 11:35:11 +0100 Subject: [PATCH 03/79] This wasn't supposed to be added --- cache_toolbox/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index df78f7b..341cf3a 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -111,7 +111,6 @@ def instance_key(model, instance_or_pk): return '%s.%s:%d' % ( model._meta.app_label, - model._meta.module_name, model_name, getattr(instance_or_pk, 'pk', instance_or_pk), ) From ee530e21978f78e684520d3353414a3249e51b48 Mon Sep 17 00:00:00 2001 From: Dan Palmer Date: Tue, 2 Jun 2015 12:23:29 +0100 Subject: [PATCH 04/79] parent_model was a private API and removed in Django 1.8 --- cache_toolbox/relation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 93685e3..9a0c467 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -108,7 +108,7 @@ def get(self): setattr(self, '_%s_cache' % related_name, instance) return instance - setattr(rel.parent_model, related_name, get) + setattr(rel.to, related_name, get) # Clearing cache @@ -122,8 +122,8 @@ def clear_pk(cls, *instances_or_pk): def clear_cache(sender, instance, *args, **kwargs): delete_instance(rel.model, instance) - setattr(rel.parent_model, '%s_clear' % related_name, clear) - setattr(rel.parent_model, '%s_clear_pk' % related_name, clear_pk) + setattr(rel.to, '%s_clear' % related_name, clear) + setattr(rel.to, '%s_clear_pk' % related_name, clear_pk) post_save.connect(clear_cache, sender=rel.model, weak=False) post_delete.connect(clear_cache, sender=rel.model, weak=False) From 3eae42451873f91c02f8719fc757e70f972f2a8d Mon Sep 17 00:00:00 2001 From: Dan Palmer Date: Tue, 2 Jun 2015 15:37:25 +0100 Subject: [PATCH 05/79] Lookup the right model --- cache_toolbox/relation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 9a0c467..a84db46 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -102,7 +102,7 @@ def get(self): pass instance = get_instance( - rel.model, self.pk, timeout, create=create, defaults=defaults + rel.field.model, self.pk, timeout, create=create, defaults=defaults ) setattr(self, '_%s_cache' % related_name, instance) From 578b2b37b44ee996890b03f8fa7860f5f0222444 Mon Sep 17 00:00:00 2001 From: Dan Palmer Date: Wed, 10 Jun 2015 15:11:01 +0100 Subject: [PATCH 06/79] Special case some field types. Otherwise we instantiate entities with a string representation --- cache_toolbox/core.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 341cf3a..fdd9067 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -79,9 +79,34 @@ def get_instance( if field.primary_key: continue - # Serialise the instance using the Field's own serialisation routines. + # By default, serialise the instance using the Field's own + # serialisation routines. + # + # It may seem odd to have a list of exceptions this way around, but + # this scheme allows custom fields to specify their own method of + # correctly serialising themselves: Field's cannot easily override + # value_from_object, but they can with value_to_string. data[field.attname] = field.value_to_string(instance) + # As special-cases: + if field.get_internal_type() in ( + # Serialise the actual value at the field's attname for any relations. + # This is to avoid caching u"None" (the string) instead of None + # (ie. NoneType) for nullable relations. + 'AutoField', + 'OneToOneField', + + # Serialise datetimes and booleans directly, otherwise we + # incorrectly try to instantiate our instance with a string + # representation. + 'BooleanField', + 'NullBooleanField', + 'TimeField', + 'DateField', + 'DateTimeField', + ): + data[field.attname] = field.value_from_object(instance) + if timeout is None: timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT From b4508fa877357d726c1d13691f708d0b06d37d6a Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sat, 8 Aug 2015 17:43:57 +0200 Subject: [PATCH 07/79] This is now model_name --- cache_toolbox/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 098ebfa..16b2cce 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -91,6 +91,6 @@ def instance_key(model, instance_or_pk): return '%s.%s:%d' % ( model._meta.app_label, - model._meta.module_name, + model._meta.model_name, getattr(instance_or_pk, 'pk', instance_or_pk), ) From b691fe7f1364c1d61721cb955400bcc2e50deeac Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 10:47:59 +0000 Subject: [PATCH 08/79] This link is no longer live --- README.rst | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 README.rst diff --git a/README.rst b/README.rst deleted file mode 100644 index cf8b23e..0000000 --- a/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -django-cache-toolbox -============================ - -Documentation: http://code.playfire.com/django-cache-toolbox/ From daf8d55cadcc19bebc827a0ddeefb9ed6280beea Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 10:48:51 +0000 Subject: [PATCH 09/79] Correct these --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6be3d7d..28d335b 100644 --- a/setup.py +++ b/setup.py @@ -6,10 +6,10 @@ name='django-cache-toolbox', description="Non-magical object caching for Django.", version='0.1', - url='http://code.playfire.com/django-cache-toolbox', + url='https://www.github.com/thread/django-cache-toolbox', - author='Playfire.com', - author_email='tech@playfire.com', + author='Thread.com', + author_email='tech@thread.com', license='BSD', packages=find_packages(), From 5338dc3ce0842e6f9159adb26e08d9c12f815445 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 10:51:05 +0000 Subject: [PATCH 10/79] Add `install_requires` --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 28d335b..df1367c 100644 --- a/setup.py +++ b/setup.py @@ -13,4 +13,8 @@ license='BSD', packages=find_packages(), + + install_requires=( + "Django>=1.8", + ) ) From 413eba1b73d25db304a8e4568bf3a468fa65edf5 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 10:52:54 +0000 Subject: [PATCH 11/79] Exclude the "tests" directory --- setup.py | 2 +- tests/__init__.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/__init__.py diff --git a/setup.py b/setup.py index df1367c..72058f7 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ author_email='tech@thread.com', license='BSD', - packages=find_packages(), + packages=find_packages(exclude=('tests',)), install_requires=( "Django>=1.8", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 66eda8461daab610d3879877dbaa5ad57f2909f6 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 10:53:13 +0000 Subject: [PATCH 12/79] Trailing comma --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 72058f7..400c152 100644 --- a/setup.py +++ b/setup.py @@ -16,5 +16,5 @@ install_requires=( "Django>=1.8", - ) + ), ) From ce4353e2803cf2af7d76596109a1a4357665ee24 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 10:53:47 +0000 Subject: [PATCH 13/79] Delete this documentation It doesn't add any value here. --- docs/Makefile | 80 ---------------------------------------------- docs/conf.py | 16 ---------- docs/index.rst | 5 --- docs/playfire.png | Bin 6037 -> 0 bytes 4 files changed, 101 deletions(-) delete mode 100644 docs/Makefile delete mode 100644 docs/conf.py delete mode 100644 docs/index.rst delete mode 100644 docs/playfire.png diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 9b6d61b..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index f8e18a5..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,16 +0,0 @@ -project = 'django-cache-toolbox' -version = '' -release = '' -copyright = '2010, 2011 UUMC Ltd.' - -html_logo = 'playfire.png' -html_theme = 'nature' -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] -html_title = "%s documentation" % project -master_doc = 'index' -exclude_trees = ['_build'] -templates_path = ['_templates'] -latex_documents = [ - ('index', '%s.tex' % project, html_title, u'Playfire', 'manual'), -] -intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index e7d90b0..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. automodule:: cache_toolbox -.. automodule:: cache_toolbox.core -.. automodule:: cache_toolbox.model -.. automodule:: cache_toolbox.relation -.. automodule:: cache_toolbox.middleware diff --git a/docs/playfire.png b/docs/playfire.png deleted file mode 100644 index 330e9c52a7e25aef3a9a89672df6216ce956360e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6037 zcmeHLcTiJl_fF^_QHped7^DkH2sMQossSQZ)D=P!NN9oN(xf8-3MeWagoRZBu~8Nf zmX4x;AcCR@h-(2AT$-$6;TMXo&Tsu@ocX^0%*@Su-+RvUobx{Kxo76RM~HTo5CLfc z5C{aZw!%BG#}f7>&&R|5r<@bN!XBgm^F4qg!wU!{F{vOE3d55Mwe}}@Qyr)zN_fys zDwYjK(w+7Idu$14GQ(e!wCbZ7<{!vLgFslLus{;omkK~VsowMe9PF>g78sOH!NFX0 zY!SABW>gy8>L8Qqc+k#?e9)I{K!F(<3Sh&~Yyy8OK!S$(`vtJjVK~?tFPeS7`WOy_ zuDJldIM{kpdu)kNGX|3i)z;L~)Y2d$bWuE|8$Xz#02Irlc%dEe78~K%GaQTt0D)*YJTx>^GZdxCV0y!m1_lOjgce*&OM~s9 z!3qxmNMRZQEQM8ybq+k0MP||i0Xic9y2?rNWCQ~^7>sR)Zje*RU$_H?*$@Prx#sf?GLXgq7%Uni@ZXVN z%xn0sLs|y{WNmibRYAXC*{dS2CCdUR@UK1A;A{J|9{o$gXjiJIBR~Sy7PcY6m!N2q zV1UM8vWN^8eN`^DeN{y$mO@6auVrnzE~|+jus&O#`q5XlMEj8fym7EF4GPtZ6zm7U z;2Q||s`8&9SopVutNdgpl>{)DUj=Za27Mdg8?MjRHLkD0t7HHj@S}c3^*QiHMSLe$ zLn4zN0ASZr^`n0`@qfs%|H$ghV!wb+RHi@chr@oetk0QZ;os%Pe!bCXvvu9f80^9) z1J(gvjJ3M{)`xCFVVZ{lBntC$-T!~P8661!zSV)vD}%{y8vlwl_b1W+liiv5*}5j<`sRK0#0{TaKHu$vhCOBY+;rj35{KO5H6 z`k%3H5dRGFLkj!(~lq`(h3|E#W`a{Z73Kji$gy8df&3H);oq6V-JKcVbX&UTZO z0T76P*&1)+6gDzlFu;7ME1n2e+0r^591_ydO{u-r=O?Pxhkl3&AIWV>{lJw~)Ot_U zDw<=LJ@gs>4)@oY85K&o_2*96Rbw7#PcNPeCCM3g7IJRu*5;1gTCQI!N)WQ)uugAH zS}EX+f9_!8qK|ujkL<9#Q`|PO>l*SD(vK=wG>3`od^EY3KQa-sKb!1a{?g~7d&IQD zj=pdiT})!h=KV3Pm#pI7&w891O({}C$716mY5LQ`qfe%)#(L65aq&E@5_O<6IHd_M za_rHrArX#G))1Kwy%VS;rmB!lTRNfd1a9YKUi-r6--dUhhPUfMS|VHp+vz;5(@i#S#Cv@7(3F(^4wky{!r=>{G>L}j;SwLMp!R>rqLHAudZoZjA zzArdmTFKe2p4|&=9x3CrCdb~+o|Sh^pqKS^@VO`GA3_#ZjEN(d+Ji-;40>l@UYEZ*JCA9)Li1Vsrs2}uYmD;^1YEMwIc0y^`&+iQe}Tq$#Uq-39p*}~%O z33Z%OFyFoV6EjY~9D$L3Bg=A@sn_hUQ1|Z$I&rqS*zBF3$z8c^Ny3W?4Vu0~g74pD z&2N4xG#|}uDXZdnyQI4JvUj)S-Lzpu)mV`%Uh_KW*CcQvMG^w{m}X&BRTM4H;!nw_ z7X@+2%T?Iuxle75VV3fHcv6};5zFBSae|V2*=D`b!~Q)%1c{|K)zq<)G!nSPT-K#- z=+ITj`{13!(zh#>!e&PmQnuXJ47wWa15Gs8U9EBcdP7pJG2bN!c=~xS;%!FdmMb5X z4@ctPy*JwDaWdZ_{SNr9gYB)>sfiDu>m8E#euWkW2e{YvdC_!LM5zLCSCQ-^sp`o` z&76Ftvmb@3crl6LhUCNAfN{>uRGs#6&-sb_#RU3c>Al^syg}62BwZ@2;FX!OM1q-g zhI#4Lz!v215&;I>=0 z=}){V70#&5$QAc&ZwbEy%~*M0OiSV?W|g0|DoQyQl7Q&ggDWn7`P-nqU2Ts9-9cHx zI_X%I)ZIGstM0qS!uFmDkeYNJE(xgc7eyp!{AT4wo;^|@iSytiu55k3TpHHtC zd}R)4`r6FGSbDR&PwB7HBeAl(Ey+=+?$ZTm307oQl=$=9S`A)FiOegup>#v7DsB>S z+=>^b&ob`vb9)`!Uz`kWJd?onBz5K85j20~wVN}-EpHa5&zRCOhoB*?-qXeXW!=)H zs2dP&vTZ`(_zBVLJ~p(1WhkotmrlV;LTA+#JNtri#_f)Uhj43L9J*I67&{dYO${VU z(2gr;KR8GD+au3@QNs|a2ICgxPc`LI4L6oNp4Bb`al8>A|FBo2Ye^U$ZWQrGAC z#Z&>H;qGxYp_=>M^3P&j@Rjqu3wF5;!v$3{OA=A^QkD~_MJ^Xq zFGi5e$F}JxB&(&MB>TDMw)O%!2A-zrBbdc&xB|47?FRR zUn!~SQMqiPAzU&47GmqxFC2KkP+Fz>G^$k0;AO#6LeSQ@U`pJjaQ^rV=L`0GL>!(V z^)gu9J_iIWO1Ca74-yl%)b2?;{-6K?w%Fn^jcF(9%eSik*>SC!4k3$~<4Z%0o;Z2m zy|(Wz@H_`7z(o1YbMt@bh-V!Y?n>gmrj!KNvI_d$a#%!%ak)a|gf>=9A@@Z?Axx3v7J%JkzO11Gz8paw;%29FL$^U03vvLs0+8_QwP?{pj z%3K|b3TAmS7`K6JHR;Gmzv=?6Xx)~ON7^do?t}+EM~K45Px2`v2WRP9bQ=v27IxKh zKAiIJa8HqscqHu?A99yFV|K2XgvBK&h{<2*w*r6Mc52^H@8dHUW~`0#hbkQ=nFYat z?=#vI55{s)&Iv zRd9RB0dEDpO)3tLo%Jcv-V(R3X77bK&J2Xs>*(j46f^MWO^6!K$S(_DIp81=A0%mM zlo69VwG;$B9ACL>;lxhvCvDty)Z(*Nxi!DsF5Q`LVek@1%2xMxrX7zhG*)#@>;%(ukBoP2Iq%){r6jKdK!Lk zfQzgu89J<8cQ+8A+_{;iwh44)PE`v4O9@RwYL^e1tW@Pw%_1hELIen7J2t`kuHuh8 z%HMk;!Bk>)ansVml)IT@=gm3i-Ab1G`*=U3zuesHqo5k#>iFMz>h!Tv{KYA!-!o1*AW{JS0L1j)QPa}uw8|@oC$(_B=wsWB+xy<)@9q3H$0ILXtO$*@0Q|zq2}Z9DPglUlca5Pmy7<& zs&`XPcGVd$9gIpb!gv<#`SXF1fhj_6By)G0RgBk;b4Cg!9!`JNm;Kd)kel7te^3!K z-4Oq_Pl!~K7t?0gR#P(%`i*v@@hLC9X8?diHuqN3o5FM(>&3H=9yXqGapSu@Gc|l_ zV*Kcd#nQ9M;=x!wYd2#lg|~(v>h!|pLJKj>%!n$iDa1zr#At}Duov$XOy_&@XVU@B zRB)>VDAMvnkG!()bd4`J>|#=6gzQ+F+KUwqK~TL)X{HP^tz-2=xwW|+{*tLj{Qm*6 C(t4Bt From 0c3fd78504b5146d9e2af1c546a70a4a9aef8ef3 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 11:22:52 +0000 Subject: [PATCH 14/79] Add a first test --- setup.py | 2 ++ tests/__init__.py | 19 +++++++++++++++++++ tests/models.py | 8 ++++++++ tests/test_cached_get.py | 15 +++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 tests/models.py create mode 100644 tests/test_cached_get.py diff --git a/setup.py b/setup.py index 400c152..c85a0b5 100644 --- a/setup.py +++ b/setup.py @@ -17,4 +17,6 @@ install_requires=( "Django>=1.8", ), + + test_suite='tests', ) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..a367231 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,19 @@ +from django import setup +from django.conf import settings +from django.core.management import call_command + +settings.configure( + DATABASES={ + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + }, + }, + INSTALLED_APPS=( + 'tests', + ), +) + +setup() + +call_command('migrate') diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 0000000..bd916a2 --- /dev/null +++ b/tests/models.py @@ -0,0 +1,8 @@ +from cache_toolbox import cache_model + +from django.db import models + +class Foo(models.Model): + bar = models.TextField() + +cache_model(Foo) diff --git a/tests/test_cached_get.py b/tests/test_cached_get.py new file mode 100644 index 0000000..e8b477a --- /dev/null +++ b/tests/test_cached_get.py @@ -0,0 +1,15 @@ +from django.test import TestCase + +from .models import Foo + +class CachedGetTest(TestCase): + def test_cached_get(self): + first_object = Foo.objects.create(bar='bees') + + # Populate the cache + Foo.get_cached(first_object.pk) + + # Get from the cache + cached_object = Foo.get_cached(first_object.pk) + + self.assertEqual(first_object.bar, cached_object.bar) From 508e071da8c177272997450c2dfcdd3c3290ffc2 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 11:34:11 +0000 Subject: [PATCH 15/79] Add a simple (currently failing)_cached relation test --- tests/models.py | 8 +++++++- tests/test_cached_relation.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tests/test_cached_relation.py diff --git a/tests/models.py b/tests/models.py index bd916a2..baa7d1b 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,8 +1,14 @@ -from cache_toolbox import cache_model +from cache_toolbox import cache_model, cache_relation from django.db import models class Foo(models.Model): bar = models.TextField() +class Bazz(models.Model): + foo = models.OneToOneField(Foo, related_name='bazz') + + value = models.IntegerField(null=True) + cache_model(Foo) +cache_relation(Foo.bazz) diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py new file mode 100644 index 0000000..fb5ee2d --- /dev/null +++ b/tests/test_cached_relation.py @@ -0,0 +1,17 @@ +from django.test import TestCase + +from .models import Foo, Bazz + +class CachedRelationTest(TestCase): + def test_cached_relation(self): + foo = Foo.objects.create(bar='bees') + + Bazz.objects.create(foo=foo, value=10) + + # Populate the cache + Foo.objects.get(pk=foo.pk).bazz_cache + + # Get from the cache + cached_object = Foo.objects.get(pk=foo.pk).bazz_cache + + self.assertEqual(cached_object.value, 10) From 76fdfd8eb485b24f121b748da79a5626d69d0b6e Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 11:40:50 +0000 Subject: [PATCH 16/79] Mark as skipped --- tests/test_cached_relation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index fb5ee2d..75e33ab 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -1,8 +1,11 @@ +from unittest import skip + from django.test import TestCase from .models import Foo, Bazz class CachedRelationTest(TestCase): + @skip("Currently broken") def test_cached_relation(self): foo = Foo.objects.create(bar='bees') From 09f410cbad7e9e4d04a57186f6691bd7c573b516 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 11:40:57 +0000 Subject: [PATCH 17/79] Add cache invalidation tests --- tests/test_cached_get.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/test_cached_get.py b/tests/test_cached_get.py index e8b477a..dccbc58 100644 --- a/tests/test_cached_get.py +++ b/tests/test_cached_get.py @@ -3,13 +3,29 @@ from .models import Foo class CachedGetTest(TestCase): - def test_cached_get(self): - first_object = Foo.objects.create(bar='bees') + def setUp(self): + self.foo = Foo.objects.create(bar='bees') + self._populate_cache() - # Populate the cache - Foo.get_cached(first_object.pk) + def _populate_cache(self): + Foo.get_cached(self.foo.pk) + def test_cached_get(self): # Get from the cache - cached_object = Foo.get_cached(first_object.pk) + cached_object = Foo.get_cached(self.foo.pk) + + self.assertEqual(self.foo.bar, cached_object.bar) + + def test_cache_invalidated_on_update(self): + self.foo.bar = 'quux' + self.foo.save() + + self._populate_cache() + + self.assertEqual(Foo.get_cached(self.foo.pk).bar, 'quux') + + def test_cache_invalidated_on_delete(self): + self.foo.bar = 'quux' + self.foo.delete() - self.assertEqual(first_object.bar, cached_object.bar) + self._populate_cache() From 870b6c07ebd57e12048ca28f42b72a4e3c90e349 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 11:43:00 +0000 Subject: [PATCH 18/79] Fix this test --- tests/test_cached_get.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cached_get.py b/tests/test_cached_get.py index dccbc58..003fc4c 100644 --- a/tests/test_cached_get.py +++ b/tests/test_cached_get.py @@ -25,7 +25,9 @@ def test_cache_invalidated_on_update(self): self.assertEqual(Foo.get_cached(self.foo.pk).bar, 'quux') def test_cache_invalidated_on_delete(self): - self.foo.bar = 'quux' + pk = self.foo.pk + self.foo.delete() - self._populate_cache() + with self.assertRaises(Foo.DoesNotExist): + Foo.get_cached(pk) From 1f6491bd99e0efb496c34e1bb85b21017491a9ab Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 11:45:03 +0000 Subject: [PATCH 19/79] Add a version number on the cache keys --- cache_toolbox/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index fdd9067..be03fa5 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -14,6 +14,8 @@ from . import app_settings +CACHE_FORMAT_VERSION = 2 + def get_instance( model, instance_or_pk, timeout=None, using=None, create=False, defaults=None @@ -134,7 +136,8 @@ def instance_key(model, instance_or_pk): # Django version <1.6 model_name = model._meta.module_name - return '%s.%s:%d' % ( + return 'cache.%d:%s.%s:%d' % ( + CACHE_FORMAT_VERSION, model._meta.app_label, model_name, getattr(instance_or_pk, 'pk', instance_or_pk), From fd22a238b30f35bd19ab5e5888add06f43e78f42 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Fri, 6 Nov 2015 11:56:35 +0000 Subject: [PATCH 20/79] Run all encoding through Pickle --- cache_toolbox/core.py | 43 ++++++++++++----------------------- tests/test_cached_relation.py | 1 - 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index be03fa5..e75cee9 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -8,6 +8,11 @@ """ +try: + import cPickle as pickle +except ImportError: + import pickle + from django.core.cache import cache from django.db import DEFAULT_DB_ALIAS @@ -49,7 +54,7 @@ def get_instance( if data is not None: try: # Try and construct instance from dictionary - instance = model(pk=pk, **data) + instance = model(pk=pk, **pickle.loads(data)) # Ensure instance knows that it already exists in the database, # otherwise we will fail any uniqueness checks when saving the @@ -81,38 +86,18 @@ def get_instance( if field.primary_key: continue - # By default, serialise the instance using the Field's own - # serialisation routines. - # - # It may seem odd to have a list of exceptions this way around, but - # this scheme allows custom fields to specify their own method of - # correctly serialising themselves: Field's cannot easily override - # value_from_object, but they can with value_to_string. - data[field.attname] = field.value_to_string(instance) - - # As special-cases: - if field.get_internal_type() in ( - # Serialise the actual value at the field's attname for any relations. - # This is to avoid caching u"None" (the string) instead of None - # (ie. NoneType) for nullable relations. - 'AutoField', - 'OneToOneField', - - # Serialise datetimes and booleans directly, otherwise we - # incorrectly try to instantiate our instance with a string - # representation. - 'BooleanField', - 'NullBooleanField', - 'TimeField', - 'DateField', - 'DateTimeField', - ): - data[field.attname] = field.value_from_object(instance) + # We also don't want to save any virtual fields. + if not field.concrete: + continue + + data[field.attname] = getattr(instance, field.attname) if timeout is None: timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT - cache.set(key, data, timeout) + # Encode through Pickle, since that allows overriding and covers (most) + # Python types we'd want to serialise. + cache.set(key, pickle.dumps(data, protocol=-1), timeout) return instance diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index 75e33ab..4fe4e00 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -5,7 +5,6 @@ from .models import Foo, Bazz class CachedRelationTest(TestCase): - @skip("Currently broken") def test_cached_relation(self): foo = Foo.objects.create(bar='bees') From a601b2ec4f4f6796dc5fa8ed613d1b834001a367 Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Mon, 9 Nov 2015 15:16:21 +0000 Subject: [PATCH 21/79] Fix the authentication middleware At some point `request.session[SESSION_KEY]` started returning a `unicode` rather than an `int`. As a result this would always fail, and thus always fall back to the parent mechanism. --- cache_toolbox/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache_toolbox/middleware.py b/cache_toolbox/middleware.py index 97f0bdb..ade11af 100644 --- a/cache_toolbox/middleware.py +++ b/cache_toolbox/middleware.py @@ -91,7 +91,7 @@ def __init__(self): def process_request(self, request): try: # Try and construct a User instance from data stored in the cache - request.user = User.get_cached(request.session[SESSION_KEY]) + request.user = User.get_cached(int(request.session[SESSION_KEY])) except: # Fallback to constructing the User from the database. super(CacheBackedAuthenticationMiddleware, self).process_request(request) From 737fea0d97f09df64a4be02b6bddd13c4fc1497f Mon Sep 17 00:00:00 2001 From: Alistair Lynn Date: Mon, 9 Nov 2015 16:09:18 +0000 Subject: [PATCH 22/79] Don't assume a numeric primary key --- cache_toolbox/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index e75cee9..e5c0e7e 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -121,7 +121,7 @@ def instance_key(model, instance_or_pk): # Django version <1.6 model_name = model._meta.module_name - return 'cache.%d:%s.%s:%d' % ( + return 'cache.%d:%s.%s:%s' % ( CACHE_FORMAT_VERSION, model._meta.app_label, model_name, From 51d0f207ef58ac5de0fc5a429e6fecf5fe473df3 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Tue, 23 Feb 2016 17:27:12 +0000 Subject: [PATCH 23/79] Reflow --- cache_toolbox/relation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index a84db46..09ea7df 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -102,7 +102,11 @@ def get(self): pass instance = get_instance( - rel.field.model, self.pk, timeout, create=create, defaults=defaults + rel.field.model, + self.pk, + timeout, + create=create, + defaults=defaults ) setattr(self, '_%s_cache' % related_name, instance) From 0e5c7d0ae74b925010b0958932ed2cd0e45bb107 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Tue, 23 Feb 2016 17:33:14 +0000 Subject: [PATCH 24/79] Specify using for relations This means that when there is a cache miss, the database that is currently in use will be used to fetch the data. For example, if you have a database slave, model Foo with a cached relation bar, which has a property baz: random_foo = Foo.objects.order_by('?').using('slave').first() # On cache misses, this will hit the database slave, as that is what was # used to fetch baz. baz = random_foo.bar_cache.baz --- cache_toolbox/relation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 09ea7df..da44d1a 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -106,7 +106,8 @@ def get(self): self.pk, timeout, create=create, - defaults=defaults + defaults=defaults, + using=self._state.db, ) setattr(self, '_%s_cache' % related_name, instance) From e56f1076d9e62c4adb57bf99d20bc55e382f2c48 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 14 Mar 2016 14:57:03 +0000 Subject: [PATCH 25/79] Releasing version 0.1.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6be3d7d..9fb56b7 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.1', + version='0.1.1', url='http://code.playfire.com/django-cache-toolbox', author='Playfire.com', From 4a95ad6d7edc7d95d3a8d84f62c4198176bf9171 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 14 Mar 2016 15:00:14 +0000 Subject: [PATCH 26/79] Releasing version 0.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e048c7..d22ee53 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.1.1', + version='0.2.0', url='https://www.github.com/thread/django-cache-toolbox', author='Thread.com', From 056b3186ce98b3ca0630e7977d9eebc987d81563 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 14 Mar 2016 15:01:44 +0000 Subject: [PATCH 27/79] Drop support for 1.6 --- cache_toolbox/core.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index e5c0e7e..dea926f 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -115,15 +115,9 @@ def instance_key(model, instance_or_pk): Returns the cache key for this (model, instance) pair. """ - try: - model_name = model._meta.model_name - except AttributeError: - # Django version <1.6 - model_name = model._meta.module_name - return 'cache.%d:%s.%s:%s' % ( CACHE_FORMAT_VERSION, model._meta.app_label, - model_name, + model._meta.model_name getattr(instance_or_pk, 'pk', instance_or_pk), ) From a56f5c46911b1d98c29f561c886f2dc233a555f4 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 14 Mar 2016 15:01:56 +0000 Subject: [PATCH 28/79] Releasing version 0.2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d22ee53..2ecd719 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.2.0', + version='0.2.1', url='https://www.github.com/thread/django-cache-toolbox', author='Thread.com', From 582bf1222204868ccd9a80d3cc024a6a84a5711f Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 21 Mar 2016 16:15:10 +0800 Subject: [PATCH 29/79] This is now on related_model, not model! --- cache_toolbox/relation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index da44d1a..e962e44 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -118,17 +118,17 @@ def get(self): # Clearing cache def clear(self): - delete_instance(rel.model, self) + delete_instance(rel.related_model, self) @classmethod def clear_pk(cls, *instances_or_pk): - delete_instance(rel.model, *instances_or_pk) + delete_instance(rel.related_model, *instances_or_pk) def clear_cache(sender, instance, *args, **kwargs): - delete_instance(rel.model, instance) + delete_instance(rel.related_model, instance) setattr(rel.to, '%s_clear' % related_name, clear) setattr(rel.to, '%s_clear_pk' % related_name, clear_pk) - post_save.connect(clear_cache, sender=rel.model, weak=False) - post_delete.connect(clear_cache, sender=rel.model, weak=False) + post_save.connect(clear_cache, sender=rel.related_model, weak=False) + post_delete.connect(clear_cache, sender=rel.related_model, weak=False) From 54d6bd71fde66adaa3593c20bf3ac8fdf6d557b3 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 21 Mar 2016 16:15:34 +0800 Subject: [PATCH 30/79] Releasing version 0.2.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2ecd719..e0921cf 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.2.1', + version='0.2.2', url='https://www.github.com/thread/django-cache-toolbox', author='Thread.com', From 4b36e4b4b59f0101262e8effa3b1832cb43398ba Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 21 Mar 2016 16:20:07 +0800 Subject: [PATCH 31/79] Add missing trailing comma --- cache_toolbox/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index dea926f..74b3c9a 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -118,6 +118,6 @@ def instance_key(model, instance_or_pk): return 'cache.%d:%s.%s:%s' % ( CACHE_FORMAT_VERSION, model._meta.app_label, - model._meta.model_name + model._meta.model_name, getattr(instance_or_pk, 'pk', instance_or_pk), ) From bc6309ae03fda66656942f5405d0fed90ac9f236 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 21 Mar 2016 16:23:11 +0800 Subject: [PATCH 32/79] Drop broken creation non-UPSET functionality Signed-off-by: Chris Lamb --- cache_toolbox/core.py | 15 ++------------- cache_toolbox/relation.py | 13 ++----------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 74b3c9a..f5469bf 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -21,10 +21,7 @@ CACHE_FORMAT_VERSION = 2 -def get_instance( - model, instance_or_pk, - timeout=None, using=None, create=False, defaults=None -): +def get_instance(model, instance_or_pk, timeout=None, using=None): """ Returns the ``model`` instance with a primary key of ``instance_or_pk``. @@ -34,9 +31,6 @@ def get_instance( If omitted, the timeout value defaults to ``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero). - If ``create`` is True, we are going to create the instance in case that it - was not found. - Example:: >>> get_instance(User, 1) # Cache miss @@ -72,12 +66,7 @@ def get_instance( cache.delete(key) # Use the default manager so we are never filtered by a .get_query_set() - queryset = model._default_manager.using(using) - if create: - # It's possible that the related object didn't exist yet - instance, _ = queryset.get_or_create(pk=pk, defaults=defaults or {}) - else: - instance = queryset.get(pk=pk) + instance = model._default_manager.using(using).get(pk=pk) data = {} for field in instance._meta.fields: diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index e962e44..844e122 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -22,16 +22,9 @@ class Foo(models.Model): name = models.CharField(max_length=20) - cache_relation(User.foo, create=True, defaults={}) + cache_relation(User.foo) (``primary_key`` being ``True`` is currently required.) - -With ``create=True`` we force the creation of an instance of `Foo` in case that -we are trying to access to user.foo_cache but ``user.foo`` doesn't exist yet. - -If ``create=True`` we are going to pass the default to the get_or_create -function. - :: >>> user = User.objects.get(pk=1) @@ -83,7 +76,7 @@ class Foo(models.Model): from .core import get_instance, delete_instance -def cache_relation(descriptor, timeout=None, create=False, defaults=None): +def cache_relation(descriptor, timeout=None): rel = descriptor.related related_name = '%s_cache' % rel.field.related_query_name() @@ -105,8 +98,6 @@ def get(self): rel.field.model, self.pk, timeout, - create=create, - defaults=defaults, using=self._state.db, ) From 6cfae0dbc65a0ed457a3b8dd204422dd67ebf359 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 21 Mar 2016 16:24:15 +0800 Subject: [PATCH 33/79] Fork --- COPYING | 1 + setup.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/COPYING b/COPYING index ae62558..8eea954 100644 --- a/COPYING +++ b/COPYING @@ -1,3 +1,4 @@ +Copyright © 2016 Chris Lamb Copyright © 2010, 2011 UUMC Ltd. All rights reserved. diff --git a/setup.py b/setup.py index e0921cf..a7a2dfd 100644 --- a/setup.py +++ b/setup.py @@ -6,10 +6,10 @@ name='django-cache-toolbox', description="Non-magical object caching for Django.", version='0.2.2', - url='https://www.github.com/thread/django-cache-toolbox', + url='https://chris-lamb.co.uk/projects/django-cache-toolbox', - author='Thread.com', - author_email='tech@thread.com', + author='Chris Lamb', + author_email='chris@chris-lamb.co.uk', license='BSD', packages=find_packages(exclude=('tests',)), From 0b65e49cfd4006743e6849e8e724dea00d655a07 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 21 Mar 2016 16:25:09 +0800 Subject: [PATCH 34/79] Drop unused import --- tests/test_cached_relation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index 4fe4e00..fb5ee2d 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -1,5 +1,3 @@ -from unittest import skip - from django.test import TestCase from .models import Foo, Bazz From 8c5d5a1546b37bca4454db5c06162629d3777143 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 21 Mar 2016 16:25:13 +0800 Subject: [PATCH 35/79] Releasing version 0.2.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a7a2dfd..5d9fb3e 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.2.2', + version='0.2.3', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From 4ae4f8ba10de5454f9098ee548bcc68b559c0ca7 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 24 Feb 2017 13:11:18 +0800 Subject: [PATCH 36/79] Move away from deprecated django.template.resolve_variable. --- cache_toolbox/templatetags/cache_toolbox.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cache_toolbox/templatetags/cache_toolbox.py b/cache_toolbox/templatetags/cache_toolbox.py index feea2af..5c4469e 100644 --- a/cache_toolbox/templatetags/cache_toolbox.py +++ b/cache_toolbox/templatetags/cache_toolbox.py @@ -1,7 +1,6 @@ from django import template from django.core.cache import cache from django.template import Node, TemplateSyntaxError, Variable -from django.template import resolve_variable register = template.Library() @@ -9,10 +8,10 @@ class CacheNode(Node): def __init__(self, nodelist, expire_time, key): self.nodelist = nodelist self.expire_time = Variable(expire_time) - self.key = key + self.key = Variable(key) def render(self, context): - key = resolve_variable(self.key, context) + key = self.key.resolve(context) expire_time = int(self.expire_time.resolve(context)) value = cache.get(key) @@ -44,10 +43,10 @@ def cachedeterministic(parser, token): class ShowIfCachedNode(Node): def __init__(self, key): - self.key = key + self.key = Variable(key) def render(self, context): - key = resolve_variable(self.key, context) + key = self.key.resolve(context) return cache.get(key) or '' @register.tag From 63caa7c6546ae80b183dc2cdb0c3d5a4aee2b7c9 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 24 Feb 2017 13:11:39 +0800 Subject: [PATCH 37/79] Releasing version 0.2.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5d9fb3e..d6b30f2 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.2.3', + version='0.2.4', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From 43a6c33f6cb57d7b94a4d10940e20ab2e0a9f4bd Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 8 Mar 2018 15:46:59 +0000 Subject: [PATCH 38/79] Ignore eggs (#1) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6d582b3..44e66be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.pyc docs/_build +/.eggs +/*.egg-info From 3864f5d7ec27f880e1a41a4fb80984026ffd8048 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 8 Mar 2018 15:47:29 +0000 Subject: [PATCH 39/79] Django 1.x (#2) * We're not compatible with Django 2.0 yet * Add on_delete which is now expected This will be required in Django 2.0. --- setup.py | 2 +- tests/models.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d6b30f2..7b00b47 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ packages=find_packages(exclude=('tests',)), install_requires=( - "Django>=1.8", + "Django>=1.8<2.0", ), test_suite='tests', diff --git a/tests/models.py b/tests/models.py index baa7d1b..ffa49cd 100644 --- a/tests/models.py +++ b/tests/models.py @@ -6,7 +6,11 @@ class Foo(models.Model): bar = models.TextField() class Bazz(models.Model): - foo = models.OneToOneField(Foo, related_name='bazz') + foo = models.OneToOneField( + Foo, + related_name='bazz', + on_delete=models.CASCADE, + ) value = models.IntegerField(null=True) From 62fd94fc15b3ad957bdb55af10db9a885b73e98c Mon Sep 17 00:00:00 2001 From: Peter Law Date: Wed, 7 Mar 2018 19:20:50 +0000 Subject: [PATCH 40/79] Switch to the test running mechanism from django-enumfield This works, whereas the previous 'python setup.py test' approach didn't seem to work for me. --- runtests.py | 15 +++++++++++++++ setup.py | 2 -- tests/__init__.py | 19 ------------------- tests/test_settings.py | 10 ++++++++++ 4 files changed, 25 insertions(+), 21 deletions(-) create mode 100755 runtests.py create mode 100644 tests/test_settings.py diff --git a/runtests.py b/runtests.py new file mode 100755 index 0000000..3361623 --- /dev/null +++ b/runtests.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +import django +from django.conf import settings +from django.test.utils import get_runner + +if __name__ == '__main__': + os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_settings' + django.setup() + TestRunner = get_runner(settings) + test_runner = TestRunner() + failures = test_runner.run_tests(sys.argv[1:]) + sys.exit(bool(failures)) diff --git a/setup.py b/setup.py index 7b00b47..85bdbdf 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,4 @@ install_requires=( "Django>=1.8<2.0", ), - - test_suite='tests', ) diff --git a/tests/__init__.py b/tests/__init__.py index a367231..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,19 +0,0 @@ -from django import setup -from django.conf import settings -from django.core.management import call_command - -settings.configure( - DATABASES={ - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - }, - }, - INSTALLED_APPS=( - 'tests', - ), -) - -setup() - -call_command('migrate') diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..0a1940c --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,10 @@ +SECRET_KEY = 'fake-key' +INSTALLED_APPS = [ + 'tests', +] +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + }, +} From 5961c37230601382fc677c96e0eac7ed7c8c81eb Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 8 Mar 2018 10:31:55 +0000 Subject: [PATCH 41/79] Support `hasattr` We need to convert the `DoesNotExist` error into a `RelatedObjectDoesNotExist` (which extends `DoesNotExist` by also inheirting from `AttributeError`) so that `hasattr` can correctly identify missing attributes. --- cache_toolbox/relation.py | 20 +++++++++++----- tests/test_cached_relation.py | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 844e122..9937c5d 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -94,12 +94,20 @@ def get(self): except AttributeError: pass - instance = get_instance( - rel.field.model, - self.pk, - timeout, - using=self._state.db, - ) + try: + instance = get_instance( + rel.field.model, + self.pk, + timeout, + using=self._state.db, + ) + except rel.related_model.DoesNotExist: + raise descriptor.RelatedObjectDoesNotExist( + "%s has no %s." % ( + rel.to.__name__, + related_name, + ), + ) setattr(self, '_%s_cache' % related_name, instance) diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index fb5ee2d..ab97c6a 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -1,8 +1,16 @@ +from django.core.cache import cache from django.test import TestCase from .models import Foo, Bazz class CachedRelationTest(TestCase): + longMessage = True + + def setUp(self): + # Ensure we start with a clear cache for each test, i.e. tests can use + # the cache hygenically + cache.clear() + def test_cached_relation(self): foo = Foo.objects.create(bar='bees') @@ -15,3 +23,39 @@ def test_cached_relation(self): cached_object = Foo.objects.get(pk=foo.pk).bazz_cache self.assertEqual(cached_object.value, 10) + + self.assertTrue( + hasattr(foo, 'bazz'), + "Foo should have 'bazz' attribute", + ) + + self.assertTrue( + hasattr(foo, 'bazz_cache'), + "Foo should have 'bazz_cache' attribute", + ) + + def test_cached_relation_not_present_hasattr(self): + foo = Foo.objects.create(bar='bees_2') + + self.assertFalse( + hasattr(foo, 'bazz_cache'), + "Foo should not have 'bazz_cache' attribute (empty cache)", + ) + + # sanity check + self.assertFalse( + hasattr(foo, 'bazz'), + "Foo should not have 'bazz' attribute", + ) + + def test_cached_relation_not_present_exception(self): + foo = Foo.objects.create(bar='bees_3') + + with self.assertRaises(Bazz.DoesNotExist) as cm: + foo.bazz_cache + + self.assertIsInstance( + cm.exception, + AttributeError, + "Raised error must also be an AttributeError (we're expecting a 'RelatedObjectDoesNotExist')", + ) From 497094e18a4a6b0ad7953df9ef5d6c765355f254 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 8 Mar 2018 10:51:16 +0000 Subject: [PATCH 42/79] Ensure `hasattr` is consistent Previously accessing the natural (uncached) relation would cause `hasattr` to always return True, even if the value wasn't actually present. This changes that behaviour to be consistent with the underlying descriptor by deferring to it in this case. --- cache_toolbox/relation.py | 6 ++---- tests/test_cached_relation.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 9937c5d..c568f31 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -83,10 +83,8 @@ def cache_relation(descriptor, timeout=None): @property def get(self): # Always use the cached "real" instance if available - try: - return getattr(self, descriptor.cache_name) - except AttributeError: - pass + if descriptor.is_cached(self): + return descriptor.__get__(self) # Lookup cached instance try: diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index ab97c6a..f27f52b 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -42,12 +42,22 @@ def test_cached_relation_not_present_hasattr(self): "Foo should not have 'bazz_cache' attribute (empty cache)", ) + self.assertFalse( + hasattr(foo, 'bazz_cache'), + "Foo should not have 'bazz_cache' attribute (warm cache; before natural access)", + ) + # sanity check self.assertFalse( hasattr(foo, 'bazz'), "Foo should not have 'bazz' attribute", ) + self.assertFalse( + hasattr(foo, 'bazz_cache'), + "Foo should not have 'bazz_cache' attribute (warm cache; after natural access)", + ) + def test_cached_relation_not_present_exception(self): foo = Foo.objects.create(bar='bees_3') From a576370a5ef73f6f250d9153cfc6045d9ecf5565 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Thu, 8 Mar 2018 07:49:54 -0800 Subject: [PATCH 43/79] Releasing version 0.3.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 85bdbdf..dc49e40 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.2.4', + version='0.3.0', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From 2a615c47480ba07f7e4ea08f46b5bc8d9f20d3da Mon Sep 17 00:00:00 2001 From: Peter Law Date: Fri, 13 Jul 2018 15:53:48 +0100 Subject: [PATCH 44/79] Avoid race condition between transaction commit and cache clear See inline comment for details. Co-Authored-By: Alistair Lynn --- cache_toolbox/core.py | 23 +++++++++++++++++++++-- tests/test_cached_get.py | 5 +++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index f5469bf..7387b10 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -14,7 +14,7 @@ import pickle from django.core.cache import cache -from django.db import DEFAULT_DB_ALIAS +from django.db import DEFAULT_DB_ALIAS, transaction from . import app_settings @@ -96,7 +96,26 @@ def delete_instance(model, *instance_or_pk): Purges the cache keys for the instances of this model. """ - cache.delete_many([instance_key(model, x) for x in instance_or_pk]) + # Only clear the cache when the current transaction commits. + # While clearing the cache earlier than that is valid, it is insufficient + # to ensure cache consistency. There is a possible race between two + # transactions as follows: + # + # Transaction 1: modifies model (and thus clears cache) + # Transaction 2: queries cache, which misses, so it populates the cache + # from the database, picking up the unmodified model + # Transaction 1: commits, without further signal to the cache + # + # At this point the cache contains the _original_ value of the model, which + # is out of step with the database. + # To avoid this we delay clearing the cache until the transaction commits. + # While this does leave a small window after the transaction has committed + # but before the cache has cleared, that is better than leaving the cache + # incorrect until the model is next updated. + + transaction.on_commit(lambda: cache.delete_many( + [instance_key(model, x) for x in instance_or_pk], + )) def instance_key(model, instance_or_pk): diff --git a/tests/test_cached_get.py b/tests/test_cached_get.py index 003fc4c..b280240 100644 --- a/tests/test_cached_get.py +++ b/tests/test_cached_get.py @@ -1,8 +1,9 @@ -from django.test import TestCase +from django.test import TransactionTestCase from .models import Foo -class CachedGetTest(TestCase): +# Use `TransactionTestCase` so that our `on_commit` actions happen when we expect. +class CachedGetTest(TransactionTestCase): def setUp(self): self.foo = Foo.objects.create(bar='bees') self._populate_cache() From 8b80fcd758c6e4e1a73452644185c707b5aa22c6 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sat, 14 Jul 2018 08:57:21 +0100 Subject: [PATCH 45/79] Releasing version 0.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dc49e40..20d6506 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.3.0', + version='0.3.1', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From c924be86d7f3392da1c2431fe6e143968da5b38f Mon Sep 17 00:00:00 2001 From: Peter Law Date: Tue, 17 Jul 2018 14:56:07 +0100 Subject: [PATCH 46/79] Support older setuptools It turns out that older setuptools requires that there be commas in dependency specifiers while newer versions are a bit more relaxed. Since older versions need to at least parse the specifiers in order to upgrade themselves, we need to use the older format if we want to support the older versions _at all_, even if we aren't targetting them. Obviously having made this change we don't actually need a newer version of setuptools, so we don't add a specific dependency there. Previously this would error with: ``` Collecting django-cache-toolbox==0.3.1 Downloading https://files.pythonhosted.org/packages/eb/f7/f39a121ade50b873b8dbd9f285c3c02810d08e96404e1ca2d6546b48db86/django-cache-toolbox-0.3.1.tar.gz Complete output from command python setup.py egg_info: error in django-cache-toolbox setup command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers ---------------------------------------- Command "python setup.py egg_info" failed with error code 1 in /tmp/pip-install-187om0y_/django-cache-toolbox/ ``` --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 20d6506..835c9d4 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(exclude=('tests',)), install_requires=( - "Django>=1.8<2.0", + "Django>=1.8,<2.0", ), ) From 8b4eb371046a9bba260fcc81cc7f5bcef7c579b6 Mon Sep 17 00:00:00 2001 From: Matthew Power Date: Fri, 20 Jul 2018 15:31:27 +0100 Subject: [PATCH 47/79] Upgrade for Django 2.0 compatibility --- cache_toolbox/middleware.py | 5 +++-- cache_toolbox/relation.py | 8 ++++---- setup.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cache_toolbox/middleware.py b/cache_toolbox/middleware.py index ade11af..deea63a 100644 --- a/cache_toolbox/middleware.py +++ b/cache_toolbox/middleware.py @@ -85,13 +85,14 @@ from .model import cache_model class CacheBackedAuthenticationMiddleware(AuthenticationMiddleware): - def __init__(self): + def __init__(self, get_response): + super(CacheBackedAuthenticationMiddleware, self).__init__(get_response) cache_model(User) def process_request(self, request): try: # Try and construct a User instance from data stored in the cache request.user = User.get_cached(int(request.session[SESSION_KEY])) - except: + except Exception: # Fallback to constructing the User from the database. super(CacheBackedAuthenticationMiddleware, self).process_request(request) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index c568f31..3bd3889 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -102,7 +102,7 @@ def get(self): except rel.related_model.DoesNotExist: raise descriptor.RelatedObjectDoesNotExist( "%s has no %s." % ( - rel.to.__name__, + rel.model.__name__, related_name, ), ) @@ -110,7 +110,7 @@ def get(self): setattr(self, '_%s_cache' % related_name, instance) return instance - setattr(rel.to, related_name, get) + setattr(rel.model, related_name, get) # Clearing cache @@ -124,8 +124,8 @@ def clear_pk(cls, *instances_or_pk): def clear_cache(sender, instance, *args, **kwargs): delete_instance(rel.related_model, instance) - setattr(rel.to, '%s_clear' % related_name, clear) - setattr(rel.to, '%s_clear_pk' % related_name, clear_pk) + setattr(rel.model, '%s_clear' % related_name, clear) + setattr(rel.model, '%s_clear_pk' % related_name, clear_pk) post_save.connect(clear_cache, sender=rel.related_model, weak=False) post_delete.connect(clear_cache, sender=rel.related_model, weak=False) diff --git a/setup.py b/setup.py index 835c9d4..65e2ef5 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(exclude=('tests',)), install_requires=( - "Django>=1.8,<2.0", + "Django>=1.8,<2.1", ), ) From 8edbc052eec58913bd448f6741564238bc44a350 Mon Sep 17 00:00:00 2001 From: Matthew Power Date: Fri, 20 Jul 2018 15:38:36 +0100 Subject: [PATCH 48/79] =?UTF-8?q?Update=20setup.py=20since=20the=20tests?= =?UTF-8?q?=20don=E2=80=99t=20pass=20under=20Django=201.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 65e2ef5..4ce23cb 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(exclude=('tests',)), install_requires=( - "Django>=1.8,<2.1", + "Django>=1.9,<2.1", ), ) From f0f0d9e234956c2bca78cca1a14cc5eb6662af87 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Wed, 18 Jul 2018 08:41:41 +0800 Subject: [PATCH 49/79] Releasing version 0.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4ce23cb..e6fa25b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.3.1', + version='0.3.2', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From e701e42a1aa8008f8bbc5c0a7f30729e82b00658 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 20 Jul 2018 22:50:42 +0800 Subject: [PATCH 50/79] Releasing version 1.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e6fa25b..30f1167 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='0.3.2', + version='1.0.0', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From ca7f6cf32ec2529d0651af0a60e395bd699a566c Mon Sep 17 00:00:00 2001 From: Dan Palmer Date: Fri, 24 Aug 2018 14:33:54 +0100 Subject: [PATCH 51/79] Bump allowed Django version to <2.2 This appears to work fine in practice on 2.1. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 30f1167..3a4cd6c 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(exclude=('tests',)), install_requires=( - "Django>=1.9,<2.1", + "Django>=1.9,<2.2", ), ) From 80ba9a64a3669e092c63ec46a0afb0749f2bf626 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 27 Aug 2018 13:47:03 +0200 Subject: [PATCH 52/79] Releasing version 1.1.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3a4cd6c..5d9c07f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='1.0.0', + version='1.1.0', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From 1c49314151775a4571b4fdce6c53d8e3be434eb0 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Mon, 21 Jan 2019 21:46:46 +0000 Subject: [PATCH 53/79] Add a README with basic usage information --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..5261d4a --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# django-cache-toolbox + +_Non-magical object caching for Django._ + +Another caching framework for Django that does not do any magic behind your +back, saving brain cycles when debugging as well as sticking to Django +principles. + +## Installation + +From [PyPI](https://pypi.org/project/django-cache-toolbox/): +``` +pip install django-cache-toolbox +``` + +## Basic Usage + +``` python +from cache_toolbox import cache_model, cache_relation +from django.db import models + +class Foo(models.Model): + ... + +class Bazz(models.Model): + foo = models.OneToOneField(Foo, related_name='bazz') + ... + +# Prepare caching of a model +cache_model(Foo) + +# Prepare caching of a relation +cache_relation(Foo.bazz) + +# Fetch the cached version of a model +foo = Foo.get_cached(pk=42) + +# Load a cached relation +print(foo.bazz) +``` + +See the module docstrings for further details. From 95f5609e2831996b7d9e17c41f109b9bc17aa022 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Tue, 22 Jan 2019 14:08:02 +0000 Subject: [PATCH 54/79] Actually demonstrate using a _cached_ relation The previous code worked, but wasn't actually showing what it intended to show. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5261d4a..257a27e 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ cache_relation(Foo.bazz) foo = Foo.get_cached(pk=42) # Load a cached relation -print(foo.bazz) +print(foo.bazz_cache) ``` See the module docstrings for further details. From 25617b8f3fddb20300578e428250bae7b890d434 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sat, 20 Jul 2019 17:47:55 -0300 Subject: [PATCH 55/79] Also support Django 2.2. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5d9c07f..1d4c432 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,6 @@ packages=find_packages(exclude=('tests',)), install_requires=( - "Django>=1.9,<2.2", + "Django>=1.9", ), ) From 8908bcf7443352b5fa95f5a21c7985a4c70db6c2 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sat, 20 Jul 2019 17:48:22 -0300 Subject: [PATCH 56/79] Releasing version 1.1.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1d4c432..8cdbd3e 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", - version='1.1.0', + version='1.1.1', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From 59cfc2e367bf0c293cb066b36fe709247f5c067e Mon Sep 17 00:00:00 2001 From: Peter Law Date: Fri, 13 Mar 2020 18:47:13 +0000 Subject: [PATCH 57/79] Include the README as the package long description --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 8cdbd3e..96c0509 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,17 @@ #!/usr/bin/env python +import os.path from setuptools import setup, find_packages +my_dir = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(my_dir, 'README.md')) as f: + long_description = f.read() + setup( name='django-cache-toolbox', description="Non-magical object caching for Django.", + long_description=long_description, + long_description_content_type='text/markdown', version='1.1.1', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', From 4033135f60bbbb5a7d1aa9cfaeb53a20c41368c5 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Fri, 13 Mar 2020 19:10:41 +0000 Subject: [PATCH 58/79] Cache negative relation lookups locally This matches what Django does. Fixes https://github.com/lamby/django-cache-toolbox/issues/15 --- cache_toolbox/relation.py | 18 ++++++++++++++--- tests/test_cached_relation.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 3bd3889..ae2ee1e 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -88,9 +88,20 @@ def get(self): # Lookup cached instance try: - return getattr(self, '_%s_cache' % related_name) + instance = getattr(self, '_%s_cache' % related_name) except AttributeError: + # no local cache pass + else: + if instance is None: + # we (locally) cached that there is no model + raise descriptor.RelatedObjectDoesNotExist( + "%s has no %s." % ( + rel.model.__name__, + related_name, + ), + ) + return instance try: instance = get_instance( @@ -100,14 +111,15 @@ def get(self): using=self._state.db, ) except rel.related_model.DoesNotExist: + instance = None raise descriptor.RelatedObjectDoesNotExist( "%s has no %s." % ( rel.model.__name__, related_name, ), ) - - setattr(self, '_%s_cache' % related_name, instance) + finally: + setattr(self, '_%s_cache' % related_name, instance) return instance setattr(rel.model, related_name, get) diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index f27f52b..36a2fad 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -69,3 +69,41 @@ def test_cached_relation_not_present_exception(self): AttributeError, "Raised error must also be an AttributeError (we're expecting a 'RelatedObjectDoesNotExist')", ) + + def test_cached_missing_relation_uses_select_related(self): + foo = Foo.objects.create(bar='bees') + + with self.assertNumQueries(1): + foo = Foo.objects.select_related('bazz').get(pk=foo.pk) + + with self.assertNumQueries(0): + with self.assertRaises(Bazz.DoesNotExist): + foo.bazz_cache + + def test_cached_missing_relation_cached_locally(self): + # Django will cache on the instance that foo.bazz doesn't exist, just + # the same as it would cache the Bazz instance if there was one. Mimic + # that behaviour in order to have comparable querying behaviour. + + foo = Foo.objects.create(bar='bees') + + with self.assertNumQueries(1): + foo = Foo.objects.get(pk=foo.pk) + + # Populate the (instance) cache + with self.assertNumQueries(1): + with self.assertRaises(Bazz.DoesNotExist): + foo.bazz_cache + + # Get from the (instance) cache + with self.assertNumQueries(0): + with self.assertRaises(Bazz.DoesNotExist): + foo.bazz_cache + + with self.assertNumQueries(1): + foo = Foo.objects.get(pk=foo.pk) + + # Prove that we haven't put anything into the remote cache + with self.assertNumQueries(1): + with self.assertRaises(Bazz.DoesNotExist): + foo.bazz_cache From 2855f644eda7ef481cd62c401f2e2d187b2b5f10 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Tue, 31 Mar 2020 21:30:43 +0100 Subject: [PATCH 59/79] Make setup.py executable. --- setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 From 823cb316ae6ee8218b567034b296cede14bf06f9 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Tue, 31 Mar 2020 21:31:17 +0100 Subject: [PATCH 60/79] Releasing version 1.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 96c0509..b1820c5 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ description="Non-magical object caching for Django.", long_description=long_description, long_description_content_type='text/markdown', - version='1.1.1', + version='1.2.0', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From ab50996df1e51cfadc84cb48359454ee81a127b8 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Wed, 29 Apr 2020 09:29:20 +0100 Subject: [PATCH 61/79] Fix a secondary error from cache exceptions If cache.get manages to raise an unexpected exception, then previously our relation handling would end up raising an `UnboundLocalError` due to the use of `instance` in the `finally` clause. This changes that behaviour by duplicating the `setattr` call. --- cache_toolbox/relation.py | 5 ++--- tests/test_cached_relation.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index ae2ee1e..5509588 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -110,16 +110,15 @@ def get(self): timeout, using=self._state.db, ) + setattr(self, '_%s_cache' % related_name, instance) except rel.related_model.DoesNotExist: - instance = None + setattr(self, '_%s_cache' % related_name, None) raise descriptor.RelatedObjectDoesNotExist( "%s has no %s." % ( rel.model.__name__, related_name, ), ) - finally: - setattr(self, '_%s_cache' % related_name, instance) return instance setattr(rel.model, related_name, get) diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index 36a2fad..8177096 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -1,3 +1,5 @@ +from unittest import mock + from django.core.cache import cache from django.test import TestCase @@ -107,3 +109,19 @@ def test_cached_missing_relation_cached_locally(self): with self.assertNumQueries(1): with self.assertRaises(Bazz.DoesNotExist): foo.bazz_cache + + def test_get_instance_error_doesnt_have_side_effect_issues(self): + foo = Foo.objects.create(bar='bees') + + class DummyException(Exception): + pass + + # Validate that the underlying error is passed through, without any + # other errors happening... + with mock.patch('cache_toolbox.core.cache.get', side_effect=DummyException): + with self.assertRaises(DummyException): + foo.bazz_cache + + # ... and that we haven't put anything bad in the cache along the way + bazz = Bazz.objects.create(foo=foo) + self.assertEqual(bazz, foo.bazz_cache) From b5e5bdfa11c5b83d211579fc34510c7099952dcc Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Wed, 29 Apr 2020 15:41:59 +0100 Subject: [PATCH 62/79] Releasing version 1.2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1820c5..024d5c7 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ description="Non-magical object caching for Django.", long_description=long_description, long_description_content_type='text/markdown', - version='1.2.0', + version='1.2.1', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From d4baa757dedbfbcaa052c7774ee435f101dec393 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Mon, 28 Dec 2020 16:59:13 +0000 Subject: [PATCH 63/79] Explicitly require cached relations are primary keys This should prevent potential issues which could otherwise arise from using the primary key of the current model in the cache key for the cached model. Fixes https://github.com/lamby/django-cache-toolbox/issues/22 --- cache_toolbox/relation.py | 10 ++++++++++ tests/models.py | 1 + tests/test_cached_relation.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 5509588..2dfb41a 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -78,6 +78,12 @@ class Foo(models.Model): def cache_relation(descriptor, timeout=None): rel = descriptor.related + + if not rel.field.primary_key: + # This is an internal limitation due to the way that we construct our + # cache keys. + raise ValueError("Cached relations must be the primary key") + related_name = '%s_cache' % rel.field.related_query_name() @property @@ -106,6 +112,10 @@ def get(self): try: instance = get_instance( rel.field.model, + # Note that we're using _our_ primary key here, rather than the + # primary key of the model being cached. This is ok since we + # know that its primary key is a foreign key to this model + # instance and therefore has the same value. self.pk, timeout, using=self._state.db, diff --git a/tests/models.py b/tests/models.py index ffa49cd..b9be0cd 100644 --- a/tests/models.py +++ b/tests/models.py @@ -10,6 +10,7 @@ class Bazz(models.Model): Foo, related_name='bazz', on_delete=models.CASCADE, + primary_key=True, ) value = models.IntegerField(null=True) diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index 8177096..2ca8cbb 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -1,10 +1,20 @@ from unittest import mock +from cache_toolbox import cache_relation + from django.core.cache import cache +from django.db import models from django.test import TestCase from .models import Foo, Bazz +class Another(models.Model): + foo = models.OneToOneField( + Foo, + related_name='another', + on_delete=models.CASCADE, + ) + class CachedRelationTest(TestCase): longMessage = True @@ -13,6 +23,10 @@ def setUp(self): # the cache hygenically cache.clear() + def test_requires_primary_key(self): + with self.assertRaises(ValueError): + cache_relation(Foo.another) + def test_cached_relation(self): foo = Foo.objects.create(bar='bees') From 9cd9ea9bc1e9cd4de7664f7489cee160e8e694d2 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 28 Dec 2020 17:05:56 +0000 Subject: [PATCH 64/79] Move to Python 3.x. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 024d5c7..eea99b4 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os.path from setuptools import setup, find_packages From e4f9dab5fdf57d833622b5325127d69fc1222d7b Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 28 Dec 2020 17:06:05 +0000 Subject: [PATCH 65/79] Releasing version 1.3.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eea99b4..8b9b861 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ description="Non-magical object caching for Django.", long_description=long_description, long_description_content_type='text/markdown', - version='1.2.1', + version='1.3.0', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From 116853307ebdeea6f97d091f59fc24d8dbfd4667 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Mon, 28 Dec 2020 17:13:34 +0000 Subject: [PATCH 66/79] Make the README example more correct --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 257a27e..b6691b6 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ class Foo(models.Model): ... class Bazz(models.Model): - foo = models.OneToOneField(Foo, related_name='bazz') + foo = models.OneToOneField(Foo, related_name='bazz', primary_key=True) ... # Prepare caching of a model From 3380c269ce650f6bec5045db39c74afda692148a Mon Sep 17 00:00:00 2001 From: Matt Dalton Date: Thu, 4 Feb 2021 09:10:53 +0000 Subject: [PATCH 67/79] Allow use of extended User model --- cache_toolbox/middleware.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cache_toolbox/middleware.py b/cache_toolbox/middleware.py index deea63a..95f3568 100644 --- a/cache_toolbox/middleware.py +++ b/cache_toolbox/middleware.py @@ -79,7 +79,7 @@ """ from django.contrib.auth import SESSION_KEY -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.auth.middleware import AuthenticationMiddleware from .model import cache_model @@ -87,12 +87,12 @@ class CacheBackedAuthenticationMiddleware(AuthenticationMiddleware): def __init__(self, get_response): super(CacheBackedAuthenticationMiddleware, self).__init__(get_response) - cache_model(User) + cache_model(get_user_model()) def process_request(self, request): try: # Try and construct a User instance from data stored in the cache - request.user = User.get_cached(int(request.session[SESSION_KEY])) + request.user = get_user_model().get_cached(int(request.session[SESSION_KEY])) except Exception: # Fallback to constructing the User from the database. super(CacheBackedAuthenticationMiddleware, self).process_request(request) From af04924aa8098294ea14bbce4a601fc7c1e52caa Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Thu, 4 Feb 2021 10:04:52 +0000 Subject: [PATCH 68/79] Releasing version 1.4.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8b9b861..43e92f7 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ description="Non-magical object caching for Django.", long_description=long_description, long_description_content_type='text/markdown', - version='1.3.0', + version='1.4.0', url='https://chris-lamb.co.uk/projects/django-cache-toolbox', author='Chris Lamb', From 4ff61ed97424d63b1fe978205f9e84c25a870cb6 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Mon, 28 Dec 2020 18:50:24 +0000 Subject: [PATCH 69/79] Extract de/serialisation helpers I'm about to make some changes in get_instance, so having these separate will make it easier. --- cache_toolbox/core.py | 67 +++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 7387b10..2e64de3 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -21,6 +21,42 @@ CACHE_FORMAT_VERSION = 2 + +def serialise(instance): + data = {} + for field in instance._meta.fields: + # Harmless to save, but saves space in the dictionary - we already know + # the primary key when we lookup + if field.primary_key: + continue + + # We also don't want to save any virtual fields. + if not field.concrete: + continue + + data[field.attname] = getattr(instance, field.attname) + + # Encode through Pickle, since that allows overriding and covers (most) + # Python types we'd want to serialise. + return pickle.dumps(data, protocol=-1) + + +def deserialise(model, data, pk, using): + # Try and construct instance from dictionary + instance = model(pk=pk, **pickle.loads(data)) + + # Ensure instance knows that it already exists in the database, + # otherwise we will fail any uniqueness checks when saving the + # instance. + instance._state.adding = False + + # Specify database so that instance is setup correctly. We don't + # namespace cached objects by their origin database, however. + instance._state.db = using or DEFAULT_DB_ALIAS + + return instance + + def get_instance(model, instance_or_pk, timeout=None, using=None): """ Returns the ``model`` instance with a primary key of ``instance_or_pk``. @@ -47,19 +83,7 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): if data is not None: try: - # Try and construct instance from dictionary - instance = model(pk=pk, **pickle.loads(data)) - - # Ensure instance knows that it already exists in the database, - # otherwise we will fail any uniqueness checks when saving the - # instance. - instance._state.adding = False - - # Specify database so that instance is setup correctly. We don't - # namespace cached objects by their origin database, however. - instance._state.db = using or DEFAULT_DB_ALIAS - - return instance + return deserialise(model, data, pk, using) except: # Error when deserialising - remove from the cache; we will # fallback and return the underlying instance @@ -68,25 +92,12 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): # Use the default manager so we are never filtered by a .get_query_set() instance = model._default_manager.using(using).get(pk=pk) - data = {} - for field in instance._meta.fields: - # Harmless to save, but saves space in the dictionary - we already know - # the primary key when we lookup - if field.primary_key: - continue - - # We also don't want to save any virtual fields. - if not field.concrete: - continue - - data[field.attname] = getattr(instance, field.attname) + data = serialise(instance) if timeout is None: timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT - # Encode through Pickle, since that allows overriding and covers (most) - # Python types we'd want to serialise. - cache.set(key, pickle.dumps(data, protocol=-1), timeout) + cache.set(key, data, timeout) return instance From 303ce9d125f312c89d4abea65beab9f9eb52c5af Mon Sep 17 00:00:00 2001 From: Peter Law Date: Mon, 28 Dec 2020 19:00:17 +0000 Subject: [PATCH 70/79] Extract a couple of utils I'm about to need these in the core logic. --- cache_toolbox/core.py | 8 ++++++++ cache_toolbox/relation.py | 16 +++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 2e64de3..9f408bf 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -22,6 +22,14 @@ CACHE_FORMAT_VERSION = 2 +def get_related_name(descriptor): + return '%s_cache' % descriptor.related.field.related_query_name() + + +def get_related_cache_name(related_name: str) -> str: + return '_%s_cache' % related_name + + def serialise(instance): data = {} for field in instance._meta.fields: diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 2dfb41a..a3bcb5b 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -73,7 +73,12 @@ class Foo(models.Model): from django.db.models.signals import post_save, post_delete -from .core import get_instance, delete_instance +from .core import ( + get_instance, + delete_instance, + get_related_name, + get_related_cache_name, +) def cache_relation(descriptor, timeout=None): @@ -84,7 +89,7 @@ def cache_relation(descriptor, timeout=None): # cache keys. raise ValueError("Cached relations must be the primary key") - related_name = '%s_cache' % rel.field.related_query_name() + related_name = get_related_name(descriptor) @property def get(self): @@ -93,8 +98,9 @@ def get(self): return descriptor.__get__(self) # Lookup cached instance + related_cache_name = get_related_cache_name(related_name) try: - instance = getattr(self, '_%s_cache' % related_name) + instance = getattr(self, related_cache_name) except AttributeError: # no local cache pass @@ -120,9 +126,9 @@ def get(self): timeout, using=self._state.db, ) - setattr(self, '_%s_cache' % related_name, instance) + setattr(self, related_cache_name, instance) except rel.related_model.DoesNotExist: - setattr(self, '_%s_cache' % related_name, None) + setattr(self, related_cache_name, None) raise descriptor.RelatedObjectDoesNotExist( "%s has no %s." % ( rel.model.__name__, From a913b62fef9ee501ee81b3bc656d789535f6ec7b Mon Sep 17 00:00:00 2001 From: Peter Law Date: Mon, 28 Dec 2020 19:19:25 +0000 Subject: [PATCH 71/79] Support always fetching some relations when loading a model This will be particularly useful when applied to relations on the User model, which is loaded by our middleware. In that case, this allows for efficient loading of several models upfront. Fixes https://github.com/lamby/django-cache-toolbox/issues/21 --- cache_toolbox/core.py | 81 ++++++++++++++++++++++++++++---- cache_toolbox/relation.py | 6 ++- tests/models.py | 17 +++++++ tests/test_cached_get_related.py | 66 ++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 tests/test_cached_get_related.py diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 9f408bf..7934ad3 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -22,6 +22,14 @@ CACHE_FORMAT_VERSION = 2 +def setattrdefault(obj, name, default): + try: + return getattr(obj, name) + except AttributeError: + setattr(obj, name, default) + return default + + def get_related_name(descriptor): return '%s_cache' % descriptor.related.field.related_query_name() @@ -30,6 +38,14 @@ def get_related_cache_name(related_name: str) -> str: return '_%s_cache' % related_name +def add_always_fetch_relation(descriptor): + setattrdefault( + descriptor.related.model, + '_cache_fetch_related', + [], + ).append(descriptor) + + def serialise(instance): data = {} for field in instance._meta.fields: @@ -86,28 +102,75 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): """ pk = getattr(instance_or_pk, 'pk', instance_or_pk) - key = instance_key(model, instance_or_pk) - data = cache.get(key) - if data is not None: + primary_model = model + descriptors = getattr(primary_model, '_cache_fetch_related', ()) + models = [model, *(d.related.field.model for d in descriptors)] + # Note: we're assuming that the relations are primary key foreign keys, and + # so all have the same primary key. This matches the assumption which + # `cache_relation` makes. + keys_to_models = { + instance_key(model, instance_or_pk): model + for model in models + } + + data_map = cache.get_many(keys_to_models.keys()) + instance_map = {} + + if data_map: try: - return deserialise(model, data, pk, using) + for key, data in data_map.items(): + model = keys_to_models[key] + instance_map[key] = deserialise(model, data, pk, using) except: # Error when deserialising - remove from the cache; we will # fallback and return the underlying instance - cache.delete(key) + cache.delete_many(keys_to_models.keys()) + + else: + key = instance_key(primary_model, instance_or_pk) + primary_instance = instance_map[key] + + for descriptor in descriptors: + related_instance = instance_map.get(instance_key( + descriptor.related.field.model, + instance_or_pk, + )) + related_cache_name = get_related_cache_name( + get_related_name(descriptor), + ) + setattr(primary_instance, related_cache_name, related_instance) + + return primary_instance + + related_names = [d.related.field.related_query_name() for d in descriptors] # Use the default manager so we are never filtered by a .get_query_set() - instance = model._default_manager.using(using).get(pk=pk) + primary_instance = primary_model._default_manager.using( + using, + ).select_related( + *related_names, + ).get(pk=pk) + + instances = [ + primary_instance, + *(getattr(primary_instance, x, None) for x in related_names), + ] + + cache_data = {} + for instance in instances: + if instance is None: + continue - data = serialise(instance) + key = instance_key(instance._meta.model, instance) + cache_data[key] = serialise(instance) if timeout is None: timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT - cache.set(key, data, timeout) + cache.set_many(cache_data, timeout) - return instance + return primary_instance def delete_instance(model, *instance_or_pk): diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index a3bcb5b..4924b2e 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -78,10 +78,11 @@ class Foo(models.Model): delete_instance, get_related_name, get_related_cache_name, + add_always_fetch_relation, ) -def cache_relation(descriptor, timeout=None): +def cache_relation(descriptor, timeout=None, *, always_fetch=False): rel = descriptor.related if not rel.field.primary_key: @@ -89,6 +90,9 @@ def cache_relation(descriptor, timeout=None): # cache keys. raise ValueError("Cached relations must be the primary key") + if always_fetch: + add_always_fetch_relation(descriptor) + related_name = get_related_name(descriptor) @property diff --git a/tests/models.py b/tests/models.py index b9be0cd..88c88e6 100644 --- a/tests/models.py +++ b/tests/models.py @@ -17,3 +17,20 @@ class Bazz(models.Model): cache_model(Foo) cache_relation(Foo.bazz) + + +class ToLoad(models.Model): + name = models.TextField() + +class AlwaysRelated(models.Model): + to_load = models.OneToOneField( + ToLoad, + related_name='always_related', + on_delete=models.CASCADE, + primary_key=True, + ) + + value = models.IntegerField(null=True) + +cache_model(ToLoad) +cache_relation(ToLoad.always_related, always_fetch=True) diff --git a/tests/test_cached_get_related.py b/tests/test_cached_get_related.py new file mode 100644 index 0000000..c25230e --- /dev/null +++ b/tests/test_cached_get_related.py @@ -0,0 +1,66 @@ +from django.core.cache import cache +from django.test import TransactionTestCase + +from .models import AlwaysRelated, ToLoad + + +# Use `TransactionTestCase` so that our `on_commit` actions happen when we expect. +class CachedGetRelatedTest(TransactionTestCase): + def setUp(self): + self.to_load = ToLoad.objects.create(name='bees') + self.always_related = AlwaysRelated.objects.create(to_load=self.to_load) + self._populate_cache() + + def _populate_cache(self): + ToLoad.get_cached(self.to_load.pk) + + def test_cached_get(self): + # Get from the cache + cached_object = ToLoad.get_cached(self.to_load.pk) + + # Validate that we're using the value we pre-loaded + cache.clear() + + with self.assertNumQueries(0): + self.assertEqual(self.to_load.name, cached_object.name) + self.assertEqual( + self.always_related, + cached_object.always_related_cache, + ) + + def test_cached_get_no_relation(self): + self.always_related.delete() + + # Get from the cache + cached_object = ToLoad.get_cached(self.to_load.pk) + + self.assertEqual(self.to_load.name, cached_object.name) + + with self.assertNumQueries(0): + with self.assertRaises(AlwaysRelated.DoesNotExist): + cached_object.always_related_cache + + # Sanity check + with self.assertNumQueries(1): + with self.assertRaises(AlwaysRelated.DoesNotExist): + cached_object.always_related + + def test_cached_get_no_relation_no_cache(self): + self.always_related.delete() + cache.clear() + + # Attempt to load, should fall back to the database and should handle + # the lack of the related instance. + with self.assertNumQueries(1): + cached_object = ToLoad.get_cached(self.to_load.pk) + + self.assertEqual(self.to_load.name, cached_object.name) + + with self.assertNumQueries(0): + with self.assertRaises(AlwaysRelated.DoesNotExist): + cached_object.always_related_cache + + # Sanity check + with self.assertNumQueries(0): + with self.assertRaises(AlwaysRelated.DoesNotExist): + cached_object.always_related From 3dfbf0708d6bcc9cadf0bd0a44f5d99f589e55e0 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sun, 21 Feb 2021 16:39:27 +0000 Subject: [PATCH 72/79] Apply 'black'. --- cache_toolbox/app_settings.py | 2 +- cache_toolbox/core.py | 51 +++++++++++---------- cache_toolbox/middleware.py | 5 +- cache_toolbox/model.py | 3 +- cache_toolbox/relation.py | 11 +++-- cache_toolbox/templatetags/cache_toolbox.py | 8 +++- runtests.py | 4 +- setup.py | 25 ++++------ tests/models.py | 9 +++- tests/test_cached_get.py | 6 +-- tests/test_cached_get_related.py | 2 +- tests/test_cached_relation.py | 32 +++++++------ tests/test_settings.py | 10 ++-- 13 files changed, 93 insertions(+), 75 deletions(-) diff --git a/cache_toolbox/app_settings.py b/cache_toolbox/app_settings.py index cb19bd3..fe42215 100644 --- a/cache_toolbox/app_settings.py +++ b/cache_toolbox/app_settings.py @@ -3,6 +3,6 @@ # Default cache timeout CACHE_TOOLBOX_DEFAULT_TIMEOUT = getattr( settings, - 'CACHE_TOOLBOX_DEFAULT_TIMEOUT', + "CACHE_TOOLBOX_DEFAULT_TIMEOUT", 60 * 60 * 24 * 7, ) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 7934ad3..f14ecc3 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -31,17 +31,17 @@ def setattrdefault(obj, name, default): def get_related_name(descriptor): - return '%s_cache' % descriptor.related.field.related_query_name() + return "%s_cache" % descriptor.related.field.related_query_name() def get_related_cache_name(related_name: str) -> str: - return '_%s_cache' % related_name + return "_%s_cache" % related_name def add_always_fetch_relation(descriptor): setattrdefault( descriptor.related.model, - '_cache_fetch_related', + "_cache_fetch_related", [], ).append(descriptor) @@ -101,18 +101,15 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): True """ - pk = getattr(instance_or_pk, 'pk', instance_or_pk) + pk = getattr(instance_or_pk, "pk", instance_or_pk) primary_model = model - descriptors = getattr(primary_model, '_cache_fetch_related', ()) + descriptors = getattr(primary_model, "_cache_fetch_related", ()) models = [model, *(d.related.field.model for d in descriptors)] # Note: we're assuming that the relations are primary key foreign keys, and # so all have the same primary key. This matches the assumption which # `cache_relation` makes. - keys_to_models = { - instance_key(model, instance_or_pk): model - for model in models - } + keys_to_models = {instance_key(model, instance_or_pk): model for model in models} data_map = cache.get_many(keys_to_models.keys()) instance_map = {} @@ -132,10 +129,12 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): primary_instance = instance_map[key] for descriptor in descriptors: - related_instance = instance_map.get(instance_key( - descriptor.related.field.model, - instance_or_pk, - )) + related_instance = instance_map.get( + instance_key( + descriptor.related.field.model, + instance_or_pk, + ) + ) related_cache_name = get_related_cache_name( get_related_name(descriptor), ) @@ -146,11 +145,15 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): related_names = [d.related.field.related_query_name() for d in descriptors] # Use the default manager so we are never filtered by a .get_query_set() - primary_instance = primary_model._default_manager.using( - using, - ).select_related( - *related_names, - ).get(pk=pk) + primary_instance = ( + primary_model._default_manager.using( + using, + ) + .select_related( + *related_names, + ) + .get(pk=pk) + ) instances = [ primary_instance, @@ -195,9 +198,11 @@ def delete_instance(model, *instance_or_pk): # but before the cache has cleared, that is better than leaving the cache # incorrect until the model is next updated. - transaction.on_commit(lambda: cache.delete_many( - [instance_key(model, x) for x in instance_or_pk], - )) + transaction.on_commit( + lambda: cache.delete_many( + [instance_key(model, x) for x in instance_or_pk], + ) + ) def instance_key(model, instance_or_pk): @@ -205,9 +210,9 @@ def instance_key(model, instance_or_pk): Returns the cache key for this (model, instance) pair. """ - return 'cache.%d:%s.%s:%s' % ( + return "cache.%d:%s.%s:%s" % ( CACHE_FORMAT_VERSION, model._meta.app_label, model._meta.model_name, - getattr(instance_or_pk, 'pk', instance_or_pk), + getattr(instance_or_pk, "pk", instance_or_pk), ) diff --git a/cache_toolbox/middleware.py b/cache_toolbox/middleware.py index 95f3568..346e28f 100644 --- a/cache_toolbox/middleware.py +++ b/cache_toolbox/middleware.py @@ -84,6 +84,7 @@ from .model import cache_model + class CacheBackedAuthenticationMiddleware(AuthenticationMiddleware): def __init__(self, get_response): super(CacheBackedAuthenticationMiddleware, self).__init__(get_response) @@ -92,7 +93,9 @@ def __init__(self, get_response): def process_request(self, request): try: # Try and construct a User instance from data stored in the cache - request.user = get_user_model().get_cached(int(request.session[SESSION_KEY])) + request.user = get_user_model().get_cached( + int(request.session[SESSION_KEY]) + ) except Exception: # Fallback to constructing the User from the database. super(CacheBackedAuthenticationMiddleware, self).process_request(request) diff --git a/cache_toolbox/model.py b/cache_toolbox/model.py index 8ac8f0d..e45ae4f 100644 --- a/cache_toolbox/model.py +++ b/cache_toolbox/model.py @@ -58,8 +58,9 @@ class Foo(models.Model): from .core import get_instance, delete_instance + def cache_model(model, timeout=None): - if hasattr(model, 'get_cached'): + if hasattr(model, "get_cached"): # Already patched return diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index 4924b2e..4a65d8a 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -112,7 +112,8 @@ def get(self): if instance is None: # we (locally) cached that there is no model raise descriptor.RelatedObjectDoesNotExist( - "%s has no %s." % ( + "%s has no %s." + % ( rel.model.__name__, related_name, ), @@ -134,13 +135,15 @@ def get(self): except rel.related_model.DoesNotExist: setattr(self, related_cache_name, None) raise descriptor.RelatedObjectDoesNotExist( - "%s has no %s." % ( + "%s has no %s." + % ( rel.model.__name__, related_name, ), ) return instance + setattr(rel.model, related_name, get) # Clearing cache @@ -155,8 +158,8 @@ def clear_pk(cls, *instances_or_pk): def clear_cache(sender, instance, *args, **kwargs): delete_instance(rel.related_model, instance) - setattr(rel.model, '%s_clear' % related_name, clear) - setattr(rel.model, '%s_clear_pk' % related_name, clear_pk) + setattr(rel.model, "%s_clear" % related_name, clear) + setattr(rel.model, "%s_clear_pk" % related_name, clear_pk) post_save.connect(clear_cache, sender=rel.related_model, weak=False) post_delete.connect(clear_cache, sender=rel.related_model, weak=False) diff --git a/cache_toolbox/templatetags/cache_toolbox.py b/cache_toolbox/templatetags/cache_toolbox.py index 5c4469e..437972f 100644 --- a/cache_toolbox/templatetags/cache_toolbox.py +++ b/cache_toolbox/templatetags/cache_toolbox.py @@ -4,6 +4,7 @@ register = template.Library() + class CacheNode(Node): def __init__(self, nodelist, expire_time, key): self.nodelist = nodelist @@ -20,6 +21,7 @@ def render(self, context): cache.set(key, value, expire_time) return value + @register.tag def cachedeterministic(parser, token): """ @@ -34,20 +36,22 @@ def cachedeterministic(parser, token): {% endcachedeterministic %} """ - nodelist = parser.parse(('endcachedeterministic',)) + nodelist = parser.parse(("endcachedeterministic",)) parser.delete_first_token() tokens = token.contents.split() if len(tokens) != 3: raise TemplateSyntaxError(u"'%r' tag requires 2 arguments." % tokens[0]) return CacheNode(nodelist, tokens[1], tokens[2]) + class ShowIfCachedNode(Node): def __init__(self, key): self.key = Variable(key) def render(self, context): key = self.key.resolve(context) - return cache.get(key) or '' + return cache.get(key) or "" + @register.tag def showifcached(parser, token): diff --git a/runtests.py b/runtests.py index 3361623..7c9cc8a 100755 --- a/runtests.py +++ b/runtests.py @@ -6,8 +6,8 @@ from django.conf import settings from django.test.utils import get_runner -if __name__ == '__main__': - os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_settings' +if __name__ == "__main__": + os.environ["DJANGO_SETTINGS_MODULE"] = "tests.test_settings" django.setup() TestRunner = get_runner(settings) test_runner = TestRunner() diff --git a/setup.py b/setup.py index 43e92f7..7a844ec 100755 --- a/setup.py +++ b/setup.py @@ -4,24 +4,19 @@ from setuptools import setup, find_packages my_dir = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(my_dir, 'README.md')) as f: +with open(os.path.join(my_dir, "README.md")) as f: long_description = f.read() setup( - name='django-cache-toolbox', + name="django-cache-toolbox", description="Non-magical object caching for Django.", long_description=long_description, - long_description_content_type='text/markdown', - version='1.4.0', - url='https://chris-lamb.co.uk/projects/django-cache-toolbox', - - author='Chris Lamb', - author_email='chris@chris-lamb.co.uk', - license='BSD', - - packages=find_packages(exclude=('tests',)), - - install_requires=( - "Django>=1.9", - ), + long_description_content_type="text/markdown", + version="1.4.0", + url="https://chris-lamb.co.uk/projects/django-cache-toolbox", + author="Chris Lamb", + author_email="chris@chris-lamb.co.uk", + license="BSD", + packages=find_packages(exclude=("tests",)), + install_requires=("Django>=1.9",), ) diff --git a/tests/models.py b/tests/models.py index 88c88e6..d0722a4 100644 --- a/tests/models.py +++ b/tests/models.py @@ -2,19 +2,22 @@ from django.db import models + class Foo(models.Model): bar = models.TextField() + class Bazz(models.Model): foo = models.OneToOneField( Foo, - related_name='bazz', + related_name="bazz", on_delete=models.CASCADE, primary_key=True, ) value = models.IntegerField(null=True) + cache_model(Foo) cache_relation(Foo.bazz) @@ -22,15 +25,17 @@ class Bazz(models.Model): class ToLoad(models.Model): name = models.TextField() + class AlwaysRelated(models.Model): to_load = models.OneToOneField( ToLoad, - related_name='always_related', + related_name="always_related", on_delete=models.CASCADE, primary_key=True, ) value = models.IntegerField(null=True) + cache_model(ToLoad) cache_relation(ToLoad.always_related, always_fetch=True) diff --git a/tests/test_cached_get.py b/tests/test_cached_get.py index b280240..697306b 100644 --- a/tests/test_cached_get.py +++ b/tests/test_cached_get.py @@ -5,7 +5,7 @@ # Use `TransactionTestCase` so that our `on_commit` actions happen when we expect. class CachedGetTest(TransactionTestCase): def setUp(self): - self.foo = Foo.objects.create(bar='bees') + self.foo = Foo.objects.create(bar="bees") self._populate_cache() def _populate_cache(self): @@ -18,12 +18,12 @@ def test_cached_get(self): self.assertEqual(self.foo.bar, cached_object.bar) def test_cache_invalidated_on_update(self): - self.foo.bar = 'quux' + self.foo.bar = "quux" self.foo.save() self._populate_cache() - self.assertEqual(Foo.get_cached(self.foo.pk).bar, 'quux') + self.assertEqual(Foo.get_cached(self.foo.pk).bar, "quux") def test_cache_invalidated_on_delete(self): pk = self.foo.pk diff --git a/tests/test_cached_get_related.py b/tests/test_cached_get_related.py index c25230e..d26bd4d 100644 --- a/tests/test_cached_get_related.py +++ b/tests/test_cached_get_related.py @@ -7,7 +7,7 @@ # Use `TransactionTestCase` so that our `on_commit` actions happen when we expect. class CachedGetRelatedTest(TransactionTestCase): def setUp(self): - self.to_load = ToLoad.objects.create(name='bees') + self.to_load = ToLoad.objects.create(name="bees") self.always_related = AlwaysRelated.objects.create(to_load=self.to_load) self._populate_cache() diff --git a/tests/test_cached_relation.py b/tests/test_cached_relation.py index 2ca8cbb..1572302 100644 --- a/tests/test_cached_relation.py +++ b/tests/test_cached_relation.py @@ -8,13 +8,15 @@ from .models import Foo, Bazz + class Another(models.Model): foo = models.OneToOneField( Foo, - related_name='another', + related_name="another", on_delete=models.CASCADE, ) + class CachedRelationTest(TestCase): longMessage = True @@ -28,7 +30,7 @@ def test_requires_primary_key(self): cache_relation(Foo.another) def test_cached_relation(self): - foo = Foo.objects.create(bar='bees') + foo = Foo.objects.create(bar="bees") Bazz.objects.create(foo=foo, value=10) @@ -41,41 +43,41 @@ def test_cached_relation(self): self.assertEqual(cached_object.value, 10) self.assertTrue( - hasattr(foo, 'bazz'), + hasattr(foo, "bazz"), "Foo should have 'bazz' attribute", ) self.assertTrue( - hasattr(foo, 'bazz_cache'), + hasattr(foo, "bazz_cache"), "Foo should have 'bazz_cache' attribute", ) def test_cached_relation_not_present_hasattr(self): - foo = Foo.objects.create(bar='bees_2') + foo = Foo.objects.create(bar="bees_2") self.assertFalse( - hasattr(foo, 'bazz_cache'), + hasattr(foo, "bazz_cache"), "Foo should not have 'bazz_cache' attribute (empty cache)", ) self.assertFalse( - hasattr(foo, 'bazz_cache'), + hasattr(foo, "bazz_cache"), "Foo should not have 'bazz_cache' attribute (warm cache; before natural access)", ) # sanity check self.assertFalse( - hasattr(foo, 'bazz'), + hasattr(foo, "bazz"), "Foo should not have 'bazz' attribute", ) self.assertFalse( - hasattr(foo, 'bazz_cache'), + hasattr(foo, "bazz_cache"), "Foo should not have 'bazz_cache' attribute (warm cache; after natural access)", ) def test_cached_relation_not_present_exception(self): - foo = Foo.objects.create(bar='bees_3') + foo = Foo.objects.create(bar="bees_3") with self.assertRaises(Bazz.DoesNotExist) as cm: foo.bazz_cache @@ -87,10 +89,10 @@ def test_cached_relation_not_present_exception(self): ) def test_cached_missing_relation_uses_select_related(self): - foo = Foo.objects.create(bar='bees') + foo = Foo.objects.create(bar="bees") with self.assertNumQueries(1): - foo = Foo.objects.select_related('bazz').get(pk=foo.pk) + foo = Foo.objects.select_related("bazz").get(pk=foo.pk) with self.assertNumQueries(0): with self.assertRaises(Bazz.DoesNotExist): @@ -101,7 +103,7 @@ def test_cached_missing_relation_cached_locally(self): # the same as it would cache the Bazz instance if there was one. Mimic # that behaviour in order to have comparable querying behaviour. - foo = Foo.objects.create(bar='bees') + foo = Foo.objects.create(bar="bees") with self.assertNumQueries(1): foo = Foo.objects.get(pk=foo.pk) @@ -125,14 +127,14 @@ def test_cached_missing_relation_cached_locally(self): foo.bazz_cache def test_get_instance_error_doesnt_have_side_effect_issues(self): - foo = Foo.objects.create(bar='bees') + foo = Foo.objects.create(bar="bees") class DummyException(Exception): pass # Validate that the underlying error is passed through, without any # other errors happening... - with mock.patch('cache_toolbox.core.cache.get', side_effect=DummyException): + with mock.patch("cache_toolbox.core.cache.get", side_effect=DummyException): with self.assertRaises(DummyException): foo.bazz_cache diff --git a/tests/test_settings.py b/tests/test_settings.py index 0a1940c..51631c7 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,10 +1,10 @@ -SECRET_KEY = 'fake-key' +SECRET_KEY = "fake-key" INSTALLED_APPS = [ - 'tests', + "tests", ] DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", }, } From c6f69c45d23add4d027a334aa093597284d94309 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sun, 21 Feb 2021 16:39:37 +0000 Subject: [PATCH 73/79] Releasing version 1.5.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a844ec..92238a9 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ description="Non-magical object caching for Django.", long_description=long_description, long_description_content_type="text/markdown", - version="1.4.0", + version='1.5.0', url="https://chris-lamb.co.uk/projects/django-cache-toolbox", author="Chris Lamb", author_email="chris@chris-lamb.co.uk", From f404b125897242c9238bb84803842530a38fe196 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 25 Feb 2021 19:35:22 +0000 Subject: [PATCH 74/79] Avoid unrelated select-relateds It turns out that select_related() when called without arguments (as might happen if we didn't have anything in our related_names collection) will try to select related any relations which are non-nullable. Since we don't want that, guard the addition of the select_related to our queryset. --- cache_toolbox/core.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index f14ecc3..cd3102f 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -145,15 +145,11 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): related_names = [d.related.field.related_query_name() for d in descriptors] # Use the default manager so we are never filtered by a .get_query_set() - primary_instance = ( - primary_model._default_manager.using( - using, - ) - .select_related( - *related_names, - ) - .get(pk=pk) - ) + queryset = primary_model._default_manager.using(using) + if related_names: + # NB: select_related without args selects all it can find, which we don't want. + queryset = queryset.select_related(*related_names) + primary_instance = queryset.get(pk=pk) instances = [ primary_instance, From 97c9296fdb8234f9d585fbe4826ae83f64119bf6 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 25 Feb 2021 19:18:36 +0000 Subject: [PATCH 75/79] Wrap these collections in tuples for compatibility Django itself seems fine with these being dict_keys, however the django-perf-rec library isn't. While it's their bug, fixing here is easy enough. --- cache_toolbox/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index cd3102f..a46baee 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -111,7 +111,7 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): # `cache_relation` makes. keys_to_models = {instance_key(model, instance_or_pk): model for model in models} - data_map = cache.get_many(keys_to_models.keys()) + data_map = cache.get_many(tuple(keys_to_models.keys())) instance_map = {} if data_map: @@ -122,7 +122,7 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): except: # Error when deserialising - remove from the cache; we will # fallback and return the underlying instance - cache.delete_many(keys_to_models.keys()) + cache.delete_many(tuple(keys_to_models.keys())) else: key = instance_key(primary_model, instance_or_pk) From 9a55e1283fbdb06ee42db2e5019da5245bee6d75 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 25 Feb 2021 20:21:29 +0000 Subject: [PATCH 76/79] Cope with only some of the related models actually being loaded The main thing this fixes is the case where the primary model is not returned in the cached results. However it also changes the handling in the case where one of the related models isn't found. Previously the former would error and the latter would silently assign `None` into the local relation cache field (and would thus never be populated even if there should have been a value there). --- cache_toolbox/core.py | 6 ++--- tests/test_cached_get_related.py | 46 +++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index a46baee..86f5c82 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -114,7 +114,7 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): data_map = cache.get_many(tuple(keys_to_models.keys())) instance_map = {} - if data_map: + if data_map.keys() == keys_to_models.keys(): try: for key, data in data_map.items(): model = keys_to_models[key] @@ -129,12 +129,12 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): primary_instance = instance_map[key] for descriptor in descriptors: - related_instance = instance_map.get( + related_instance = instance_map[ instance_key( descriptor.related.field.model, instance_or_pk, ) - ) + ] related_cache_name = get_related_cache_name( get_related_name(descriptor), ) diff --git a/tests/test_cached_get_related.py b/tests/test_cached_get_related.py index d26bd4d..fb766cc 100644 --- a/tests/test_cached_get_related.py +++ b/tests/test_cached_get_related.py @@ -1,3 +1,5 @@ +from cache_toolbox.core import delete_instance + from django.core.cache import cache from django.test import TransactionTestCase @@ -28,6 +30,45 @@ def test_cached_get(self): cached_object.always_related_cache, ) + def test_cached_get_primary_absent_from_cache(self): + # Remove the instance from the cache, for example as a result of it + # being saved. + delete_instance(ToLoad, self.to_load) + + with self.assertNumQueries(1): + # Get from the cache - this will involve actually loading from the + # database. + cached_object = ToLoad.get_cached(self.to_load.pk) + + # Validate that we're using the value we pre-loaded + cache.clear() + + with self.assertNumQueries(0): + self.assertEqual(self.to_load.name, cached_object.name) + self.assertEqual( + self.always_related, + cached_object.always_related_cache, + ) + + def test_cached_get_relation_absent_from_cache(self): + # Remove the instance from the cache, for example as a result of it + # being saved. + delete_instance(AlwaysRelated, self.always_related) + + # Attempt to load, should fall back to the database due to the lack of + # the related instance in the cache results. + with self.assertNumQueries(1): + cached_object = ToLoad.get_cached(self.to_load.pk) + + self.assertEqual(self.to_load.name, cached_object.name) + + with self.assertNumQueries(0): + self.assertEqual(self.to_load.name, cached_object.name) + self.assertEqual( + self.always_related, + cached_object.always_related_cache, + ) + def test_cached_get_no_relation(self): self.always_related.delete() @@ -41,9 +82,8 @@ def test_cached_get_no_relation(self): cached_object.always_related_cache # Sanity check - with self.assertNumQueries(1): - with self.assertRaises(AlwaysRelated.DoesNotExist): - cached_object.always_related + with self.assertRaises(AlwaysRelated.DoesNotExist): + cached_object.always_related def test_cached_get_no_relation_no_cache(self): self.always_related.delete() From 544966d30c740088670c31ddf1712fb1cdf7735c Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 26 Feb 2021 08:55:00 +0000 Subject: [PATCH 77/79] Releasing version 1.6.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92238a9..7743519 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ description="Non-magical object caching for Django.", long_description=long_description, long_description_content_type="text/markdown", - version='1.5.0', + version='1.6.0', url="https://chris-lamb.co.uk/projects/django-cache-toolbox", author="Chris Lamb", author_email="chris@chris-lamb.co.uk", From 303b38f972f89b0bd9ac6155350056c4958655ff Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Thu, 16 Jun 2022 09:52:56 +0100 Subject: [PATCH 78/79] Configure setup.cfg to generate a wheel during the release process. --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2be6836 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = True From cdca670acacaaf3e9af6921529d2321b8b59a177 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Thu, 16 Jun 2022 09:53:15 +0100 Subject: [PATCH 79/79] Releasing version 1.6.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7743519..479a529 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ description="Non-magical object caching for Django.", long_description=long_description, long_description_content_type="text/markdown", - version='1.6.0', + version='1.6.1', url="https://chris-lamb.co.uk/projects/django-cache-toolbox", author="Chris Lamb", author_email="chris@chris-lamb.co.uk",