diff --git a/.codecov.yml b/.codecov.yml index 2c5519267..219845f24 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,6 +1,6 @@ codecov: notify: - require_ci_to_pass: yes + after_n_builds: 4 coverage: status: diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..16fede88f --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +concurrency = multiprocessing diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..50ce4aaa1 --- /dev/null +++ b/.flake8 @@ -0,0 +1,22 @@ +[flake8] +enable-extensions = G +max-doc-length = 90 +max-line-length = 90 +select = A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z,B901,B902,B903,B950 +# E226: Missing whitespace around arithmetic operators can help group things together. +# E501,W505: Superseeded by B950 (from Bugbear) +# E722: Superseeded by B001 (from Bugbear) +# W503: Mutually exclusive with W504. +ignore = E226,E501,E722,W503,W505 +per-file-ignores = + # S*: Bandit security checks not useful in tests. + tests/*:S + +# flake8-import-order +application-import-names = aiocache +import-order-style = pycharm + +# flake8-quotes +inline-quotes = " +# flake8-requirements +requirements-file = requirements-dev.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..b9fb8a6e0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml new file mode 100644 index 000000000..288e9faff --- /dev/null +++ b/.github/workflows/auto-merge.yml @@ -0,0 +1,22 @@ +name: Dependabot auto-merge +on: pull_request_target + +permissions: + pull-requests: write + contents: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v1.6.0 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Enable auto-merge for Dependabot PRs + run: gh pr merge --auto --squash "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..b78f4a040 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,160 @@ +name: CI + +on: + push: + branches: + - master + - '[0-9].[0-9]+' # matches to backport branches, e.g. 3.6 + tags: [ 'v*' ] + pull_request: + branches: + - master + - '[0-9].[0-9]+' + +jobs: + lint: + name: Linter + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + - name: Pre-Commit hooks + uses: pre-commit/action@v3.0.0 + - name: Install dependencies + uses: py-actions/py-dependency-install@v4 + with: + path: requirements-dev.txt + - name: Install itself + run: | + pip install . + - name: Run linter + run: | + make lint + - name: Prepare twine checker + run: | + pip install -U build twine wheel + python -m build + - name: Run twine checker + run: | + twine check dist/* + + test: + name: Test + strategy: + matrix: + os: [ubuntu] + pyver: ['3.8', '3.9', '3.10', '3.11'] + redis: ['latest'] + ujson: [''] + include: + - os: ubuntu + pyver: pypy-3.8 + redis: 'latest' + - os: ubuntu + pyver: '3.9' + redis: '5.0.14' + - os: ubuntu + pyver: '3.9' + redis: 'latest' + ujson: 'ujson' + services: + redis: + image: redis:${{ matrix.redis }} + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + memcached: + image: memcached + ports: + - 11211:11211 + runs-on: ${{ matrix.os }}-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python ${{ matrix.pyver }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyver }} + allow-prereleases: true + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + - name: Install ujson + if: ${{ matrix.ujson == 'ujson' }} + run: pip install ujson + - name: Install dependencies + uses: py-actions/py-dependency-install@v4 + with: + path: requirements.txt + - name: Run unittests + env: + COLOR: 'yes' + run: pytest tests --cov-report xml --cov-report html + - name: Run functional tests + run: bash examples/run_all.sh + - name: Uninstall optional backends + run: pip uninstall -y aiomcache redis + - name: Run unittests with minimal backend set + env: + COLOR: 'yes' + run: | + pytest --cov-report xml --cov-report html --cov-append tests/acceptance tests/ut -m "not memcached and not redis" --ignore "tests/ut/backends/test_memcached.py" --ignore "tests/ut/backends/test_redis.py" + - name: Produce coverage report + run: python -m coverage xml + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unit + fail_ci_if_error: false + + check: # This job does nothing and is only used for the branch protection + if: always() + + needs: [lint, test] + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + deploy: + name: Deploy + environment: release + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') + needs: [check] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Update pip, wheel, setuptools, build, twine + run: | + python -m pip install -U pip wheel setuptools build twine + - name: Build dists + run: | + python -m build + - name: Make Release + uses: aio-libs/create-release@v1.6.6 + with: + changes_file: CHANGES.rst + name: aiocache + version_file: aiocache/__init__.py + github_token: ${{ secrets.GITHUB_TOKEN }} + pypi_token: ${{ secrets.PYPI_API_TOKEN }} + dist_dir: dist + fix_issue_regex: "`#(\\d+) `" + fix_issue_repl: "(#\\1)" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..f3bdc0304 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: "28 18 * * 3" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 000000000..ceba7e6a5 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,30 @@ +[mypy] +files = aiocache, examples, tests +#check_untyped_defs = True +follow_imports_for_stubs = True +#disallow_any_decorated = True +disallow_any_generics = True +disallow_incomplete_defs = True +disallow_subclassing_any = True +#disallow_untyped_calls = True +disallow_untyped_decorators = True +#disallow_untyped_defs = True +implicit_reexport = False +no_implicit_optional = True +show_error_codes = True +strict_equality = True +warn_incomplete_stub = True +warn_redundant_casts = True +warn_unreachable = True +warn_unused_ignores = True +disallow_any_unimported = True +#warn_return_any = True + +[mypy-tests.*] +disallow_any_decorated = False +disallow_untyped_calls = False +disallow_untyped_defs = False + + +[mypy-msgpack.*] +ignore_missing_imports = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..efe223296 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files +- repo: https://github.com/PyCQA/flake8 + rev: '4.0.1' + hooks: + - id: flake8 + exclude: "^docs/" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ac929f716..000000000 --- a/.travis.yml +++ /dev/null @@ -1,94 +0,0 @@ -language: python -cache: pip - -install: - - pip install tox - -services: - - redis-server - - memcached - -jobs: - include: - - stage: syntax - script: tox - python: 3.8 - env: TOXENV=syntax - - script: tox - python: 3.8 - env: TOXENV=docs-html - - - stage: test - script: tox - env: TOXENV=py36,codecov - python: 3.6 - - script: tox - env: TOXENV=py36,codecov - python: 3.6-dev - - script: tox - env: TOXENV=py36-ujson,codecov - python: 3.6 - - script: tox - env: TOXENV=py36-deps-lowest,codecov - python: 3.6 - - script: tox - env: TOXENV=py36-deps-devel,codecov - python: 3.6 - - - script: tox - env: TOXENV=py37,codecov - python: 3.7 - - script: tox - env: TOXENV=py37,codecov - python: 3.7-dev - - script: tox - env: TOXENV=py37-ujson,codecov - python: 3.7 - - script: tox - env: TOXENV=py37-deps-lowest,codecov - python: 3.7 - - script: tox - env: TOXENV=py37-deps-devel,codecov - python: 3.7 - - - script: tox - env: TOXENV=py38,codecov - python: 3.8 - - script: tox - env: TOXENV=py38,codecov - python: 3.8-dev - - script: tox - env: TOXENV=py38-ujson,codecov - python: 3.8 - - script: tox - env: TOXENV=py38-deps-lowest,codecov - python: 3.8 - - script: tox - env: TOXENV=py38-deps-devel,codecov - python: 3.8 - - - script: tox - env: TOXENV=py39,codecov - python: 3.9-dev - - script: tox - env: TOXENV=py39-ujson,codecov - python: 3.9-dev - - script: tox - env: TOXENV=py39-deps-lowest,codecov - python: 3.9-dev - - script: tox - env: TOXENV=py39-deps-devel,codecov - python: 3.9-dev - - - stage: deploy - script: skip - python: 3.8 - deploy: &pypi - provider: pypi - user: blck - password: - secure: 6EzuQM9MJeyiBbD0KGHWhRxW3Q5Z0iWSTPwA3DNzfDKF3iKlz27FFauRIKvv8nnq3FhHos5TziSFWnhNqNNRqynlkJM7n+YGH6HwXQjFJBCwa4PgaSGJxAnAjaq0l2E6CGxzGFG4Bef52IzJiFS5njjjK9jnpbkGKMm+LQ8QFXmLjPQhHnRYEDTE7zrGR6Q/u9WuB6J9Rhsec0ncJm9E14dWui9ap7+bEfBbCADvxZrSNBtSkpq0IN0ui0ZLoSkingLVkXFg1rIkIxxmYwsVKsPi+IvF3Ig8XxvhcqV2+420SgbM2yMGUmx6cW78vCcc3LikeeGKVGhsMDqACCySy7SUsL6AcBOM6A0xbPTIfQPvzwelhpA7j8G4p9vnlOFdnLNcnFt4H8l3il/lWUyEmf6SdeCNNL1m53qYqZQn8LAQyu7VfKn5oTNcMEFsgZ7rnY8Y3UiJ8uXfBhf4e/LF2QxKgDFNYftC3kU0EnX8mAeKyrNmaLdjz/lbQ0Yaq6w4dFkDm1NMp6vY5RMFCntkjNOGGuqlwCZY1YM+SFs13RdOJi/CGuwWDFJv/v28pAk+UTBtT23n/kWx7TI1dcUWYE56Lq7qbR3It0cgBicvh9Uq8ya1om1WwmzsY8zDayk5eIcm9kZ3lCZi595BEbGtFn0IIAEYmHPwwEM/acAu2rw= - distributions: "sdist bdist_wheel" - on: - tags: true - python: 3.8 diff --git a/CHANGELOG.md b/CHANGES.rst similarity index 80% rename from CHANGELOG.md rename to CHANGES.rst index 25ec961d4..8962edf4b 100644 --- a/CHANGELOG.md +++ b/CHANGES.rst @@ -1,12 +1,53 @@ -# CHANGELOG +======= +CHANGES +======= +.. towncrier release notes start -## 0.11.1 (2019-07-31) +1.0.0 (2023-xx-xx) +================== + +Migration instructions +++++++++++++++++++++++ + +There are a number of backwards-incompatible changes. These points should help with migrating from an older release: + +* ``RedisBackend`` now expects a ``redis.Redis`` instance as an argument, instead of creating one internally from keyword arguments. +* The ``key_builder`` parameter for caches now expects a callback which accepts 2 strings and returns a string in all cache implementations, making the builders simpler and interchangeable. +* The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` +* When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). +* ``BaseCache`` and ``BaseSerializer`` are now ``ABC``s, so cannot be instantiated directly. +* If subclassing ``BaseCache`` to implement a custom backend: + + * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically). + * The ``build_key()`` method must now be defined (this should generally involve calling ``self._str_build_key()`` as a helper). + + +0.12.0 (2023-01-13) +=================== + +* Added ``async with`` support to ``BaseCache``. +* Added initial typing support. +* Migrated to ``redis`` library (``aioredis`` is no longer supported). +* ``SimpleMemoryBackend`` now has a cache per instance, rather than a global cache. +* Improved support for ``build_key(key, namespace)`` [#569](https://github.com/aio-libs/aiocache/issues/569) -- Padraic Shafer +* Removed deprecated ``loop`` parameters. +* Removed deprecated ``cache`` parameter from ``create()``. +* Added support for keyword arguments in ``TimingPlugin`` methods. +* Fixed inconsistent enum keys between different Python versions. -- Padraic Shafer +* Fixed ``.clear()`` breaking when no keys are present. +* Fixed ``from aiocache import *``. +* Fixed ``.delete()`` when values are falsy. + + +0.11.1 (2019-07-31) +=================== * Don't hardcode import redis and memcached in factory [#461](https://github.com/argaen/aiocache/issues/461) - Manuel Miranda -## 0.11.0 (2019-07-31) +0.11.0 (2019-07-31) +=================== * Support str for timeout and ttl [#454](https://github.com/argaen/aiocache/issues/454) - Manuel Miranda @@ -27,7 +68,8 @@ * Add Cache class factory [#430](https://github.com/argaen/aiocache/issues/430) - Manuel Miranda -## 0.10.1 (2018-11-15) +0.10.1 (2018-11-15) +=================== * Cancel the previous ttl timer if exists when setting a new value in the in-memory cache [#424](https://github.com/argaen/aiocache/issues/424) - Minh Tu Le @@ -40,12 +82,16 @@ * Format code with black [#410](https://github.com/argaen/aiocache/issues/410) - Manuel Miranda -## 0.10.0 (2018-06-17) +0.10.0 (2018-06-17) +=================== * Cache can be disabled in decorated functions using `cache_read` and `cache_write` [#404](https://github.com/argaen/aiocache/issues/404) - Josep Cugat * Cache constructor can receive now default ttl [#405](https://github.com/argaen/aiocache/issues/405) - Josep Cugat -## 0.9.1 (2018-04-27) + + +0.9.1 (2018-04-27) +================== * Single deploy step [#395](https://github.com/argaen/aiocache/issues/395) - Manuel Miranda @@ -54,7 +100,8 @@ * Lazy load redis asyncio.Lock [#397](https://github.com/argaen/aiocache/issues/397) - Jordi Soucheiron -## 0.9.0 (2018-04-24) +0.9.0 (2018-04-24) +================== * Bug #389/propagate redlock exceptions [#394](https://github.com/argaen/aiocache/issues/394) - Manuel Miranda ___aexit__ was returning whether asyncio Event was removed or not. In @@ -76,7 +123,8 @@ raise always any exception raised from inside_ * Fixed spelling error in serializers.py [#371](https://github.com/argaen/aiocache/issues/371) - Jared Shields -## 0.8.0 (2017-11-08) +0.8.0 (2017-11-08) +================== * Add pypy support in build pipeline [#359](https://github.com/argaen/aiocache/issues/359) - Manuel Miranda @@ -93,25 +141,24 @@ raise always any exception raised from inside_ * Add key_builder param to caches to customize keys [#315](https://github.com/argaen/aiocache/issues/315) - Manuel Miranda -## 0.7.2 (2017-07-23) - -#### Other +0.7.2 (2017-07-23) +================== * Add key_builder param to caches to customize keys [#310](https://github.com/argaen/aiocache/issues/310) - Manuel Miranda * Propagate correct message on memcached connector error [#309](https://github.com/argaen/aiocache/issues/309) - Manuel Miranda - -## 0.7.1 (2017-07-15) - +0.7.1 (2017-07-15) +================== * Remove explicit loop usages [#305](https://github.com/argaen/aiocache/issues/305) - Manuel Miranda * Remove bad logging configuration [#304](https://github.com/argaen/aiocache/issues/304) - Manuel Miranda -## 0.7.0 (2017-07-01) +0.7.0 (2017-07-01) +================== * Upgrade to aioredis 0.3.3. - Manuel Miranda @@ -137,9 +184,9 @@ and it behaves as expected._ * Removed settings module. - Manuel Miranda -## 0.6.1 (2017-06-12) -#### Other +0.6.1 (2017-06-12) +================== * Removed connection reusage for decorators [#267](https://github.com/argaen/aiocache/issues/267)- Manuel Miranda (thanks @dmzkrsk) _when decorated function is costly connections where being kept while @@ -153,10 +200,11 @@ when saving the keys_ * Updated aioredis (0.3.1) and aiomcache (0.5.2) versions - Manuel Miranda +0.6.0 (2017-06-05) +================== -## 0.6.0 (2017-06-05) - -#### New +New ++++ * Cached supports stampede locking [#249](https://github.com/argaen/aiocache/issues/249) - Manuel Miranda @@ -173,7 +221,8 @@ when saving the keys_ * `caches.create` works without alias [#253](https://github.com/argaen/aiocache/issues/253) - Manuel Miranda -#### Changes +Changes ++++++++ * Decorators use JsonSerializer by default now [#258](https://github.com/argaen/aiocache/issues/258) - Manuel Miranda @@ -193,7 +242,8 @@ there is big expected concurrency for that given function_ cache if needed (same behavior for aiomcache and ofc memory)_ -## 0.5.2 +0.5.2 +===== * Reuse connection context manager [#225](https://github.com/argaen/aiocache/issues/225) [argaen] * Add performance footprint tests [#228](https://github.com/argaen/aiocache/issues/228) [argaen] @@ -202,13 +252,15 @@ cache if needed (same behavior for aiomcache and ofc memory)_ * Added performance concurrency tests [#216](https://github.com/argaen/aiocache/issues/216) [argaen] -## 0.5.1 +0.5.1 +===== * Deprecate settings module [#215](https://github.com/argaen/aiocache/issues/215) [argaen] * Decorators support introspection [#213](https://github.com/argaen/aiocache/issues/213) [argaen] -## 0.5.0 (2017-04-29) +0.5.0 (2017-04-29) +================== * Removed pool reusage for redis. A new one is created for each instance [argaen] @@ -225,7 +277,8 @@ cache if needed (same behavior for aiomcache and ofc memory)_ * Added example for compression serializer [#179](https://github.com/argaen/aiocache/issues/179) [argaen] * Added BasePlugin.add_hook helper [#173](https://github.com/argaen/aiocache/issues/173) [argaen] -#### Breaking +Breaking +++++++++ * Refactored how settings and defaults work. Now aliases are the only way. [#193](https://github.com/argaen/aiocache/issues/193) [argaen] @@ -234,7 +287,8 @@ cache if needed (same behavior for aiomcache and ofc memory)_ when using DefaultSerializer [#191](https://github.com/argaen/aiocache/issues/191) [argaen] -## 0.3.3 (2017-04-06) +0.3.3 (2017-04-06) +================== * Added CHANGELOG and release process [#172](https://github.com/argaen/aiocache/issues/172) [argaen] * Added pool_min_size pool_max_size to redisbackend [#167](https://github.com/argaen/aiocache/issues/167) [argaen] @@ -243,13 +297,15 @@ cache if needed (same behavior for aiomcache and ofc memory)_ * Cache instance in decorators is built in every call [#135](https://github.com/argaen/aiocache/issues/135) [argaen] -## 0.3.1 (2017-02-13) +0.3.1 (2017-02-13) +================== * Changed add redis to use set with not existing flag [#119](https://github.com/argaen/aiocache/issues/119) [argaen] * Memcached multi_set with ensure_future [#114](https://github.com/argaen/aiocache/issues/114) [argaen] -## 0.3.0 (2017-01-12) +0.3.0 (2017-01-12) +================== * Fixed asynctest issues for timeout tests [#109](https://github.com/argaen/aiocache/issues/109) [argaen] * Created new API class [#108](https://github.com/argaen/aiocache/issues/108) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..014e08fe2 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +include CHANGES.rst +include LICENSE +include README.rst +include Makefile +include requirements.txt +include requirements-dev.txt +include setup.cfg +include .coveragerc +graft aiocache +graft docs +graft examples +graft tests +global-exclude *.pyc diff --git a/Makefile b/Makefile index 104cf80ec..c2f834812 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,6 @@ cov-report = true lint: flake8 tests/ aiocache/ - black -l 100 --check tests/ aiocache/ - -format: - black -l 100 tests/ aiocache/ install-dev: pip install -e .[redis,memcached,msgpack,dev] diff --git a/README.rst b/README.rst index 555c5f51a..e1c9271b8 100644 --- a/README.rst +++ b/README.rst @@ -61,12 +61,12 @@ Using a cache is as simple as .. code-block:: python >>> import asyncio - >>> loop = asyncio.get_event_loop() >>> from aiocache import Cache >>> cache = Cache(Cache.MEMORY) # Here you can also use Cache.REDIS and Cache.MEMCACHED, default is Cache.MEMORY - >>> loop.run_until_complete(cache.set('key', 'value')) + >>> with asyncio.Runner() as runner: + >>> runner.run(cache.set('key', 'value')) True - >>> loop.run_until_complete(cache.get('key')) + >>> runner.run(cache.get('key')) 'value' Or as a decorator @@ -92,16 +92,15 @@ Or as a decorator return Result("content", 200) - def run(): - loop = asyncio.get_event_loop() - loop.run_until_complete(cached_call()) - loop.run_until_complete(cached_call()) - loop.run_until_complete(cached_call()) + async def run(): + await cached_call() + await cached_call() + await cached_call() cache = Cache(Cache.REDIS, endpoint="127.0.0.1", port=6379, namespace="main") - loop.run_until_complete(cache.delete("key")) + await cache.delete("key") if __name__ == "__main__": - run() + asyncio.run(run()) The recommended approach to instantiate a new cache is using the `Cache` constructor. However you can also instantiate directly using `aiocache.RedisCache`, `aiocache.SimpleMemoryCache` or `aiocache.MemcachedCache`. @@ -150,16 +149,15 @@ You can also setup cache aliases so its easy to reuse configurations assert await cache.get("key") == "value" - def test_alias(): - loop = asyncio.get_event_loop() - loop.run_until_complete(default_cache()) - loop.run_until_complete(alt_cache()) + async def test_alias(): + await default_cache() + await alt_cache() - loop.run_until_complete(caches.get('redis_alt').delete("key")) + await caches.get("redis_alt").delete("key") if __name__ == "__main__": - test_alias() + asyncio.run(test_alias()) How does it work @@ -167,7 +165,7 @@ How does it work Aiocache provides 3 main entities: -- **backends**: Allow you specify which backend you want to use for your cache. Currently supporting: SimpleMemoryCache, RedisCache using aioredis_ and MemCache using aiomcache_. +- **backends**: Allow you specify which backend you want to use for your cache. Currently supporting: SimpleMemoryCache, RedisCache using redis_ and MemCache using aiomcache_. - **serializers**: Serialize and deserialize the data between your code and the backends. This allows you to save any Python object into your cache. Currently supporting: StringSerializer, PickleSerializer, JsonSerializer, and MsgPackSerializer. But you can also build custom ones. - **plugins**: Implement a hooks system that allows to execute extra behavior before and after of each command. @@ -210,5 +208,5 @@ Documentation - `Examples `_ -.. _aioredis: https://github.com/aio-libs/aioredis +.. _redis: https://github.com/redis/redis-py .. _aiomcache: https://github.com/aio-libs/aiomcache diff --git a/aiocache/__init__.py b/aiocache/__init__.py index 41625c5b2..c2b5b765a 100644 --- a/aiocache/__init__.py +++ b/aiocache/__init__.py @@ -1,37 +1,37 @@ import logging +from typing import Any, Dict, Type from .backends.memory import SimpleMemoryCache -from ._version import __version__ +from .base import BaseCache +__version__ = "1.0.0a0" logger = logging.getLogger(__name__) -AIOCACHE_CACHES = {SimpleMemoryCache.NAME: SimpleMemoryCache} - +AIOCACHE_CACHES: Dict[str, Type[BaseCache[Any]]] = {SimpleMemoryCache.NAME: SimpleMemoryCache} try: - import aioredis + import redis except ImportError: - logger.info("aioredis not installed, RedisCache unavailable") + logger.debug("redis not installed, RedisCache unavailable") else: from aiocache.backends.redis import RedisCache AIOCACHE_CACHES[RedisCache.NAME] = RedisCache - del aioredis + del redis try: import aiomcache except ImportError: - logger.info("aiomcache not installed, Memcached unavailable") + logger.debug("aiomcache not installed, Memcached unavailable") else: from aiocache.backends.memcached import MemcachedCache AIOCACHE_CACHES[MemcachedCache.NAME] = MemcachedCache del aiomcache - -from .factory import caches, Cache # noqa: E402 -from .decorators import cached, cached_stampede, multi_cached # noqa: E402 +from .decorators import cached, cached_stampede, multi_cached # noqa: E402,I202 +from .factory import Cache, caches # noqa: E402 __all__ = ( @@ -41,5 +41,4 @@ "cached_stampede", "multi_cached", *(c.__name__ for c in AIOCACHE_CACHES.values()), - "__version__", ) diff --git a/aiocache/_version.py b/aiocache/_version.py deleted file mode 100644 index fee46bd8c..000000000 --- a/aiocache/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.11.1" diff --git a/aiocache/backends/memcached.py b/aiocache/backends/memcached.py index 99966007d..76ac34e1a 100644 --- a/aiocache/backends/memcached.py +++ b/aiocache/backends/memcached.py @@ -1,19 +1,20 @@ import asyncio +from typing import Optional + import aiomcache from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer -class MemcachedBackend: - def __init__(self, endpoint="127.0.0.1", port=11211, pool_size=2, loop=None, **kwargs): +class MemcachedBackend(BaseCache[bytes]): + def __init__(self, host="127.0.0.1", port=11211, pool_size=2, **kwargs): super().__init__(**kwargs) - self.endpoint = endpoint + self.host = host self.port = port self.pool_size = int(pool_size) - self._loop = loop self.client = aiomcache.Client( - self.endpoint, self.port, loop=self._loop, pool_size=self.pool_size + self.host, self.port, pool_size=self.pool_size ) async def _get(self, key, encoding="utf-8", _conn=None): @@ -105,7 +106,7 @@ async def _clear(self, namespace=None, _conn=None): async def _raw(self, command, *args, encoding="utf-8", _conn=None, **kwargs): value = await getattr(self.client, command)(*args, **kwargs) - if command in ["get", "multi_get"]: + if command in {"get", "multi_get"}: if encoding is not None and value is not None: return value.decode(encoding) return value @@ -118,8 +119,12 @@ async def _redlock_release(self, key, _): async def _close(self, *args, _conn=None, **kwargs): await self.client.close() + def build_key(self, key: str, namespace: Optional[str] = None) -> bytes: + ns_key = self._str_build_key(key, namespace).replace(" ", "_") + return str.encode(ns_key) + -class MemcachedCache(MemcachedBackend, BaseCache): +class MemcachedCache(MemcachedBackend): """ Memcached cache implementation with the following components as defaults: - serializer: :class:`aiocache.serializers.JsonSerializer` @@ -130,7 +135,7 @@ class MemcachedCache(MemcachedBackend, BaseCache): :param serializer: obj derived from :class:`aiocache.serializers.BaseSerializer`. :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None + the backend. Default is an empty string, "". :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. :param endpoint: str with the endpoint to connect to. Default is 127.0.0.1. @@ -141,16 +146,11 @@ class MemcachedCache(MemcachedBackend, BaseCache): NAME = "memcached" def __init__(self, serializer=None, **kwargs): - super().__init__(**kwargs) - self.serializer = serializer or JsonSerializer() + super().__init__(serializer=serializer or JsonSerializer(), **kwargs) @classmethod - def parse_uri_path(self, path): + def parse_uri_path(cls, path): return {} - def _build_key(self, key, namespace=None): - ns_key = super()._build_key(key, namespace=namespace).replace(" ", "_") - return str.encode(ns_key) - def __repr__(self): # pragma: no cover - return "MemcachedCache ({}:{})".format(self.endpoint, self.port) + return "MemcachedCache ({}:{})".format(self.host, self.port) diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index 2e718145f..e36627075 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -1,40 +1,42 @@ import asyncio +from typing import Any, Dict, Optional from aiocache.base import BaseCache from aiocache.serializers import NullSerializer -class SimpleMemoryBackend: +class SimpleMemoryBackend(BaseCache[str]): """ Wrapper around dict operations to use it as a cache backend """ - _cache = {} - _handlers = {} - - def __init__(self, **kwargs): + # TODO(PY312): https://peps.python.org/pep-0692/ + def __init__(self, **kwargs: Any): super().__init__(**kwargs) + self._cache: Dict[str, object] = {} + self._handlers: Dict[str, asyncio.TimerHandle] = {} + async def _get(self, key, encoding="utf-8", _conn=None): - return SimpleMemoryBackend._cache.get(key) + return self._cache.get(key) async def _gets(self, key, encoding="utf-8", _conn=None): return await self._get(key, encoding=encoding, _conn=_conn) async def _multi_get(self, keys, encoding="utf-8", _conn=None): - return [SimpleMemoryBackend._cache.get(key) for key in keys] + return [self._cache.get(key) for key in keys] async def _set(self, key, value, ttl=None, _cas_token=None, _conn=None): - if _cas_token is not None and _cas_token != SimpleMemoryBackend._cache.get(key): + if _cas_token is not None and _cas_token != self._cache.get(key): return 0 - if key in SimpleMemoryBackend._handlers: - SimpleMemoryBackend._handlers[key].cancel() + if key in self._handlers: + self._handlers[key].cancel() - SimpleMemoryBackend._cache[key] = value + self._cache[key] = value if ttl: - loop = asyncio.get_event_loop() - SimpleMemoryBackend._handlers[key] = loop.call_later(ttl, self.__delete, key) + loop = asyncio.get_running_loop() + self._handlers[key] = loop.call_later(ttl, self.__delete, key) return True async def _multi_set(self, pairs, ttl=None, _conn=None): @@ -43,33 +45,33 @@ async def _multi_set(self, pairs, ttl=None, _conn=None): return True async def _add(self, key, value, ttl=None, _conn=None): - if key in SimpleMemoryBackend._cache: + if key in self._cache: raise ValueError("Key {} already exists, use .set to update the value".format(key)) await self._set(key, value, ttl=ttl) return True async def _exists(self, key, _conn=None): - return key in SimpleMemoryBackend._cache + return key in self._cache async def _increment(self, key, delta, _conn=None): - if key not in SimpleMemoryBackend._cache: - SimpleMemoryBackend._cache[key] = delta + if key not in self._cache: + self._cache[key] = delta else: try: - SimpleMemoryBackend._cache[key] = int(SimpleMemoryBackend._cache[key]) + delta + self._cache[key] = int(self._cache[key]) + delta except ValueError: raise TypeError("Value is not an integer") from None - return SimpleMemoryBackend._cache[key] + return self._cache[key] async def _expire(self, key, ttl, _conn=None): - if key in SimpleMemoryBackend._cache: - handle = SimpleMemoryBackend._handlers.pop(key, None) + if key in self._cache: + handle = self._handlers.pop(key, None) if handle: handle.cancel() if ttl: - loop = asyncio.get_event_loop() - SimpleMemoryBackend._handlers[key] = loop.call_later(ttl, self.__delete, key) + loop = asyncio.get_running_loop() + self._handlers[key] = loop.call_later(ttl, self.__delete, key) return True return False @@ -79,35 +81,37 @@ async def _delete(self, key, _conn=None): async def _clear(self, namespace=None, _conn=None): if namespace: - for key in list(SimpleMemoryBackend._cache): + for key in list(self._cache): if key.startswith(namespace): self.__delete(key) else: - SimpleMemoryBackend._cache = {} - SimpleMemoryBackend._handlers = {} + self._cache = {} + self._handlers = {} return True async def _raw(self, command, *args, encoding="utf-8", _conn=None, **kwargs): - return getattr(SimpleMemoryBackend._cache, command)(*args, **kwargs) + return getattr(self._cache, command)(*args, **kwargs) async def _redlock_release(self, key, value): - if SimpleMemoryBackend._cache.get(key) == value: - SimpleMemoryBackend._cache.pop(key) + if self._cache.get(key) == value: + self._cache.pop(key) return 1 return 0 - @classmethod - def __delete(cls, key): - if cls._cache.pop(key, None) is not None: - handle = cls._handlers.pop(key, None) + def __delete(self, key): + if self._cache.pop(key, None) is not None: + handle = self._handlers.pop(key, None) if handle: handle.cancel() return 1 return 0 + def build_key(self, key: str, namespace: Optional[str] = None) -> str: + return self._str_build_key(key, namespace) -class SimpleMemoryCache(SimpleMemoryBackend, BaseCache): + +class SimpleMemoryCache(SimpleMemoryBackend): """ Memory cache implementation with the following components as defaults: - serializer: :class:`aiocache.serializers.NullSerializer` @@ -118,7 +122,7 @@ class SimpleMemoryCache(SimpleMemoryBackend, BaseCache): :param serializer: obj derived from :class:`aiocache.serializers.BaseSerializer`. :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None. + the backend. Default is an empty string, "". :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. """ @@ -126,8 +130,7 @@ class SimpleMemoryCache(SimpleMemoryBackend, BaseCache): NAME = "memory" def __init__(self, serializer=None, **kwargs): - super().__init__(**kwargs) - self.serializer = serializer or NullSerializer() + super().__init__(serializer=serializer or NullSerializer(), **kwargs) @classmethod def parse_uri_path(cls, path): diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index d167cff23..b150fbdd3 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -1,35 +1,20 @@ -import asyncio import itertools -import functools +from typing import Any, Callable, Optional, TYPE_CHECKING -import aioredis +import redis.asyncio as redis +from redis.exceptions import ResponseError as IncrbyException from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer +if TYPE_CHECKING: # pragma: no cover + from aiocache.serializers import BaseSerializer -AIOREDIS_BEFORE_ONE = aioredis.__version__.startswith("0.") +_NOT_SET = object() -def conn(func): - @functools.wraps(func) - async def wrapper(self, *args, _conn=None, **kwargs): - if _conn is None: - - pool = await self._get_pool() - conn_context = await pool - with conn_context as _conn: - if not AIOREDIS_BEFORE_ONE: - _conn = aioredis.Redis(_conn) - return await func(self, *args, _conn=_conn, **kwargs) - - return await func(self, *args, _conn=_conn, **kwargs) - - return wrapper - - -class RedisBackend: +class RedisBackend(BaseCache[str]): RELEASE_SCRIPT = ( "if redis.call('get',KEYS[1]) == ARGV[1] then" " return redis.call('del',KEYS[1])" @@ -50,179 +35,128 @@ class RedisBackend: " end" ) - pools = {} - def __init__( self, - endpoint="127.0.0.1", - port=6379, - db=0, - password=None, - pool_min_size=1, - pool_max_size=10, - loop=None, - create_connection_timeout=None, - **kwargs + client: redis.Redis, + **kwargs, ): super().__init__(**kwargs) - self.endpoint = endpoint - self.port = int(port) - self.db = int(db) - self.password = password - self.pool_min_size = int(pool_min_size) - self.pool_max_size = int(pool_max_size) - self.create_connection_timeout = ( - float(create_connection_timeout) if create_connection_timeout else None - ) - self.__pool_lock = None - self._loop = loop - self._pool = None - - @property - def _pool_lock(self): - if self.__pool_lock is None: - self.__pool_lock = asyncio.Lock() - return self.__pool_lock - - async def acquire_conn(self): - await self._get_pool() - conn = await self._pool.acquire() - if not AIOREDIS_BEFORE_ONE: - conn = aioredis.Redis(conn) - return conn - - async def release_conn(self, _conn): - if AIOREDIS_BEFORE_ONE: - self._pool.release(_conn) - else: - self._pool.release(_conn.connection) - @conn + # NOTE: decoding can't be controlled on API level after switching to + # redis, we need to disable decoding on global/connection level + # (decode_responses=False), because some of the values are saved as + # bytes directly, like pickle serialized values, which may raise an + # exception when decoded with 'utf-8'. + if client.connection_pool.connection_kwargs['decode_responses']: + raise ValueError("redis client must be constructed with decode_responses set to False") + self.client = client + async def _get(self, key, encoding="utf-8", _conn=None): - return await _conn.get(key, encoding=encoding) + value = await self.client.get(key) + if encoding is None or value is None: + return value + return value.decode(encoding) - @conn async def _gets(self, key, encoding="utf-8", _conn=None): return await self._get(key, encoding=encoding, _conn=_conn) - @conn async def _multi_get(self, keys, encoding="utf-8", _conn=None): - return await _conn.mget(*keys, encoding=encoding) + values = await self.client.mget(*keys) + if encoding is None: + return values + return [v if v is None else v.decode(encoding) for v in values] - @conn async def _set(self, key, value, ttl=None, _cas_token=None, _conn=None): if _cas_token is not None: return await self._cas(key, value, _cas_token, ttl=ttl, _conn=_conn) if ttl is None: - return await _conn.set(key, value) - return await _conn.setex(key, ttl, value) + return await self.client.set(key, value) + if isinstance(ttl, float): + ttl = int(ttl * 1000) + return await self.client.psetex(key, ttl, value) + return await self.client.setex(key, ttl, value) - @conn async def _cas(self, key, value, token, ttl=None, _conn=None): - args = [value, token] + args = () if ttl is not None: - if isinstance(ttl, float): - args += ["PX", int(ttl * 1000)] - else: - args += ["EX", ttl] - res = await self._raw("eval", self.CAS_SCRIPT, [key], args, _conn=_conn) - return res - - @conn + args = ("PX", int(ttl * 1000)) if isinstance(ttl, float) else ("EX", ttl) + return await self._raw("eval", self.CAS_SCRIPT, 1, key, value, token, *args, _conn=_conn) + async def _multi_set(self, pairs, ttl=None, _conn=None): ttl = ttl or 0 flattened = list(itertools.chain.from_iterable((key, value) for key, value in pairs)) if ttl: - await self.__multi_set_ttl(_conn, flattened, ttl) + await self.__multi_set_ttl(flattened, ttl) else: - await _conn.mset(*flattened) + await self.client.execute_command("MSET", *flattened) return True - async def __multi_set_ttl(self, conn, flattened, ttl): - redis = conn.multi_exec() - redis.mset(*flattened) - for key in flattened[::2]: - redis.expire(key, timeout=ttl) - await redis.execute() + async def __multi_set_ttl(self, flattened, ttl): + async with self.client.pipeline(transaction=True) as p: + p.execute_command("MSET", *flattened) + ttl, exp = (int(ttl * 1000), p.pexpire) if isinstance(ttl, float) else (ttl, p.expire) + for key in flattened[::2]: + exp(key, time=ttl) + await p.execute() - @conn async def _add(self, key, value, ttl=None, _conn=None): - expx = {"expire": ttl} + kwargs = {"nx": True} if isinstance(ttl, float): - expx = {"pexpire": int(ttl * 1000)} - was_set = await _conn.set(key, value, exist=_conn.SET_IF_NOT_EXIST, **expx) + kwargs["px"] = int(ttl * 1000) + else: + kwargs["ex"] = ttl + was_set = await self.client.set(key, value, **kwargs) if not was_set: raise ValueError("Key {} already exists, use .set to update the value".format(key)) return was_set - @conn async def _exists(self, key, _conn=None): - exists = await _conn.exists(key) - return True if exists > 0 else False + number = await self.client.exists(key) + return bool(number) - @conn async def _increment(self, key, delta, _conn=None): try: - return await _conn.incrby(key, delta) - except aioredis.errors.ReplyError: + return await self.client.incrby(key, delta) + except IncrbyException: raise TypeError("Value is not an integer") from None - @conn async def _expire(self, key, ttl, _conn=None): if ttl == 0: - return await _conn.persist(key) - return await _conn.expire(key, ttl) + return await self.client.persist(key) + return await self.client.expire(key, ttl) - @conn async def _delete(self, key, _conn=None): - return await _conn.delete(key) + return await self.client.delete(key) - @conn async def _clear(self, namespace=None, _conn=None): if namespace: - keys = await _conn.keys("{}:*".format(namespace)) + keys = await self.client.keys("{}:*".format(namespace)) if keys: - await _conn.delete(*keys) + await self.client.delete(*keys) else: - await _conn.flushdb() + await self.client.flushdb() return True - @conn async def _raw(self, command, *args, encoding="utf-8", _conn=None, **kwargs): - if command in ["get", "mget"]: - kwargs["encoding"] = encoding - return await getattr(_conn, command)(*args, **kwargs) + value = await getattr(self.client, command)(*args, **kwargs) + if encoding is not None: + if command == "get" and value is not None: + value = value.decode(encoding) + elif command in {"keys", "mget"}: + value = [v if v is None else v.decode(encoding) for v in value] + return value async def _redlock_release(self, key, value): - return await self._raw("eval", self.RELEASE_SCRIPT, [key], [value]) - - async def _close(self, *args, **kwargs): - if self._pool is not None: - await self._pool.clear() - - async def _get_pool(self): - async with self._pool_lock: - if self._pool is None: - kwargs = { - "db": self.db, - "password": self.password, - "loop": self._loop, - "encoding": "utf-8", - "minsize": self.pool_min_size, - "maxsize": self.pool_max_size, - } - if not AIOREDIS_BEFORE_ONE: - kwargs["create_connection_timeout"] = self.create_connection_timeout + return await self._raw("eval", self.RELEASE_SCRIPT, 1, key, value) - self._pool = await aioredis.create_pool((self.endpoint, self.port), **kwargs) + def build_key(self, key: str, namespace: Optional[str] = None) -> str: + return self._str_build_key(key, namespace) - return self._pool - -class RedisCache(RedisBackend, BaseCache): +class RedisCache(RedisBackend): """ Redis cache implementation with the following components as defaults: - serializer: :class:`aiocache.serializers.JsonSerializer` @@ -233,27 +167,32 @@ class RedisCache(RedisBackend, BaseCache): :param serializer: obj derived from :class:`aiocache.serializers.BaseSerializer`. :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None. + the backend. Default is an empty string, "". :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. - :param endpoint: str with the endpoint to connect to. Default is "127.0.0.1". - :param port: int with the port to connect to. Default is 6379. - :param db: int indicating database to use. Default is 0. - :param password: str indicating password to use. Default is None. - :param pool_min_size: int minimum pool size for the redis connections pool. Default is 1 - :param pool_max_size: int maximum pool size for the redis connections pool. Default is 10 - :param create_connection_timeout: int timeout for the creation of connection, - only for aioredis>=1. Default is None + :param client: redis.Redis which is an active client for working with redis """ NAME = "redis" - def __init__(self, serializer=None, **kwargs): - super().__init__(**kwargs) - self.serializer = serializer or JsonSerializer() + def __init__( + self, + client: redis.Redis, + serializer: Optional["BaseSerializer"] = None, + namespace: str = "", + key_builder: Callable[[str, str], str] = lambda k, ns: f"{ns}:{k}" if ns else k, + **kwargs: Any, + ): + super().__init__( + client=client, + serializer=serializer or JsonSerializer(), + namespace=namespace, + key_builder=key_builder, + **kwargs, + ) @classmethod - def parse_uri_path(self, path): + def parse_uri_path(cls, path): """ Given a uri path, return the Redis specific configuration options in that path string according to iana definition @@ -268,12 +207,6 @@ def parse_uri_path(self, path): options["db"] = db return options - def _build_key(self, key, namespace=None): - if namespace is not None: - return "{}{}{}".format(namespace, ":" if namespace else "", key) - if self.namespace is not None: - return "{}{}{}".format(self.namespace, ":" if self.namespace else "", key) - return key - def __repr__(self): # pragma: no cover - return "RedisCache ({}:{})".format(self.endpoint, self.port) + connection_kwargs = self.client.connection_pool.connection_kwargs + return "RedisCache ({}:{})".format(connection_kwargs['host'], connection_kwargs['port']) diff --git a/aiocache/base.py b/aiocache/base.py index a14de06ce..f64edeb68 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -1,20 +1,29 @@ -import os -import time +import asyncio import functools import logging -import asyncio +import os +import time +from abc import ABC, abstractmethod +from enum import Enum +from types import TracebackType +from typing import Callable, Generic, List, Optional, Set, TYPE_CHECKING, Type, TypeVar + +from aiocache.serializers import StringSerializer -from aiocache import serializers +if TYPE_CHECKING: # pragma: no cover + from aiocache.plugins import BasePlugin + from aiocache.serializers import BaseSerializer logger = logging.getLogger(__name__) SENTINEL = object() +CacheKeyType = TypeVar("CacheKeyType") class API: - CMDS = set() + CMDS: Set[Callable[..., object]] = set() @classmethod def register(cls, func): @@ -84,7 +93,7 @@ async def _plugins(self, *args, **kwargs): return _plugins -class BaseCache: +class BaseCache(Generic[CacheKeyType], ABC): """ Base class that agregates the common logic for the different caches that may exist. Cache related available options are: @@ -94,28 +103,34 @@ class BaseCache: :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. Default is empty list. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None - :param key_builder: alternative callable to build the key. Receives the key and the namespace as - params and should return something that can be used as key by the underlying backend. + the backend. Default is an empty string, "". + :param key_builder: alternative callable to build the key. Receives the key and the namespace + as params and should return a string that can be used as a key by the underlying backend. :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. Use 0 or None if you want to disable it. :param ttl: int the expiration time in seconds to use as a default in all operations of the backend. It can be overriden in the specific calls. """ + NAME: str + def __init__( - self, serializer=None, plugins=None, namespace=None, key_builder=None, timeout=5, ttl=None + self, + serializer: Optional["BaseSerializer"] = None, + plugins: Optional[List["BasePlugin"]] = None, + namespace: str = "", + key_builder: Callable[[str, str], str] = lambda k, ns: f"{ns}{k}", + timeout: Optional[float] = 5, + ttl: Optional[float] = None, ): - self.timeout = float(timeout) if timeout is not None else timeout - self.namespace = namespace - self.ttl = float(ttl) if ttl is not None else ttl - self.build_key = key_builder or self._build_key + self.timeout = float(timeout) if timeout is not None else None + self.ttl = float(ttl) if ttl is not None else None - self._serializer = None - self.serializer = serializer or serializers.StringSerializer() + self.namespace = namespace + self._build_key = key_builder - self._plugins = None - self.plugins = plugins or [] + self._serializer = serializer or StringSerializer() + self._plugins = plugins or [] @property def serializer(self): @@ -157,14 +172,15 @@ async def add(self, key, value, ttl=SENTINEL, dumps_fn=None, namespace=None, _co - :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - dumps = dumps_fn or self._serializer.dumps - ns_key = self.build_key(key, namespace=namespace) + dumps = dumps_fn or self.serializer.dumps + ns_key = self.build_key(key, namespace) await self._add(ns_key, dumps(value), ttl=self._get_ttl(ttl), _conn=_conn) logger.debug("ADD %s %s (%.4f)s", ns_key, True, time.monotonic() - start) return True + @abstractmethod async def _add(self, key, value, ttl, _conn=None): raise NotImplementedError() @@ -186,17 +202,22 @@ async def get(self, key, default=None, loads_fn=None, namespace=None, _conn=None :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - loads = loads_fn or self._serializer.loads - ns_key = self.build_key(key, namespace=namespace) + loads = loads_fn or self.serializer.loads + ns_key = self.build_key(key, namespace) value = loads(await self._get(ns_key, encoding=self.serializer.encoding, _conn=_conn)) logger.debug("GET %s %s (%.4f)s", ns_key, value is not None, time.monotonic() - start) return value if value is not None else default + @abstractmethod async def _get(self, key, encoding, _conn=None): raise NotImplementedError() + @abstractmethod + async def _gets(self, key, encoding="utf-8", _conn=None): + raise NotImplementedError() + @API.register @API.aiocache_enabled(fake_return=[]) @API.timeout @@ -214,9 +235,9 @@ async def multi_get(self, keys, loads_fn=None, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - loads = loads_fn or self._serializer.loads + loads = loads_fn or self.serializer.loads - ns_keys = [self.build_key(key, namespace=namespace) for key in keys] + ns_keys = [self.build_key(key, namespace) for key in keys] values = [ loads(value) for value in await self._multi_get( @@ -232,6 +253,7 @@ async def multi_get(self, keys, loads_fn=None, namespace=None, _conn=None): ) return values + @abstractmethod async def _multi_get(self, keys, encoding, _conn=None): raise NotImplementedError() @@ -258,8 +280,8 @@ async def set( :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - dumps = dumps_fn or self._serializer.dumps - ns_key = self.build_key(key, namespace=namespace) + dumps = dumps_fn or self.serializer.dumps + ns_key = self.build_key(key, namespace) res = await self._set( ns_key, dumps(value), ttl=self._get_ttl(ttl), _cas_token=_cas_token, _conn=_conn @@ -268,6 +290,7 @@ async def set( logger.debug("SET %s %d (%.4f)s", ns_key, True, time.monotonic() - start) return res + @abstractmethod async def _set(self, key, value, ttl, _cas_token=None, _conn=None): raise NotImplementedError() @@ -291,22 +314,23 @@ async def multi_set(self, pairs, ttl=SENTINEL, dumps_fn=None, namespace=None, _c :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - dumps = dumps_fn or self._serializer.dumps + dumps = dumps_fn or self.serializer.dumps tmp_pairs = [] for key, value in pairs: - tmp_pairs.append((self.build_key(key, namespace=namespace), dumps(value))) + tmp_pairs.append((self.build_key(key, namespace), dumps(value))) await self._multi_set(tmp_pairs, ttl=self._get_ttl(ttl), _conn=_conn) logger.debug( "MULTI_SET %s %d (%.4f)s", [key for key, value in tmp_pairs], - len(pairs), + len(tmp_pairs), time.monotonic() - start, ) return True + @abstractmethod async def _multi_set(self, pairs, ttl, _conn=None): raise NotImplementedError() @@ -326,11 +350,12 @@ async def delete(self, key, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._delete(ns_key, _conn=_conn) logger.debug("DELETE %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret + @abstractmethod async def _delete(self, key, _conn=None): raise NotImplementedError() @@ -350,11 +375,12 @@ async def exists(self, key, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._exists(ns_key, _conn=_conn) logger.debug("EXISTS %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret + @abstractmethod async def _exists(self, key, _conn=None): raise NotImplementedError() @@ -377,11 +403,12 @@ async def increment(self, key, delta=1, namespace=None, _conn=None): :raises: :class:`TypeError` if value is not incrementable """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._increment(ns_key, delta, _conn=_conn) logger.debug("INCREMENT %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret + @abstractmethod async def _increment(self, key, delta, _conn=None): raise NotImplementedError() @@ -402,11 +429,12 @@ async def expire(self, key, ttl, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._expire(ns_key, ttl, _conn=_conn) logger.debug("EXPIRE %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret + @abstractmethod async def _expire(self, key, ttl, _conn=None): raise NotImplementedError() @@ -430,6 +458,7 @@ async def clear(self, namespace=None, _conn=None): logger.debug("CLEAR %s %d (%.4f)s", namespace, ret, time.monotonic() - start) return ret + @abstractmethod async def _clear(self, namespace, _conn=None): raise NotImplementedError() @@ -458,9 +487,14 @@ async def raw(self, command, *args, _conn=None, **kwargs): logger.debug("%s (%.4f)s", command, time.monotonic() - start) return ret + @abstractmethod async def _raw(self, command, *args, **kwargs): raise NotImplementedError() + @abstractmethod + async def _redlock_release(self, key, value): + raise NotImplementedError() + @API.timeout async def close(self, *args, _conn=None, **kwargs): """ @@ -478,12 +512,15 @@ async def close(self, *args, _conn=None, **kwargs): async def _close(self, *args, **kwargs): pass - def _build_key(self, key, namespace=None): - if namespace is not None: - return "{}{}".format(namespace, key) - if self.namespace is not None: - return "{}{}".format(self.namespace, key) - return key + @abstractmethod + def build_key(self, key: str, namespace: Optional[str] = None) -> CacheKeyType: + raise NotImplementedError() + + def _str_build_key(self, key: str, namespace: Optional[str] = None) -> str: + """Simple key builder that can be used in subclasses for build_key().""" + key_name = key.value if isinstance(key, Enum) else key + ns = self.namespace if namespace is None else namespace + return self._build_key(key_name, ns) def _get_ttl(self, ttl): return ttl if ttl is not SENTINEL else self.ttl @@ -497,6 +534,15 @@ async def acquire_conn(self): async def release_conn(self, conn): pass + async def __aenter__(self): + return self + + async def __aexit__( + self, exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], tb: Optional[TracebackType] + ) -> None: + await self.close() + class _Conn: def __init__(self, cache): diff --git a/aiocache/decorators.py b/aiocache/decorators.py index cb6d9079a..d5cdac5dd 100644 --- a/aiocache/decorators.py +++ b/aiocache/decorators.py @@ -1,29 +1,29 @@ import asyncio -import inspect import functools +import inspect import logging -from aiocache import Cache, caches from aiocache.base import SENTINEL +from aiocache.factory import Cache, caches from aiocache.lock import RedLock - logger = logging.getLogger(__name__) class cached: """ - Caches the functions return value into a key generated with module_name, function_name and args. - The cache is available in the function object as ``.cache``. + Caches the functions return value into a key generated with module_name, function_name + and args. The cache is available in the function object as ``.cache``. In some cases you will need to send more args to configure the cache object. An example would be endpoint and port for the Redis cache. You can send those args as kwargs and they will be propagated accordingly. - Only one cache instance is created per decorated call. If you expect high concurrency of calls - to the same function, you should adapt the pool size as needed. + Only one cache instance is created per decorated call. If you expect high concurrency of + calls to the same function, you should adapt the pool size as needed. - Extra args that are injected in the function that you can use to control the cache behavior are: + Extra args that are injected in the function that you can use to control the cache + behavior are: - ``cache_read``: Controls whether the function call will try to read from cache first or not. Enabled by default. @@ -34,20 +34,24 @@ class cached: happens in the background. Enabled by default :param ttl: int seconds to store the function call. Default is None which means no expiration. - :param key: str value to set as key for the function return. Takes precedence over - key_builder param. If key and key_builder are not passed, it will use module_name - + function_name + args + kwargs + :param namespace: string to use as default prefix for the key used in all operations of + the backend. Default is an empty string, "". :param key_builder: Callable that allows to build the function dynamically. It receives the function plus same args and kwargs passed to the function. + This behavior is necessarily different than ``BaseCache.build_key()`` + :param skip_cache_func: Callable that receives the result after calling the + wrapped function and should return `True` if the value should skip the + cache (or `False` to store in the cache). + e.g. to avoid caching `None` results: `lambda r: r is None` :param cache: cache class to use when calling the ``set``/``get`` operations. Default is :class:`aiocache.SimpleMemoryCache`. :param serializer: serializer instance to use when calling the ``dumps``/``loads``. If its None, default one from the cache backend is used. :param plugins: list plugins to use when calling the cmd hooks Default is pulled from the cache class being used. - :param alias: str specifying the alias to load the config from. If alias is passed, other config - parameters are ignored. Same cache identified by alias is used on every call. If you need - a per function cache, specify the parameters explicitly without using alias. + :param alias: str specifying the alias to load the config from. If alias is passed, other + config parameters are ignored. Same cache identified by alias is used on every call. If + you need a per function cache, specify the parameters explicitly without using alias. :param noself: bool if you are decorating a class function, by default self is also used to generate the key. This will result in same function calls done by different class instances to use different cache keys. Use noself=True if you want to ignore it. @@ -56,36 +60,42 @@ class cached: def __init__( self, ttl=SENTINEL, - key=None, + namespace="", key_builder=None, + skip_cache_func=lambda x: False, cache=Cache.MEMORY, serializer=None, plugins=None, alias=None, noself=False, - **kwargs + **kwargs, ): self.ttl = ttl - self.key = key self.key_builder = key_builder + self.skip_cache_func = skip_cache_func self.noself = noself self.alias = alias self.cache = None self._cache = cache self._serializer = serializer + self._namespace = namespace self._plugins = plugins self._kwargs = kwargs def __call__(self, f): if self.alias: self.cache = caches.get(self.alias) + for arg in ("serializer", "namespace", "plugins"): + if getattr(self, f'_{arg}', None) is not None: + logger.warning(f"Using cache alias; ignoring {arg!r} argument.") else: self.cache = _get_cache( cache=self._cache, serializer=self._serializer, + namespace=self._namespace, plugins=self._plugins, - **self._kwargs + **self._kwargs, ) @functools.wraps(f) @@ -107,17 +117,19 @@ async def decorator( result = await f(*args, **kwargs) + if self.skip_cache_func(result): + return result + if cache_write: if aiocache_wait_for_write: await self.set_in_cache(key, result) else: - asyncio.ensure_future(self.set_in_cache(key, result)) + # TODO: Use aiojobs to avoid warnings. + asyncio.create_task(self.set_in_cache(key, result)) return result def get_cache_key(self, f, args, kwargs): - if self.key: - return self.key if self.key_builder: return self.key_builder(f, *args, **kwargs) @@ -134,10 +146,10 @@ def _key_from_args(self, func, args, kwargs): async def get_from_cache(self, key): try: - value = await self.cache.get(key) - return value + return await self.cache.get(key) except Exception: logger.exception("Couldn't retrieve %s, unexpected error", key) + return None async def set_in_cache(self, key, value): try: @@ -162,18 +174,24 @@ class cached_stampede(cached): If 0 or None, no locking happens (default is 2). redis and memory backends support float ttls :param ttl: int seconds to store the function call. Default is None which means no expiration. - :param key: str value to set as key for the function return. Takes precedence over - key_from_attr param. If key and key_from_attr are not passed, it will use module_name - + function_name + args + kwargs :param key_from_attr: str arg or kwarg name from the function to use as a key. + :param namespace: string to use as default prefix for the key used in all operations of + the backend. Default is an empty string, "". + :param key_builder: Callable that allows to build the function dynamically. It receives + the function plus same args and kwargs passed to the function. + This behavior is necessarily different than ``BaseCache.build_key()`` + :param skip_cache_func: Callable that receives the result after calling the + wrapped function and should return `True` if the value should skip the + cache (or `False` to store in the cache). + e.g. to avoid caching `None` results: `lambda r: r is None` :param cache: cache class to use when calling the ``set``/``get`` operations. Default is :class:`aiocache.SimpleMemoryCache`. :param serializer: serializer instance to use when calling the ``dumps``/``loads``. Default is JsonSerializer. :param plugins: list plugins to use when calling the cmd hooks Default is pulled from the cache class being used. - :param alias: str specifying the alias to load the config from. If alias is passed, other config - parameters are ignored. New cache is created every time. + :param alias: str specifying the alias to load the config from. If alias is passed, + other config parameters are ignored. New cache is created every time. :param noself: bool if you are decorating a class function, by default self is also used to generate the key. This will result in same function calls done by different class instances to use different cache keys. Use noself=True if you want to ignore it. @@ -197,13 +215,16 @@ async def decorator(self, f, *args, **kwargs): result = await f(*args, **kwargs) + if self.skip_cache_func(result): + return result + await self.set_in_cache(key, result) return result def _get_cache(cache=Cache.MEMORY, serializer=None, plugins=None, **cache_kwargs): - return cache(serializer=serializer, plugins=plugins, **cache_kwargs) + return Cache(cache, serializer=serializer, plugins=plugins, **cache_kwargs) def _get_args_dict(func, args, kwargs): @@ -219,23 +240,32 @@ def _get_args_dict(func, args, kwargs): class multi_cached: """ Only supports functions that return dict-like structures. This decorator caches each key/value - of the dict-like object returned by the function. Note that in this decorator, the function - name is not prefixed in the key when stored so, if there is another function returning a dict - with same keys, they will be overwritten. To avoid this, use a specific namespace in each - cache decorator or pass a key_builder. + of the dict-like object returned by the function. The dict keys of the returned data should + match the set of keys that are passed to the decorated callable in an iterable object. + The name of that argument is passed to this decorator via the parameter + ``keys_from_attr``. ``keys_from_attr`` can be the name of a positional or keyword argument. - The cache is available in the function object as ``.cache``. + If the argument specified by ``keys_from_attr`` is an empty list, the cache will be ignored + and the function will be called. If only some of the keys in ``keys_from_attr``are cached + (and ``cache_read`` is True) those values will be fetched from the cache, and only the + uncached keys will be passed to the callable via the argument specified by ``keys_from_attr``. - If key_builder is passed, before storing the key, it will be transformed according to the output - of the function. + By default, the callable's name and call signature are not incorporated into the cache key, + so if there is another cached function returning a dict with same keys, those keys will be + overwritten. To avoid this, use a specific ``namespace`` in each cache decorator or pass a + ``key_builder``. - If the attribute specified to be the key is an empty list, the cache will be ignored and - the function will be called as expected. + If ``key_builder`` is passed, then the values of ``keys_from_attr`` will be transformed + before requesting them from the cache. Equivalently, the keys in the dict-like mapping + returned by the decorated callable will be transformed before storing them in the cache. + + The cache is available in the function object as ``.cache``. Only one cache instance is created per decorated function. If you expect high concurrency of calls to the same function, you should adapt the pool size as needed. - Extra args that are injected in the function that you can use to control the cache behavior are: + Extra args that are injected in the function that you can use to control the cache + behavior are: - ``cache_read``: Controls whether the function call will try to read from cache first or not. Enabled by default. @@ -245,10 +275,18 @@ class multi_cached: value in the cache to be written. If set to False, the write happens in the background. Enabled by default - :param keys_from_attr: arg or kwarg name from the function containing an iterable to use - as keys to index in the cache. - :param key_builder: Callable that allows to change the format of the keys before storing. - Receives the key the function and same args and kwargs as the called function. + :param keys_from_attr: name of the arg or kwarg in the decorated callable that contains + an iterable that yields the keys returned by the decorated callable. + :param namespace: string to use as default prefix for the key used in all operations of + the backend. Default is an empty string, "". + :param key_builder: Callable that enables mapping the decorated function's keys to the keys + used by the cache. Receives a key from the iterable corresponding to + ``keys_from_attr``, the decorated callable, and the positional and keyword arguments + that were passed to the decorated callable. This behavior is necessarily different than + ``BaseCache.build_key()`` and the call signature differs from ``cached.key_builder``. + :param skip_cache_func: Callable that receives both key and value and returns True + if that key-value pair should not be cached (or False to store in cache). + The keys and values to be passed are taken from the wrapped function result. :param ttl: int seconds to store the keys. Default is 0 which means no expiration. :param cache: cache class to use when calling the ``multi_set``/``multi_get`` operations. Default is :class:`aiocache.SimpleMemoryCache`. @@ -256,42 +294,51 @@ class multi_cached: If its None, default one from the cache backend is used. :param plugins: plugins to use when calling the cmd hooks Default is pulled from the cache class being used. - :param alias: str specifying the alias to load the config from. If alias is passed, other config - parameters are ignored. Same cache identified by alias is used on every call. If you need - a per function cache, specify the parameters explicitly without using alias. + :param alias: str specifying the alias to load the config from. If alias is passed, + other config parameters are ignored. Same cache identified by alias is used on + every call. If you need a per function cache, specify the parameters explicitly + without using alias. """ def __init__( self, keys_from_attr, + namespace="", key_builder=None, + skip_cache_func=lambda k, v: False, ttl=SENTINEL, cache=Cache.MEMORY, serializer=None, plugins=None, alias=None, - **kwargs + **kwargs, ): self.keys_from_attr = keys_from_attr self.key_builder = key_builder or (lambda key, f, *args, **kwargs: key) + self.skip_cache_func = skip_cache_func self.ttl = ttl self.alias = alias self.cache = None self._cache = cache self._serializer = serializer + self._namespace = namespace self._plugins = plugins self._kwargs = kwargs def __call__(self, f): if self.alias: self.cache = caches.get(self.alias) + for arg in ("serializer", "namespace", "plugins"): + if getattr(self, f'_{arg}', None) is not None: + logger.warning(f"Using cache alias; ignoring {arg!r} argument.") else: self.cache = _get_cache( cache=self._cache, serializer=self._serializer, + namespace=self._namespace, plugins=self._plugins, - **self._kwargs + **self._kwargs, ) @functools.wraps(f) @@ -306,19 +353,19 @@ async def decorator( ): missing_keys = [] partial = {} - keys, new_args, args_index = self.get_cache_keys(f, args, kwargs) + orig_keys, cache_keys, new_args, args_index = self.get_cache_keys(f, args, kwargs) if cache_read: - values = await self.get_from_cache(*keys) - for key, value in zip(keys, values): + values = await self.get_from_cache(*cache_keys) + for orig_key, value in zip(orig_keys, values): if value is None: - missing_keys.append(key) + missing_keys.append(orig_key) else: - partial[key] = value + partial[orig_key] = value if values and None not in values: return partial else: - missing_keys = list(keys) + missing_keys = list(orig_keys) if args_index > -1: new_args[args_index] = missing_keys @@ -328,27 +375,32 @@ async def decorator( result = await f(*new_args, **kwargs) result.update(partial) + to_cache = {k: v for k, v in result.items() if not self.skip_cache_func(k, v)} + + if not to_cache: + return result + if cache_write: if aiocache_wait_for_write: - await self.set_in_cache(result, f, args, kwargs) + await self.set_in_cache(to_cache, f, args, kwargs) else: - asyncio.ensure_future(self.set_in_cache(result, f, args, kwargs)) + # TODO: Use aiojobs to avoid warnings. + asyncio.create_task(self.set_in_cache(to_cache, f, args, kwargs)) return result def get_cache_keys(self, f, args, kwargs): args_dict = _get_args_dict(f, args, kwargs) - keys = args_dict.get(self.keys_from_attr, []) or [] - keys = [self.key_builder(key, f, *args, **kwargs) for key in keys] + orig_keys = args_dict.get(self.keys_from_attr, []) or [] + cache_keys = [self.key_builder(key, f, *args, **kwargs) for key in orig_keys] args_names = f.__code__.co_varnames[: f.__code__.co_argcount] new_args = list(args) keys_index = -1 if self.keys_from_attr in args_names and self.keys_from_attr not in kwargs: keys_index = args_names.index(self.keys_from_attr) - new_args[keys_index] = keys - return keys, new_args, keys_index + return orig_keys, cache_keys, new_args, keys_index async def get_from_cache(self, *keys): if not keys: diff --git a/aiocache/factory.py b/aiocache/factory.py index 99a41051f..1a4346a41 100644 --- a/aiocache/factory.py +++ b/aiocache/factory.py @@ -1,11 +1,15 @@ -from copy import deepcopy import logging import urllib -import warnings +from contextlib import suppress +from copy import deepcopy +from typing import Dict -from aiocache.exceptions import InvalidCacheType from aiocache import AIOCACHE_CACHES from aiocache.base import BaseCache +from aiocache.exceptions import InvalidCacheType + +with suppress(ImportError): + import redis.asyncio as redis logger = logging.getLogger(__name__) @@ -18,7 +22,7 @@ def _class_from_string(class_path): def _create_cache(cache, serializer=None, plugins=None, **kwargs): - + kwargs = deepcopy(kwargs) if serializer is not None: cls = serializer.pop("class") cls = _class_from_string(cls) if isinstance(cls, str) else cls @@ -30,10 +34,17 @@ def _create_cache(cache, serializer=None, plugins=None, **kwargs): cls = plugin.pop("class") cls = _class_from_string(cls) if isinstance(cls, str) else cls plugins_instances.append(cls(**plugin)) - cache = _class_from_string(cache) if isinstance(cache, str) else cache - instance = cache(serializer=serializer, plugins=plugins_instances, **kwargs) - return instance + if cache == AIOCACHE_CACHES.get("redis"): + return cache( + serializer=serializer, + plugins=plugins_instances, + namespace=kwargs.pop('namespace', ''), + ttl=kwargs.pop('ttl', None), + client=redis.Redis(**kwargs) + ) + else: + return cache(serializer=serializer, plugins=plugins_instances, **kwargs) class Cache: @@ -61,12 +72,10 @@ class Cache: MEMCACHED = AIOCACHE_CACHES.get("memcached") def __new__(cls, cache_class=MEMORY, **kwargs): - try: - assert issubclass(cache_class, BaseCache) - except AssertionError as e: + if not issubclass(cache_class, BaseCache): raise InvalidCacheType( "Invalid cache type, you can only use {}".format(list(AIOCACHE_CACHES.keys())) - ) from e + ) instance = cache_class.__new__(cache_class, **kwargs) instance.__init__(**kwargs) return instance @@ -97,12 +106,12 @@ def from_url(cls, url): a more advanced usage using queryparams to configure the cache: >>> from aiocache import Cache - >>> cache = Cache.from_url('redis://localhost:10/1?pool_min_size=1') + >>> cache = Cache.from_url('redis://localhost:10/1?pool_max_size=1') >>> cache RedisCache (localhost:10) >>> cache.db 1 - >>> cache.pool_min_size + >>> cache.pool_max_size 1 :param url: string identifying the resource uri of the cache to connect to @@ -115,7 +124,7 @@ def from_url(cls, url): kwargs.update(cache_class.parse_uri_path(parsed_url.path)) if parsed_url.hostname: - kwargs["endpoint"] = parsed_url.hostname + kwargs["host"] = parsed_url.hostname if parsed_url.port: kwargs["port"] = parsed_url.port @@ -123,12 +132,18 @@ def from_url(cls, url): if parsed_url.password: kwargs["password"] = parsed_url.password - return Cache(cache_class, **kwargs) + for arg in ['max_connections', 'socket_connect_timeout']: + if arg in kwargs: + kwargs[arg] = int(kwargs[arg]) + if cache_class == cls.REDIS: + return Cache(cache_class, client=redis.Redis(**kwargs)) + else: + return Cache(cache_class, **kwargs) class CacheHandler: - _config = { + _config: Dict[str, Dict[str, object]] = { "default": { "cache": "aiocache.SimpleMemoryCache", "serializer": {"class": "aiocache.serializers.StringSerializer"}, @@ -138,7 +153,7 @@ class CacheHandler: def __init__(self): self._caches = {} - def add(self, alias: str, config: dict) -> None: + def add(self, alias: str, config: Dict[str, object]) -> None: """ Add a cache to the current config. If the key already exists, it will overwrite it:: @@ -155,7 +170,7 @@ def add(self, alias: str, config: dict) -> None: """ self._config[alias] = config - def get(self, alias: str): + def get(self, alias: str) -> object: """ Retrieve cache identified by alias. Will return always the same instance @@ -175,32 +190,17 @@ def get(self, alias: str): self._caches[alias] = cache return cache - def create(self, alias=None, cache=None, **kwargs): - """ - Create a new cache. Either alias or cache params are required. You can use - kwargs to pass extra parameters to configure the cache. + def create(self, alias, **kwargs): + """Create a new cache. - .. deprecated:: 0.11.0 - Only creating a cache passing an alias is supported. If you want to - create a cache passing explicit cache and kwargs use ``aiocache.Cache``. + You can use kwargs to pass extra parameters to configure the cache. - :param alias: str alias to pull configuration from - :param cache: str or class cache class to use for creating the - new cache (when no alias is used) + :param alias: alias to pull configuration from :return: New cache instance """ - if alias: - config = self.get_alias_config(alias) - elif cache: - warnings.warn( - "Creating a cache with an explicit config is deprecated, use 'aiocache.Cache'", - DeprecationWarning, - ) - config = {"cache": cache} - else: - raise TypeError("create call needs to receive an alias or a cache") - cache = _create_cache(**{**config, **kwargs}) - return cache + config = self.get_alias_config(alias) + # TODO(PY39): **config | kwargs + return _create_cache(**{**config, **kwargs}) def get_alias_config(self, alias): config = self.get_config() @@ -232,7 +232,7 @@ def set_config(self, config): }, 'redis_alt': { 'cache': "aiocache.RedisCache", - 'endpoint': "127.0.0.10", + 'host': "127.0.0.10", 'port': 6378, 'serializer': { 'class': "aiocache.serializers.PickleSerializer" diff --git a/aiocache/lock.py b/aiocache/lock.py index 51498e769..34e2299c9 100644 --- a/aiocache/lock.py +++ b/aiocache/lock.py @@ -1,12 +1,11 @@ import asyncio import uuid +from typing import Any, Dict, Generic, Union -from typing import Union, Any +from aiocache.base import BaseCache, CacheKeyType -from aiocache.base import BaseCache - -class RedLock: +class RedLock(Generic[CacheKeyType]): """ Implementation of `Redlock `_ with a single instance because aiocache is focused on single @@ -61,11 +60,11 @@ class RedLock: result of ``super_expensive_function``. """ - _EVENTS = {} + _EVENTS: Dict[str, asyncio.Event] = {} - def __init__(self, client: BaseCache, key: str, lease: Union[int, float]): + def __init__(self, client: BaseCache[CacheKeyType], key: str, lease: Union[int, float]): self.client = client - self.key = self.client._build_key(key + "-lock") + self.key = self.client.build_key(key + "-lock") self.lease = lease self._value = "" @@ -97,7 +96,7 @@ async def _release(self): RedLock._EVENTS.pop(self.key).set() -class OptimisticLock: +class OptimisticLock(Generic[CacheKeyType]): """ Implementation of `optimistic lock `_ @@ -134,10 +133,10 @@ class OptimisticLock: If the lock is created with an unexisting key, there will never be conflicts. """ - def __init__(self, client: BaseCache, key: str): + def __init__(self, client: BaseCache[CacheKeyType], key: str): self.client = client self.key = key - self.ns_key = self.client._build_key(key) + self.ns_key = self.client.build_key(key) self._token = None async def __aenter__(self): @@ -150,7 +149,7 @@ async def _acquire(self): async def __aexit__(self, exc_type, exc_value, traceback): pass - async def cas(self, value: Any, **kwargs) -> bool: + async def cas(self, value: Any, **kwargs: Any) -> bool: """ Checks and sets the specified value for the locked key. If the value has changed since the lock was created, it will raise an :class:`aiocache.lock.OptimisticLockError` diff --git a/aiocache/py.typed b/aiocache/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/aiocache/serializers/__init__.py b/aiocache/serializers/__init__.py index d2ee5d9d1..c7499335b 100644 --- a/aiocache/serializers/__init__.py +++ b/aiocache/serializers/__init__.py @@ -2,10 +2,10 @@ from .serializers import ( BaseSerializer, + JsonSerializer, NullSerializer, - StringSerializer, PickleSerializer, - JsonSerializer, + StringSerializer, ) logger = logging.getLogger(__name__) diff --git a/aiocache/serializers/serializers.py b/aiocache/serializers/serializers.py index e32a004d3..39a67e61b 100644 --- a/aiocache/serializers/serializers.py +++ b/aiocache/serializers/serializers.py @@ -1,13 +1,15 @@ import logging -import pickle +import pickle # noqa: S403 +from abc import ABC, abstractmethod +from typing import Any, Optional logger = logging.getLogger(__name__) try: - import ujson as json + import ujson as json # noqa: I900 except ImportError: logger.debug("ujson module not found, using json") - import json + import json # type: ignore[no-redef] try: import msgpack @@ -19,19 +21,21 @@ _NOT_SET = object() -class BaseSerializer: +class BaseSerializer(ABC): - DEFAULT_ENCODING = "utf-8" + DEFAULT_ENCODING: Optional[str] = "utf-8" def __init__(self, *args, encoding=_NOT_SET, **kwargs): self.encoding = self.DEFAULT_ENCODING if encoding is _NOT_SET else encoding super().__init__(*args, **kwargs) - def dumps(self, value): - raise NotImplementedError("dumps method must be implemented") + @abstractmethod + def dumps(self, value: Any, /) -> Any: + """Serialise the value to be stored in the backend.""" - def loads(self, value): - raise NotImplementedError("loads method must be implemented") + @abstractmethod + def loads(self, value: Any, /) -> Any: + """Decode the value retrieved from the backend.""" class NullSerializer(BaseSerializer): @@ -121,7 +125,7 @@ def loads(self, value): """ if value is None: return None - return pickle.loads(value) + return pickle.loads(value) # noqa: S301 class JsonSerializer(BaseSerializer): diff --git a/docs/conf.py b/docs/conf.py index 88f3d841e..1e409b073 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,8 +20,10 @@ import re import os import sys -sys.path.insert(0, os.path.abspath('..')) -sys.path.insert(0, os.path.abspath('.')) +from pathlib import Path + +sys.path.insert(0, os.path.abspath("..")) +sys.path.insert(0, os.path.abspath(".")) # -- General configuration ------------------------------------------------ @@ -33,47 +35,42 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'aiocache' -copyright = '2016, Manuel Miranda' -author = 'Manuel Miranda' +project = "aiocache" +copyright = "2016, Manuel Miranda" +author = "Manuel Miranda" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # - -with open( - os.path.join( - os.path.abspath(os.path.dirname(__file__)), - '../aiocache/_version.py')) as fp: - try: - version = re.findall( - r"^__version__ = \"([^']+)\"\r?$", fp.read())[0] - release = version - except IndexError: - raise RuntimeError('Unable to determine version.') +_path = Path(__file__).parent.parent / "aiocache/__init__.py" +try: + version = re.findall(r'__version__ = "(.+?)"', _path.read_text())[0] + release = version +except IndexError: + raise RuntimeError("Unable to determine version.") # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -94,7 +91,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -116,7 +113,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -133,11 +130,11 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'default' -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +html_theme = "default" +on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' + html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme @@ -172,7 +169,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -252,34 +249,33 @@ # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'aiocachedoc' +htmlhelp_basename = "aiocachedoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'aiocache.tex', 'aiocache Documentation', - 'Manuel Miranda', 'manual'), + (master_doc, "aiocache.tex", "aiocache Documentation", "Manuel Miranda", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -319,10 +315,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'aiocache', 'aiocache Documentation', - [author], 1) -] +man_pages = [(master_doc, "aiocache", "aiocache Documentation", [author], 1)] # If true, show URL addresses after external links. # @@ -335,9 +328,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'aiocache', 'aiocache Documentation', - author, 'aiocache', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "aiocache", + "aiocache Documentation", + author, + "aiocache", + "One line description of project.", + "Miscellaneous" + ), ] # Documents to append as an appendix to all manuals. diff --git a/docs/index.rst b/docs/index.rst index 3da30b561..5966e1a90 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,12 +24,12 @@ Using a cache is as simple as .. code-block:: python >>> import asyncio - >>> loop = asyncio.get_event_loop() >>> from aiocache import Cache >>> cache = Cache() - >>> loop.run_until_complete(cache.set('key', 'value')) + >>> with asyncio.Runner() as runner: + >>> runner.run(cache.set("key", "value")) True - >>> loop.run_until_complete(cache.get('key')) + >>> runner.run(cache.get("key")) 'value' Here we are using the :ref:`simplememorycache` but you can use any other listed in :ref:`caches`. All caches contain the same minimum interface which consists on the following functions: diff --git a/docs/readthedocs.yml b/docs/readthedocs.yml index e9caa342e..202ee7663 100644 --- a/docs/readthedocs.yml +++ b/docs/readthedocs.yml @@ -5,7 +5,7 @@ build: image: latest python: - version: 3.6 + version: 3.11 pip_install: true extra_requirements: - redis diff --git a/docs/testing.rst b/docs/testing.rst index 8393d90b4..67fb5aa1a 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -5,6 +5,6 @@ It's really easy to cut the dependency with aiocache functionality: .. literalinclude:: ../examples/testing.py -Note that we are passing the :ref:`basecache` as the spec for the Mock (you need to install ``asynctest``). +Note that we are passing the :ref:`basecache` as the spec for the Mock. Also, for debuging purposes you can use `AIOCACHE_DISABLE = 1 python myscript.py` to disable caching. diff --git a/examples/alt_key_builder.py b/examples/alt_key_builder.py new file mode 100644 index 000000000..de13a9ea3 --- /dev/null +++ b/examples/alt_key_builder.py @@ -0,0 +1,293 @@ +"""alt_key_builder.py + + ``key_builder`` is used in two contexts within ``aiocache``, + with different meanings. + 1. Custom ``key_builder`` for a cache -- Prepends a namespace to the key + 2. Custom ``key_builder`` for a cache decorator -- Creates a cache key from + the decorated callable and the callable's arguments + + -------------------------------------------------------------------------- + 1. A custom ``key_builder`` for a cache can manipulate the name of a + cache key; for example to meet naming requirements of the backend. + + ``key_builder`` can also optionally mark the key as belonging to a + namespace group. This enables commonly used key names to be disambiguated + by their ``namespace`` value. It also enables bulk operation on cache keys, + such as expiring all keys in the same namespace. + + ``key_builder`` is expected (but not required) to prefix the passed key + argument with the namespace argument. After initializing the cache object, + the key builder can be accessed via the cache's ``build_key`` member. + + Args: + key (str): undecorated key name + namespace (str, optional): Prefix to add to the key. Defaults to "". + + Returns: + By default, ``cache.build_key()`` returns ``f'{namespace}{sep}{key}'``, + where some backends might include an optional separator, ``sep``. + Some backends might strip or replace illegal characters, and encode + the result before returning it. Typically str or bytes. + + -------------------------------------------------------------------------- + 2. Custom ``key_builder`` for a cache decorator automatically generates a + cache key from the call signature of the decorated callable. It does + not accept a ``namespace`` parameter, and it should not add a + naemspace to the key that it outputs. + + Args: + func (callable): name of the decorated callable + *args: Positional arguments when ``func`` was called. + **kwargs: Keyword arguments when ``func`` was called. + + Returns (str): + By default, the output key is a concatenation of the module and name + of ``func`` + the positional arguments + the sorted keyword arguments. +""" +import asyncio +from typing import List, Dict + +from aiocache import Cache, cached + + +async def demo_key_builders(): + await demo_cache_key_builders() + await demo_cache_key_builders(namespace="demo") + await demo_decorator_key_builders() + + +# 1. Custom ``key_builder`` for a cache +# ------------------------------------- + +def ensure_no_spaces(key, namespace, replace="_"): + """Prefix key with namespace; replace each space with ``replace``""" + aggregate_key = f"{namespace}{key}" + custom_key = aggregate_key.replace(' ', replace) + return custom_key + + +def bytes_key(key, namespace): + """Prefix key with namespace; convert output to bytes""" + aggregate_key = f"{namespace}{key}" + custom_key = aggregate_key.encode() + return custom_key + + +def fixed_key(key, namespace): + """Ignore input, generate a fixed key""" + unchanging_key = "universal key" + return unchanging_key + + +async def demo_cache_key_builders(namespace=None): + """Demonstrate usage and behavior of the custom key_builder functions""" + cache_ns = "cache_namespace" + async with Cache(Cache.MEMORY, key_builder=ensure_no_spaces, namespace=cache_ns) as cache: + raw_key = "Key With Unwanted Spaces" + return_value = 42 + await cache.add(raw_key, return_value, namespace=namespace) + exists = await cache.exists(raw_key, namespace=namespace) + assert exists is True + custom_key = cache.build_key(raw_key, namespace=namespace) + assert ' ' not in custom_key + if namespace is not None: + assert custom_key.startswith(namespace) + else: + # Using cache.namespace instead + exists = await cache.exists(raw_key, namespace=cache_ns) + assert exists is True + custom_key = cache.build_key(raw_key, namespace=cache_ns) + assert custom_key.startswith(cache_ns) + cached_value = await cache.get(raw_key, namespace=namespace) + assert cached_value == return_value + await cache.delete(raw_key, namespace=namespace) + + async with Cache(Cache.MEMORY, key_builder=bytes_key) as cache: + raw_key = "string-key" + return_value = 42 + await cache.add(raw_key, return_value, namespace=namespace) + exists = await cache.exists(raw_key, namespace=namespace) + assert exists is True + custom_key = cache.build_key(raw_key, namespace=namespace) + assert isinstance(custom_key, bytes) + cached_value = await cache.get(raw_key, namespace=namespace) + assert cached_value == return_value + await cache.delete(raw_key, namespace=namespace) + + async with Cache(Cache.MEMORY, key_builder=fixed_key) as cache: + unchanging_key = "universal key" + + for raw_key, return_value in zip( + ("key_1", "key_2", "key_3"), + ("val_1", "val_2", "val_3")): + await cache.set(raw_key, return_value, namespace=namespace) + exists = await cache.exists(raw_key, namespace=namespace) + assert exists is True + custom_key = cache.build_key(raw_key, namespace=namespace) + assert custom_key == unchanging_key + cached_value = await cache.get(raw_key, namespace=namespace) + assert cached_value == return_value + + # Cache key exists regardless of raw_key name + for raw_key in ("key_1", "key_2", "key_3"): + exists = await cache.exists(raw_key, namespace=namespace) + assert exists is True + + cached_value = await cache.get(raw_key, namespace=namespace) + assert cached_value == "val_3" # The last value that was set + await cache.delete(raw_key, namespace=namespace) + + # Deleting one cache key deletes them all + for raw_key in ("key_1", "key_2", "key_3"): + exists = await cache.exists(raw_key, namespace=namespace) + assert exists is False + + +# 2. Custom ``key_builder`` for a cache decorator +# ----------------------------------------------- + +def ignore_kwargs(func, *args, **kwargs): + """Do not use keyword arguments in the cache key's name""" + return ( + (func.__module__ or "") + + func.__name__ + + str(args) + ) + + +def module_override(func, *args, **kwargs): + """Override the module-name prefix for the cache key""" + ordered_kwargs = sorted(kwargs.items()) + return ( + "my_module_alias" + + func.__name__ + + str(args) + + str(ordered_kwargs) + ) + + +def hashed_args(*args, **kwargs): + """Return a hashable key from a callable's parameters""" + key = tuple() + for arg in args: + if isinstance(arg, List): + key += tuple(hashed_args(_arg) for _arg in arg) + elif isinstance(arg, Dict): + key += tuple(sorted( + (_key, hashed_args(_value)) for (_key, _value) in arg.items() + )) + else: + key += (arg, ) + key += tuple(sorted( + (_key, hashed_args(_value)) for (_key, _value) in kwargs.items() + )) + return key + + +def structured_key(func, *args, **kwargs): + """String representation of a structured call signature""" + key = tuple() + key += (func.__module__ or '', ) + key += (func.__qualname__ or func.__name__, ) + key += hashed_args(*args, **kwargs) + return str(key) + + +async def demo_decorator_key_builders(): + """Demonstrate usage and behavior of the custom key_builder functions""" + await demo_ignore_kwargs_decorator() + await demo_module_override_decorator() + await demo_structured_key_decorator() + + +async def demo_ignore_kwargs_decorator(): + """Cache key from positional arguments in call to decorated function""" + @cached(key_builder=ignore_kwargs) + async def fn(a, b=2, c=3): + return (a, b) + + (a, b) = (5, 1) + demo_params = ( + dict(args=(a, b), kwargs=dict(c=3), ret=(a, b)), + dict(args=(a, ), kwargs=dict(b=b, c=3), ret=(a, b)), + dict(args=(a, ), kwargs=dict(c=3), ret=(a, b)), # b from previous call + dict(args=(a, b, 6), kwargs={}, ret=(a, b)), + ) + demo_keys = list() + + for params in demo_params: + args = params["args"] + kwargs = params["kwargs"] + + await fn(*args, **kwargs) + cache = fn.cache + decorator = cached(key_builder=ignore_kwargs) + key = decorator.get_cache_key(fn, args=args, kwargs=kwargs) + exists = await cache.exists(key) + assert exists is True + assert key.endswith(str(args)) + cached_value = await cache.get(key) + assert cached_value == params["ret"] + demo_keys.append(key) + + assert demo_keys[1] == demo_keys[2] + assert demo_keys[0] != demo_keys[1] + assert demo_keys[0] != demo_keys[3] + assert demo_keys[1] != demo_keys[3] + + for key in set(demo_keys): + await cache.delete(key) + + +async def demo_module_override_decorator(): + """Cache key uses custom module name for decorated function""" + @cached(key_builder=module_override) + async def fn(a, b=2, c=3): + return (a, b) + + (a, b) = (5, 1) + args = (a, b) + kwargs = dict(c=3) + return_value = (a, b) + + await fn(*args, **kwargs) + cache = fn.cache + decorator = cached(key_builder=module_override) + key = decorator.get_cache_key(fn, args=args, kwargs=kwargs) + exists = await cache.exists(key) + assert exists is True + assert key.startswith("my_module_alias") + cached_value = await cache.get(key) + assert cached_value == return_value + await cache.delete(key) + + +async def demo_structured_key_decorator(): + """Cache key expresses structure of decorated function call""" + @cached(key_builder=structured_key) + async def fn(a, b=2, c=3): + return (a, b) + + (a, b) = (5, 1) + args = (a, b) + kwargs = dict(c=3) + return_value = (a, b) + fn_module = fn.__module__ or '' + fn_name = fn.__qualname__ or fn.__name__ + key_name = str((fn_module, fn_name) + hashed_args(*args, **kwargs)) + + await fn(*args, **kwargs) + cache = fn.cache + decorator = cached(key_builder=structured_key) + key = decorator.get_cache_key(fn, args=args, kwargs=kwargs) + exists = await cache.exists(key) + assert exists is True + assert key == key_name + cached_value = await cache.get(key) + assert cached_value == return_value + await cache.delete(key) + +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + asyncio.run(demo_key_builders()) diff --git a/examples/cached_alias_config.py b/examples/cached_alias_config.py index 00870365b..27aea69b2 100644 --- a/examples/cached_alias_config.py +++ b/examples/cached_alias_config.py @@ -1,5 +1,7 @@ import asyncio +import redis.asyncio as redis + from aiocache import caches, Cache from aiocache.serializers import StringSerializer, PickleSerializer @@ -12,9 +14,9 @@ }, 'redis_alt': { 'cache': "aiocache.RedisCache", - 'endpoint': "127.0.0.1", + "host": "127.0.0.1", 'port': 6379, - 'timeout': 1, + "socket_connect_timeout": 1, 'serializer': { 'class': "aiocache.serializers.PickleSerializer" }, @@ -36,32 +38,32 @@ async def default_cache(): async def alt_cache(): - # This generates a new instance every time! You can also use `caches.create('alt')` - # or even `caches.create('alt', namespace="test", etc...)` to override extra args - cache = caches.create(**caches.get_alias_config('redis_alt')) + # This generates a new instance every time! You can also use + # `caches.create("alt", namespace="test", etc...)` to override extra args + cache = caches.create("redis_alt") await cache.set("key", "value") assert await cache.get("key") == "value" assert isinstance(cache, Cache.REDIS) assert isinstance(cache.serializer, PickleSerializer) assert len(cache.plugins) == 2 - assert cache.endpoint == "127.0.0.1" - assert cache.timeout == 1 - assert cache.port == 6379 + connection_args = cache.client.connection_pool.connection_kwargs + assert connection_args["host"] == "127.0.0.1" + assert connection_args["socket_connect_timeout"] == 1 + assert connection_args["port"] == 6379 await cache.close() -def test_alias(): - loop = asyncio.get_event_loop() - loop.run_until_complete(default_cache()) - loop.run_until_complete(alt_cache()) +async def test_alias(): + await default_cache() + await alt_cache() - cache = Cache(Cache.REDIS) - loop.run_until_complete(cache.delete("key")) - loop.run_until_complete(cache.close()) + cache = Cache(Cache.REDIS, client=redis.Redis()) + await cache.delete("key") + await cache.close() - loop.run_until_complete(caches.get('default').close()) + await caches.get("default").close() if __name__ == "__main__": - test_alias() + asyncio.run(test_alias()) diff --git a/examples/cached_decorator.py b/examples/cached_decorator.py index 6449037ed..78d1cb112 100644 --- a/examples/cached_decorator.py +++ b/examples/cached_decorator.py @@ -1,6 +1,7 @@ import asyncio from collections import namedtuple +import redis.asyncio as redis from aiocache import cached, Cache from aiocache.serializers import PickleSerializer @@ -9,20 +10,19 @@ @cached( - ttl=10, cache=Cache.REDIS, key="key", serializer=PickleSerializer(), - port=6379, namespace="main") + ttl=10, cache=Cache.REDIS, key_builder=lambda *args, **kw: "key", + serializer=PickleSerializer(), namespace="main", client=redis.Redis()) async def cached_call(): return Result("content", 200) -def test_cached(): - cache = Cache(Cache.REDIS, endpoint="127.0.0.1", port=6379, namespace="main") - loop = asyncio.get_event_loop() - loop.run_until_complete(cached_call()) - assert loop.run_until_complete(cache.exists("key")) is True - loop.run_until_complete(cache.delete("key")) - loop.run_until_complete(cache.close()) +async def test_cached(): + async with Cache(Cache.REDIS, namespace="main", client=redis.Redis()) as cache: + await cached_call() + exists = await cache.exists("key") + assert exists is True + await cache.delete("key") if __name__ == "__main__": - test_cached() + asyncio.run(test_cached()) diff --git a/examples/frameworks/aiohttp_example.py b/examples/frameworks/aiohttp_example.py index c373f8435..e612b30a3 100644 --- a/examples/frameworks/aiohttp_example.py +++ b/examples/frameworks/aiohttp_example.py @@ -35,6 +35,7 @@ async def get_from_cache(self, key): return value except Exception: logging.exception("Couldn't retrieve %s, unexpected error", key) + return None @CachedOverride(key="route_key", serializer=JsonSerializer()) diff --git a/examples/marshmallow_serializer_class.py b/examples/marshmallow_serializer_class.py index 25c9f9232..f45a2ed75 100644 --- a/examples/marshmallow_serializer_class.py +++ b/examples/marshmallow_serializer_class.py @@ -1,6 +1,7 @@ import random import string import asyncio +from typing import Any from marshmallow import fields, Schema, post_load @@ -21,24 +22,32 @@ def __eq__(self, obj): return self.__dict__ == obj.__dict__ -class MarshmallowSerializer(Schema, BaseSerializer): +class RandomSchema(Schema): int_type = fields.Integer() str_type = fields.String() dict_type = fields.Dict() list_type = fields.List(fields.Integer()) - # marshmallow Schema class doesn't play nicely with multiple inheritance and won't call - # BaseSerializer.__init__ - encoding = 'utf-8' - @post_load - def build_my_type(self, data, **kwargs): + def build_my_type(self, data, **kwargs): return RandomModel(**data) class Meta: strict = True +class MarshmallowSerializer(BaseSerializer): + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.schema = RandomSchema() + + def dumps(self, value: Any) -> str: + return self.schema.dumps(value) + + def loads(self, value: str) -> Any: + return self.schema.loads(value) + + cache = Cache(serializer=MarshmallowSerializer(), namespace="main") @@ -54,11 +63,10 @@ async def serializer(): assert result.list_type == model.list_type -def test_serializer(): - loop = asyncio.get_event_loop() - loop.run_until_complete(serializer()) - loop.run_until_complete(cache.delete("key")) +async def test_serializer(): + await serializer() + await cache.delete("key") if __name__ == "__main__": - test_serializer() + asyncio.run(test_serializer()) diff --git a/examples/multicached_decorator.py b/examples/multicached_decorator.py index 7b751aea0..59c0db806 100644 --- a/examples/multicached_decorator.py +++ b/examples/multicached_decorator.py @@ -1,5 +1,7 @@ import asyncio +import redis.asyncio as redis + from aiocache import multi_cached, Cache DICT = { @@ -9,37 +11,35 @@ 'd': "W" } +cache = Cache(Cache.REDIS, namespace="main", client=redis.Redis()) + -@multi_cached("ids", cache=Cache.REDIS, namespace="main") +@multi_cached("ids", cache=Cache.REDIS, namespace="main", client=cache.client) async def multi_cached_ids(ids=None): return {id_: DICT[id_] for id_ in ids} -@multi_cached("keys", cache=Cache.REDIS, namespace="main") +@multi_cached("keys", cache=Cache.REDIS, namespace="main", client=cache.client) async def multi_cached_keys(keys=None): return {id_: DICT[id_] for id_ in keys} -cache = Cache(Cache.REDIS, endpoint="127.0.0.1", port=6379, namespace="main") - - -def test_multi_cached(): - loop = asyncio.get_event_loop() - loop.run_until_complete(multi_cached_ids(ids=['a', 'b'])) - loop.run_until_complete(multi_cached_ids(ids=['a', 'c'])) - loop.run_until_complete(multi_cached_keys(keys=['d'])) +async def test_multi_cached(): + await multi_cached_ids(ids=("a", "b")) + await multi_cached_ids(ids=("a", "c")) + await multi_cached_keys(keys=("d",)) - assert loop.run_until_complete(cache.exists('a')) - assert loop.run_until_complete(cache.exists('b')) - assert loop.run_until_complete(cache.exists('c')) - assert loop.run_until_complete(cache.exists('d')) + assert await cache.exists("a") + assert await cache.exists("b") + assert await cache.exists("c") + assert await cache.exists("d") - loop.run_until_complete(cache.delete("a")) - loop.run_until_complete(cache.delete("b")) - loop.run_until_complete(cache.delete("c")) - loop.run_until_complete(cache.delete("d")) - loop.run_until_complete(cache.close()) + await cache.delete("a") + await cache.delete("b") + await cache.delete("c") + await cache.delete("d") + await cache.close() if __name__ == "__main__": - test_multi_cached() + asyncio.run(test_multi_cached()) diff --git a/examples/optimistic_lock.py b/examples/optimistic_lock.py index f4417f535..422973e47 100644 --- a/examples/optimistic_lock.py +++ b/examples/optimistic_lock.py @@ -2,12 +2,13 @@ import logging import random +import redis.asyncio as redis + from aiocache import Cache from aiocache.lock import OptimisticLock, OptimisticLockError - logger = logging.getLogger(__name__) -cache = Cache(Cache.REDIS, endpoint='127.0.0.1', port=6379, namespace='main') +cache = Cache(Cache.REDIS, namespace="main", client=redis.Redis()) async def expensive_function(): @@ -36,12 +37,11 @@ async def concurrent(): await asyncio.gather(my_view(), my_view(), my_view()) -def test_redis(): - loop = asyncio.get_event_loop() - loop.run_until_complete(concurrent()) - loop.run_until_complete(cache.delete('key')) - loop.run_until_complete(cache.close()) +async def test_redis(): + await concurrent() + await cache.delete("key") + await cache.close() if __name__ == '__main__': - test_redis() + asyncio.run(test_redis()) diff --git a/examples/plugins.py b/examples/plugins.py index c5731ca2d..b870d82b2 100644 --- a/examples/plugins.py +++ b/examples/plugins.py @@ -46,14 +46,13 @@ async def run(): print(cache.profiling) -def test_run(): - loop = asyncio.get_event_loop() - loop.run_until_complete(run()) - loop.run_until_complete(cache.delete("a")) - loop.run_until_complete(cache.delete("b")) - loop.run_until_complete(cache.delete("c")) - loop.run_until_complete(cache.delete("d")) +async def test_run(): + await run() + await cache.delete("a") + await cache.delete("b") + await cache.delete("c") + await cache.delete("d") if __name__ == "__main__": - test_run() + asyncio.run(test_run()) diff --git a/examples/python_object.py b/examples/python_object.py index a0cc8f11e..984fad4c8 100644 --- a/examples/python_object.py +++ b/examples/python_object.py @@ -1,12 +1,14 @@ import asyncio from collections import namedtuple +import redis.asyncio as redis + + from aiocache import Cache from aiocache.serializers import PickleSerializer - MyObject = namedtuple("MyObject", ["x", "y"]) -cache = Cache(Cache.REDIS, serializer=PickleSerializer(), namespace="main") +cache = Cache(Cache.REDIS, serializer=PickleSerializer(), namespace="main", client=redis.Redis()) async def complex_object(): @@ -18,12 +20,11 @@ async def complex_object(): assert my_object.y == 2 -def test_python_object(): - loop = asyncio.get_event_loop() - loop.run_until_complete(complex_object()) - loop.run_until_complete(cache.delete("key")) - loop.run_until_complete(cache.close()) +async def test_python_object(): + await complex_object() + await cache.delete("key") + await cache.close() if __name__ == "__main__": - test_python_object() + asyncio.run(test_python_object()) diff --git a/examples/redlock.py b/examples/redlock.py index a3f37c70e..38d703d77 100644 --- a/examples/redlock.py +++ b/examples/redlock.py @@ -1,12 +1,13 @@ import asyncio import logging +import redis.asyncio as redis + from aiocache import Cache from aiocache.lock import RedLock - logger = logging.getLogger(__name__) -cache = Cache(Cache.REDIS, endpoint='127.0.0.1', port=6379, namespace='main') +cache = Cache(Cache.REDIS, namespace="main", client=redis.Redis()) async def expensive_function(): @@ -32,12 +33,11 @@ async def concurrent(): await asyncio.gather(my_view(), my_view(), my_view()) -def test_redis(): - loop = asyncio.get_event_loop() - loop.run_until_complete(concurrent()) - loop.run_until_complete(cache.delete('key')) - loop.run_until_complete(cache.close()) +async def test_redis(): + await concurrent() + await cache.delete("key") + await cache.close() if __name__ == '__main__': - test_redis() + asyncio.run(test_redis()) diff --git a/examples/serializer_class.py b/examples/serializer_class.py index 4d1c05b29..a91548433 100644 --- a/examples/serializer_class.py +++ b/examples/serializer_class.py @@ -1,6 +1,8 @@ import asyncio import zlib +import redis.asyncio as redis + from aiocache import Cache from aiocache.serializers import BaseSerializer @@ -25,7 +27,7 @@ def loads(self, value): return decompressed -cache = Cache(Cache.REDIS, serializer=CompressionSerializer(), namespace="main") +cache = Cache(Cache.REDIS, serializer=CompressionSerializer(), namespace="main", client=redis.Redis()) async def serializer(): @@ -43,12 +45,11 @@ async def serializer(): assert len(compressed_value) < len(real_value.encode()) -def test_serializer(): - loop = asyncio.get_event_loop() - loop.run_until_complete(serializer()) - loop.run_until_complete(cache.delete("key")) - loop.run_until_complete(cache.close()) +async def test_serializer(): + await serializer() + await cache.delete("key") + await cache.close() if __name__ == "__main__": - test_serializer() + asyncio.run(test_serializer()) diff --git a/examples/serializer_function.py b/examples/serializer_function.py index 8dfdc4ec8..05c5ba04e 100644 --- a/examples/serializer_function.py +++ b/examples/serializer_function.py @@ -1,6 +1,8 @@ import asyncio import json +import redis.asyncio as redis + from marshmallow import Schema, fields, post_load from aiocache import Cache @@ -28,7 +30,7 @@ def loads(value): return MyTypeSchema().loads(value) -cache = Cache(Cache.REDIS, namespace="main") +cache = Cache(Cache.REDIS, namespace="main", client=redis.Redis()) async def serializer_function(): @@ -42,12 +44,11 @@ async def serializer_function(): assert json.loads(await cache.raw("get", "main:key")) == {"y": 2.0, "x": 1.0} -def test_serializer_function(): - loop = asyncio.get_event_loop() - loop.run_until_complete(serializer_function()) - loop.run_until_complete(cache.delete("key")) - loop.run_until_complete(cache.close()) +async def test_serializer_function(): + await serializer_function() + await cache.delete("key") + await cache.close() if __name__ == "__main__": - test_serializer_function() + asyncio.run(test_serializer_function()) diff --git a/examples/simple_redis.py b/examples/simple_redis.py index 7d1b74496..1f429623e 100644 --- a/examples/simple_redis.py +++ b/examples/simple_redis.py @@ -2,8 +2,9 @@ from aiocache import Cache +import redis.asyncio as redis -cache = Cache(Cache.REDIS, endpoint="127.0.0.1", port=6379, namespace="main") +cache = Cache(Cache.REDIS, namespace="main", client=redis.Redis()) async def redis(): @@ -15,13 +16,12 @@ async def redis(): assert await cache.raw("ttl", "main:expire_me") > 0 -def test_redis(): - loop = asyncio.get_event_loop() - loop.run_until_complete(redis()) - loop.run_until_complete(cache.delete("key")) - loop.run_until_complete(cache.delete("expire_me")) - loop.run_until_complete(cache.close()) +async def test_redis(): + await redis() + await cache.delete("key") + await cache.delete("expire_me") + await cache.close() if __name__ == "__main__": - test_redis() + asyncio.run(test_redis()) diff --git a/examples/testing.py b/examples/testing.py index eae4347e0..bd8e4d740 100644 --- a/examples/testing.py +++ b/examples/testing.py @@ -1,16 +1,14 @@ import asyncio - -from asynctest import MagicMock +from unittest.mock import MagicMock from aiocache.base import BaseCache -async def async_main(): +async def main(): mocked_cache = MagicMock(spec=BaseCache) mocked_cache.get.return_value = "world" print(await mocked_cache.get("hello")) if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(async_main()) + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..dd5f13236 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 99 +target-version = ['py38', 'py39', 'py310', 'py311'] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..5bfece6ae --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,10 @@ +-r requirements.txt + +flake8==6.1.0 +flake8-bandit==4.1.1 +flake8-bugbear==23.9.16 +flake8-import-order==0.18.2 +flake8-requirements==1.7.8 +mypy==1.5.1; implementation_name=="cpython" +types-redis==4.6.0.7 +types-ujson==5.8.0.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..c23564fdd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +-e . + +aiomcache==0.8.1 +aiohttp==3.8.5 +marshmallow==3.19.0 +msgpack==1.0.7 +pytest==7.4.2 +pytest-asyncio==0.21.1 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +redis==5.0.1 diff --git a/setup.cfg b/setup.cfg index 2f3f20025..2e23f1ae6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,12 +1,22 @@ [bdist_wheel] universal=1 -[flake8] -max-line-length=100 - [pep8] max-line-length=100 +[tool:pytest] +addopts = --cov=aiocache --cov=tests/ --cov-report term --strict-markers +asyncio_mode = auto +junit_suite_name = aiohttp_test_suite +filterwarnings= + error +testpaths = tests/ +junit_family=xunit2 +xfail_strict = true +markers = + memcached: tests requiring memcached backend + redis: tests requiring redis backend + [coverage:run] branch = True parallel = True diff --git a/setup.py b/setup.py index 95f373cfc..dbd918800 100644 --- a/setup.py +++ b/setup.py @@ -1,62 +1,40 @@ import re -import os +from pathlib import Path -from setuptools import setup, find_packages +from setuptools import setup -with open( - os.path.join( - os.path.abspath(os.path.dirname(__file__)), - 'aiocache/_version.py')) as fp: - try: - version = re.findall( - r"^__version__ = \"([^']+)\"\r?$", fp.read(), re.M)[0] - except IndexError: - raise RuntimeError('Unable to determine version.') +p = Path(__file__).with_name("aiocache") / "__init__.py" +try: + version = re.findall(r"^__version__ = \"([^']+)\"\r?$", p.read_text(), re.M)[0] +except IndexError: + raise RuntimeError("Unable to determine version.") - -with open('README.rst', 'rt', encoding='utf8') as f: - readme = f.read() +readme = Path(__file__).with_name("README.rst").read_text() setup( - name='aiocache', + name="aiocache", version=version, - author='Manuel Miranda', - url='https://github.com/aio-libs/aiocache', - author_email='manu.mirandad@gmail.com', - description='multi backend asyncio cache', + author="Manuel Miranda", + url="https://github.com/aio-libs/aiocache", + author_email="manu.mirandad@gmail.com", + description="multi backend asyncio cache", long_description=readme, classifiers=[ - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Framework :: AsyncIO', + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Framework :: AsyncIO", ], - packages=find_packages(), + packages=("aiocache",), install_requires=None, extras_require={ - 'redis:python_version<"3.7"': ['aioredis>=0.3.3'], - 'redis:python_version>="3.8"': ['aioredis>=1.3.0'], - 'redis:python_version>="3.7" and python_version<"3.8"': ['aioredis>=1.0.0'], - 'memcached': ['aiomcache>=0.5.2'], - 'msgpack': ['msgpack>=0.5.5'], - 'dev': [ - 'asynctest>=0.11.0', - 'black;python_version>="3.6"', - 'codecov', - 'coverage', - 'flake8', - 'ipdb', - 'marshmallow>=3', - 'pystache', - 'pytest', - 'pytest-asyncio', - 'pytest-mock', - 'sphinx', - 'sphinx-autobuild', - 'sphinx-rtd-theme', - ] - } + "redis": ["redis>=4.2.0"], + "memcached": ["aiomcache>=0.5.2"], + "msgpack": ["msgpack>=0.5.5"], + }, + include_package_data=True, ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/acceptance/conftest.py b/tests/acceptance/conftest.py index 37b5190e0..0d5a306f5 100644 --- a/tests/acceptance/conftest.py +++ b/tests/acceptance/conftest.py @@ -1,17 +1,9 @@ +import asyncio + import pytest from aiocache import Cache, caches -from aiocache.backends.redis import RedisBackend - - -def pytest_configure(): - """ - Before pytest_namespace was being used to set the keys for - testing but the feature was removed - https://docs.pytest.org/en/latest/deprecations.html#pytest-namespace - """ - pytest.KEY = "key" - pytest.KEY_1 = "random" +from ..utils import KEY_LOCK, Keys @pytest.fixture(autouse=True) @@ -27,44 +19,32 @@ def reset_caches(): ) -@pytest.fixture(autouse=True) -def reset_redis_pools(): - RedisBackend.pools = {} - - @pytest.fixture -def redis_cache(event_loop): - cache = Cache(Cache.REDIS, namespace="test") - yield cache - - event_loop.run_until_complete(cache.delete(pytest.KEY)) - event_loop.run_until_complete(cache.delete(pytest.KEY_1)) - event_loop.run_until_complete(cache.delete(pytest.KEY + "-lock")) - event_loop.run_until_complete(cache.close()) +async def redis_cache(redis_client): + async with Cache(Cache.REDIS, namespace="test", client=redis_client) as cache: + yield cache + await asyncio.gather(*(cache.delete(k) for k in (*Keys, KEY_LOCK))) @pytest.fixture -def memory_cache(event_loop): - cache = Cache(namespace="test") - yield cache - - event_loop.run_until_complete(cache.delete(pytest.KEY)) - event_loop.run_until_complete(cache.delete(pytest.KEY_1)) - event_loop.run_until_complete(cache.delete(pytest.KEY + "-lock")) - event_loop.run_until_complete(cache.close()) +async def memory_cache(): + async with Cache(namespace="test") as cache: + yield cache + await asyncio.gather(*(cache.delete(k) for k in (*Keys, KEY_LOCK))) @pytest.fixture -def memcached_cache(event_loop): - cache = Cache(Cache.MEMCACHED, namespace="test") - yield cache - - event_loop.run_until_complete(cache.delete(pytest.KEY)) - event_loop.run_until_complete(cache.delete(pytest.KEY_1)) - event_loop.run_until_complete(cache.delete(pytest.KEY + "-lock")) - event_loop.run_until_complete(cache.close()) - - -@pytest.fixture(params=["redis_cache", "memory_cache", "memcached_cache"]) +async def memcached_cache(): + async with Cache(Cache.MEMCACHED, namespace="test") as cache: + yield cache + await asyncio.gather(*(cache.delete(k) for k in (*Keys, KEY_LOCK))) + + +@pytest.fixture( + params=( + pytest.param("redis_cache", marks=pytest.mark.redis), + "memory_cache", + pytest.param("memcached_cache", marks=pytest.mark.memcached), + )) def cache(request): return request.getfixturevalue(request.param) diff --git a/tests/acceptance/test_base.py b/tests/acceptance/test_base.py index 5452e60b4..1a9e6fc0c 100644 --- a/tests/acceptance/test_base.py +++ b/tests/acceptance/test_base.py @@ -1,8 +1,10 @@ -import pytest import asyncio -from aiocache import RedisCache, SimpleMemoryCache, MemcachedCache +import pytest + +from aiocache.backends.memory import SimpleMemoryCache from aiocache.base import _Conn +from ..utils import Keys class TestCache: @@ -12,268 +14,239 @@ class TestCache: cache fixture """ - @pytest.mark.asyncio async def test_setup(self, cache): assert cache.namespace == "test" - @pytest.mark.asyncio async def test_get_missing(self, cache): - assert await cache.get(pytest.KEY) is None - assert await cache.get(pytest.KEY, default=1) == 1 + assert await cache.get(Keys.KEY) is None + assert await cache.get(Keys.KEY, default=1) == 1 - @pytest.mark.asyncio async def test_get_existing(self, cache): - await cache.set(pytest.KEY, "value") - assert await cache.get(pytest.KEY) == "value" + await cache.set(Keys.KEY, "value") + assert await cache.get(Keys.KEY) == "value" - @pytest.mark.asyncio async def test_multi_get(self, cache): - await cache.set(pytest.KEY, "value") - assert await cache.multi_get([pytest.KEY, pytest.KEY_1]) == ["value", None] + await cache.set(Keys.KEY, "value") + assert await cache.multi_get([Keys.KEY, Keys.KEY_1]) == ["value", None] - @pytest.mark.asyncio async def test_delete_missing(self, cache): - assert await cache.delete(pytest.KEY) == 0 + result = await cache.delete(Keys.KEY) + assert result == 0 - @pytest.mark.asyncio async def test_delete_existing(self, cache): - await cache.set(pytest.KEY, "value") - assert await cache.delete(pytest.KEY) == 1 + await cache.set(Keys.KEY, "value") + result = await cache.delete(Keys.KEY) + assert result == 1 - assert await cache.get(pytest.KEY) is None + value = await cache.get(Keys.KEY) + assert value is None - @pytest.mark.asyncio async def test_set(self, cache): - assert await cache.set(pytest.KEY, "value") is True + assert await cache.set(Keys.KEY, "value") is True - @pytest.mark.asyncio async def test_set_cancel_previous_ttl_handle(self, cache): - await cache.set(pytest.KEY, "value", ttl=2) + await cache.set(Keys.KEY, "value", ttl=4) - await asyncio.sleep(1) - assert await cache.get(pytest.KEY) == "value" - await cache.set(pytest.KEY, "new_value", ttl=2) + await asyncio.sleep(2.1) + # Smaller ttl seems flaky, as if this call takes >0.5s... + result = await cache.get(Keys.KEY) + assert result == "value" + await cache.set(Keys.KEY, "new_value", ttl=4) - await asyncio.sleep(1) - assert await cache.get(pytest.KEY) == "new_value" + await asyncio.sleep(2) + result = await cache.get(Keys.KEY) + assert result == "new_value" - @pytest.mark.asyncio async def test_multi_set(self, cache): - pairs = [(pytest.KEY, "value"), [pytest.KEY_1, "random_value"]] + pairs = [(Keys.KEY, "value"), [Keys.KEY_1, "random_value"]] assert await cache.multi_set(pairs) is True - assert await cache.multi_get([pytest.KEY, pytest.KEY_1]) == ["value", "random_value"] + assert await cache.multi_get([Keys.KEY, Keys.KEY_1]) == ["value", "random_value"] - @pytest.mark.asyncio async def test_multi_set_with_ttl(self, cache): - pairs = [(pytest.KEY, "value"), [pytest.KEY_1, "random_value"]] + pairs = [(Keys.KEY, "value"), [Keys.KEY_1, "random_value"]] assert await cache.multi_set(pairs, ttl=1) is True await asyncio.sleep(1.1) - assert await cache.multi_get([pytest.KEY, pytest.KEY_1]) == [None, None] + assert await cache.multi_get([Keys.KEY, Keys.KEY_1]) == [None, None] - @pytest.mark.asyncio async def test_set_with_ttl(self, cache): - await cache.set(pytest.KEY, "value", ttl=1) + await cache.set(Keys.KEY, "value", ttl=1) await asyncio.sleep(1.1) - assert await cache.get(pytest.KEY) is None + assert await cache.get(Keys.KEY) is None - @pytest.mark.asyncio async def test_add_missing(self, cache): - assert await cache.add(pytest.KEY, "value", ttl=1) is True + assert await cache.add(Keys.KEY, "value", ttl=1) is True - @pytest.mark.asyncio async def test_add_existing(self, cache): - await cache.set(pytest.KEY, "value") is True + assert await cache.set(Keys.KEY, "value") is True with pytest.raises(ValueError): - await cache.add(pytest.KEY, "value") + await cache.add(Keys.KEY, "value") - @pytest.mark.asyncio async def test_exists_missing(self, cache): - assert await cache.exists(pytest.KEY) is False + assert await cache.exists(Keys.KEY) is False - @pytest.mark.asyncio async def test_exists_existing(self, cache): - await cache.set(pytest.KEY, "value") - assert await cache.exists(pytest.KEY) is True + await cache.set(Keys.KEY, "value") + assert await cache.exists(Keys.KEY) is True - @pytest.mark.asyncio async def test_increment_missing(self, cache): - assert await cache.increment(pytest.KEY, delta=2) == 2 - assert await cache.increment(pytest.KEY_1, delta=-2) == -2 + assert await cache.increment(Keys.KEY, delta=2) == 2 + assert await cache.increment(Keys.KEY_1, delta=-2) == -2 - @pytest.mark.asyncio async def test_increment_existing(self, cache): - await cache.set(pytest.KEY, 2) - assert await cache.increment(pytest.KEY, delta=2) == 4 - assert await cache.increment(pytest.KEY, delta=1) == 5 - assert await cache.increment(pytest.KEY, delta=-3) == 2 + await cache.set(Keys.KEY, 2) + assert await cache.increment(Keys.KEY, delta=2) == 4 + assert await cache.increment(Keys.KEY, delta=1) == 5 + assert await cache.increment(Keys.KEY, delta=-3) == 2 - @pytest.mark.asyncio async def test_increment_typeerror(self, cache): - await cache.set(pytest.KEY, "value") + await cache.set(Keys.KEY, "value") with pytest.raises(TypeError): - assert await cache.increment(pytest.KEY) + assert await cache.increment(Keys.KEY) - @pytest.mark.asyncio async def test_expire_existing(self, cache): - await cache.set(pytest.KEY, "value") - assert await cache.expire(pytest.KEY, 1) is True + await cache.set(Keys.KEY, "value") + assert await cache.expire(Keys.KEY, 1) is True await asyncio.sleep(1.1) - assert await cache.exists(pytest.KEY) is False + assert await cache.exists(Keys.KEY) is False - @pytest.mark.asyncio async def test_expire_with_0(self, cache): - await cache.set(pytest.KEY, "value", 1) - assert await cache.expire(pytest.KEY, 0) is True + await cache.set(Keys.KEY, "value", 1) + assert await cache.expire(Keys.KEY, 0) is True await asyncio.sleep(1.1) - assert await cache.exists(pytest.KEY) is True + assert await cache.exists(Keys.KEY) is True - @pytest.mark.asyncio async def test_expire_missing(self, cache): - assert await cache.expire(pytest.KEY, 1) is False + assert await cache.expire(Keys.KEY, 1) is False - @pytest.mark.asyncio async def test_clear(self, cache): - await cache.set(pytest.KEY, "value") + await cache.set(Keys.KEY, "value") await cache.clear() - assert await cache.exists(pytest.KEY) is False + assert await cache.exists(Keys.KEY) is False - @pytest.mark.asyncio async def test_close_pool_only_clears_resources(self, cache): - await cache.set(pytest.KEY, "value") + await cache.set(Keys.KEY, "value") await cache.close() - assert await cache.set(pytest.KEY, "value") is True - assert await cache.get(pytest.KEY) == "value" + assert await cache.set(Keys.KEY, "value") is True + assert await cache.get(Keys.KEY) == "value" - @pytest.mark.asyncio async def test_single_connection(self, cache): async with cache.get_connection() as conn: assert isinstance(conn, _Conn) - assert await conn.set(pytest.KEY, "value") is True - assert await conn.get(pytest.KEY) == "value" + assert await conn.set(Keys.KEY, "value") is True + assert await conn.get(Keys.KEY) == "value" class TestMemoryCache: - @pytest.mark.asyncio async def test_accept_explicit_args(self): with pytest.raises(TypeError): SimpleMemoryCache(random_attr="wtf") - @pytest.mark.asyncio async def test_set_float_ttl(self, memory_cache): - await memory_cache.set(pytest.KEY, "value", ttl=0.1) + await memory_cache.set(Keys.KEY, "value", ttl=0.1) await asyncio.sleep(0.15) - assert await memory_cache.get(pytest.KEY) is None + assert await memory_cache.get(Keys.KEY) is None - @pytest.mark.asyncio async def test_multi_set_float_ttl(self, memory_cache): - pairs = [(pytest.KEY, "value"), [pytest.KEY_1, "random_value"]] + pairs = [(Keys.KEY, "value"), [Keys.KEY_1, "random_value"]] assert await memory_cache.multi_set(pairs, ttl=0.1) is True await asyncio.sleep(0.15) - assert await memory_cache.multi_get([pytest.KEY, pytest.KEY_1]) == [None, None] + assert await memory_cache.multi_get([Keys.KEY, Keys.KEY_1]) == [None, None] - @pytest.mark.asyncio async def test_raw(self, memory_cache): await memory_cache.raw("setdefault", "key", "value") assert await memory_cache.raw("get", "key") == "value" assert list(await memory_cache.raw("keys")) == ["key"] - @pytest.mark.asyncio async def test_clear_with_namespace_memory(self, memory_cache): - await memory_cache.set(pytest.KEY, "value", namespace="test") + await memory_cache.set(Keys.KEY, "value", namespace="test") await memory_cache.clear(namespace="test") - assert await memory_cache.exists(pytest.KEY, namespace="test") is False + assert await memory_cache.exists(Keys.KEY, namespace="test") is False +@pytest.mark.memcached class TestMemcachedCache: - @pytest.mark.asyncio async def test_accept_explicit_args(self): + from aiocache.backends.memcached import MemcachedCache + with pytest.raises(TypeError): MemcachedCache(random_attr="wtf") - @pytest.mark.asyncio async def test_set_too_long_key(self, memcached_cache): with pytest.raises(TypeError) as exc_info: await memcached_cache.set("a" * 2000, "value") assert str(exc_info.value).startswith("aiomcache error: invalid key") - @pytest.mark.asyncio async def test_set_float_ttl_fails(self, memcached_cache): with pytest.raises(TypeError) as exc_info: - await memcached_cache.set(pytest.KEY, "value", ttl=0.1) + await memcached_cache.set(Keys.KEY, "value", ttl=0.1) assert str(exc_info.value) == "aiomcache error: exptime not int: 0.1" - @pytest.mark.asyncio async def test_multi_set_float_ttl(self, memcached_cache): with pytest.raises(TypeError) as exc_info: - pairs = [(pytest.KEY, "value"), [pytest.KEY_1, "random_value"]] + pairs = [(Keys.KEY, "value"), [Keys.KEY_1, "random_value"]] assert await memcached_cache.multi_set(pairs, ttl=0.1) is True assert str(exc_info.value) == "aiomcache error: exptime not int: 0.1" - @pytest.mark.asyncio async def test_raw(self, memcached_cache): await memcached_cache.raw("set", b"key", b"value") assert await memcached_cache.raw("get", b"key") == "value" assert await memcached_cache.raw("prepend", b"key", b"super") is True assert await memcached_cache.raw("get", b"key") == "supervalue" - @pytest.mark.asyncio async def test_clear_with_namespace_memcached(self, memcached_cache): - await memcached_cache.set(pytest.KEY, "value", namespace="test") + await memcached_cache.set(Keys.KEY, "value", namespace="test") with pytest.raises(ValueError): await memcached_cache.clear(namespace="test") - assert await memcached_cache.exists(pytest.KEY, namespace="test") is True + assert await memcached_cache.exists(Keys.KEY, namespace="test") is True - @pytest.mark.asyncio async def test_close(self, memcached_cache): - await memcached_cache.set(pytest.KEY, "value") + await memcached_cache.set(Keys.KEY, "value") await memcached_cache._close() assert memcached_cache.client._pool._pool.qsize() == 0 +@pytest.mark.redis class TestRedisCache: - @pytest.mark.asyncio async def test_accept_explicit_args(self): + from aiocache.backends.redis import RedisCache + with pytest.raises(TypeError): RedisCache(random_attr="wtf") - @pytest.mark.asyncio async def test_float_ttl(self, redis_cache): - await redis_cache.set(pytest.KEY, "value", ttl=0.1) + await redis_cache.set(Keys.KEY, "value", ttl=0.1) await asyncio.sleep(0.15) - assert await redis_cache.get(pytest.KEY) is None + assert await redis_cache.get(Keys.KEY) is None - @pytest.mark.asyncio async def test_multi_set_float_ttl(self, redis_cache): - pairs = [(pytest.KEY, "value"), [pytest.KEY_1, "random_value"]] + pairs = [(Keys.KEY, "value"), [Keys.KEY_1, "random_value"]] assert await redis_cache.multi_set(pairs, ttl=0.1) is True await asyncio.sleep(0.15) - assert await redis_cache.multi_get([pytest.KEY, pytest.KEY_1]) == [None, None] + assert await redis_cache.multi_get([Keys.KEY, Keys.KEY_1]) == [None, None] - @pytest.mark.asyncio async def test_raw(self, redis_cache): await redis_cache.raw("set", "key", "value") assert await redis_cache.raw("get", "key") == "value" assert await redis_cache.raw("keys", "k*") == ["key"] + # .raw() doesn't build key with namespace prefix, clear it manually + await redis_cache.raw("delete", "key") - @pytest.mark.asyncio async def test_clear_with_namespace_redis(self, redis_cache): - await redis_cache.set(pytest.KEY, "value", namespace="test") + await redis_cache.set(Keys.KEY, "value", namespace="test") await redis_cache.clear(namespace="test") - assert await redis_cache.exists(pytest.KEY, namespace="test") is False + assert await redis_cache.exists(Keys.KEY, namespace="test") is False - @pytest.mark.asyncio async def test_close(self, redis_cache): - await redis_cache.set(pytest.KEY, "value") + await redis_cache.set(Keys.KEY, "value") await redis_cache._close() - assert redis_cache._pool.size == 0 diff --git a/tests/acceptance/test_decorators.py b/tests/acceptance/test_decorators.py index 188ab12b7..ad99aca74 100644 --- a/tests/acceptance/test_decorators.py +++ b/tests/acceptance/test_decorators.py @@ -1,45 +1,42 @@ import asyncio -import pytest import random - from unittest import mock +import pytest + from aiocache import cached, cached_stampede, multi_cached +from ..utils import Keys, ensure_key async def return_dict(keys=None): ret = {} - for value, key in enumerate(keys or [pytest.KEY, pytest.KEY_1]): + for value, key in enumerate(keys or [Keys.KEY, Keys.KEY_1]): ret[key] = str(value) return ret -async def stub(*args, key=None, seconds=0, **kwargs): +async def stub(arg: float, seconds: int = 0) -> str: await asyncio.sleep(seconds) - if key: - return str(key) return str(random.randint(1, 50)) class TestCached: @pytest.fixture(autouse=True) def default_cache(self, mocker, cache): - mocker.patch("aiocache.decorators._get_cache", return_value=cache) + mocker.patch("aiocache.decorators._get_cache", autospec=True, return_value=cache) - @pytest.mark.asyncio async def test_cached_ttl(self, cache): - @cached(ttl=1, key=pytest.KEY) + @cached(ttl=2, key_builder=lambda *args, **kw: Keys.KEY) async def fn(): return str(random.randint(1, 50)) resp1 = await fn() resp2 = await fn() - assert await cache.get(pytest.KEY) == resp1 == resp2 - await asyncio.sleep(1) - assert await cache.get(pytest.KEY) is None + assert await cache.get(Keys.KEY) == resp1 == resp2 + await asyncio.sleep(2.1) + assert await cache.get(Keys.KEY) is None - @pytest.mark.asyncio async def test_cached_key_builder(self, cache): def build_key(f, self, a, b): return "{}_{}_{}_{}".format(self, f.__name__, a, b) @@ -51,26 +48,74 @@ async def fn(self, a, b=2): await fn("self", 1, 3) assert await cache.exists(build_key(fn, "self", 1, 3)) is True + @pytest.mark.parametrize("decorator", (cached, cached_stampede)) + async def test_cached_skip_cache_func(self, cache, decorator): + @decorator(skip_cache_func=lambda r: r is None) + async def sk_func(x): + return x if x > 0 else None + + arg = 1 + res = await sk_func(arg) + assert res + + key = decorator().get_cache_key(sk_func, args=(1,), kwargs={}) + + assert key + assert await cache.exists(key) + assert await cache.get(key) == res + + arg = -1 + + await sk_func(arg) + + key = decorator().get_cache_key(sk_func, args=(-1,), kwargs={}) + + assert key + assert not await cache.exists(key) + + async def test_cached_without_namespace(self, cache): + """Default cache key is created when no namespace is provided""" + @cached(namespace=None) + async def fn(): + return "1" + + await fn() + decorator = cached(namespace=None) + key = decorator.get_cache_key(fn, args=(), kwargs={}) + assert await cache.exists(key, namespace=None) is True + + async def test_cached_with_namespace(self, cache): + """Cache key is prefixed with provided namespace""" + key_prefix = "test" + + @cached(namespace=key_prefix) + async def ns_fn(): + return "1" + + await ns_fn() + decorator = cached(namespace=key_prefix) + key = decorator.get_cache_key(ns_fn, args=(), kwargs={}) + assert await cache.exists(key, namespace=key_prefix) is True + class TestCachedStampede: @pytest.fixture(autouse=True) def default_cache(self, mocker, cache): - mocker.patch("aiocache.decorators._get_cache", return_value=cache) + mocker.patch("aiocache.decorators._get_cache", autospec=True, return_value=cache) - @pytest.mark.asyncio async def test_cached_stampede(self, mocker, cache): mocker.spy(cache, "get") mocker.spy(cache, "set") - decorator = cached_stampede(ttl=10, lease=2) + decorator = cached_stampede(ttl=10, lease=3) await asyncio.gather(decorator(stub)(0.5), decorator(stub)(0.5)) - cache.get.assert_called_with("acceptance.test_decoratorsstub(0.5,)[]") + cache.get.assert_called_with("tests.acceptance.test_decoratorsstub(0.5,)[]") assert cache.get.call_count == 4 - cache.set.assert_called_with("acceptance.test_decoratorsstub(0.5,)[]", mock.ANY, ttl=10) - assert cache.set.call_count == 1 + cache.set.assert_called_with("tests.acceptance.test_decoratorsstub(0.5,)[]", + mock.ANY, ttl=10) + assert cache.set.call_count == 1, cache.set.call_args_list - @pytest.mark.asyncio async def test_locking_dogpile_lease_expiration(self, mocker, cache): mocker.spy(cache, "get") mocker.spy(cache, "set") @@ -85,8 +130,7 @@ async def test_locking_dogpile_lease_expiration(self, mocker, cache): assert cache.get.call_count == 6 assert cache.set.call_count == 3 - @pytest.mark.asyncio - async def test_locking_dogpile_task_cancellation(self, mocker, cache): + async def test_locking_dogpile_task_cancellation(self, cache): @cached_stampede() async def cancel_task(): raise asyncio.CancelledError() @@ -98,51 +142,59 @@ async def cancel_task(): class TestMultiCachedDecorator: @pytest.fixture(autouse=True) def default_cache(self, mocker, cache): - mocker.patch("aiocache.decorators._get_cache", return_value=cache) + mocker.patch("aiocache.decorators._get_cache", autospec=True, return_value=cache) - @pytest.mark.asyncio async def test_multi_cached(self, cache): multi_cached_decorator = multi_cached("keys") - default_keys = {pytest.KEY, pytest.KEY_1} + default_keys = {Keys.KEY, Keys.KEY_1} await multi_cached_decorator(return_dict)(keys=default_keys) for key in default_keys: assert await cache.get(key) is not None - @pytest.mark.asyncio async def test_keys_without_kwarg(self, cache): @multi_cached("keys") async def fn(keys): - return {pytest.KEY: 1} + return {Keys.KEY: 1} - await fn([pytest.KEY]) - assert await cache.exists(pytest.KEY) is True + await fn([Keys.KEY]) + assert await cache.exists(Keys.KEY) is True - @pytest.mark.asyncio async def test_multi_cached_key_builder(self, cache): def build_key(key, f, self, keys, market="ES"): - return "{}_{}_{}".format(f.__name__, key, market) + return "{}_{}_{}".format(f.__name__, ensure_key(key), market) @multi_cached(keys_from_attr="keys", key_builder=build_key) async def fn(self, keys, market="ES"): - return {pytest.KEY: 1, pytest.KEY_1: 2} + return {Keys.KEY: 1, Keys.KEY_1: 2} + + await fn("self", keys=[Keys.KEY, Keys.KEY_1]) + assert await cache.exists("fn_" + ensure_key(Keys.KEY) + "_ES") is True + assert await cache.exists("fn_" + ensure_key(Keys.KEY_1) + "_ES") is True + + async def test_multi_cached_skip_keys(self, cache): + @multi_cached(keys_from_attr="keys", skip_cache_func=lambda _, v: v is None) + async def multi_sk_fn(keys, values): + return {k: v for k, v in zip(keys, values)} + + res = await multi_sk_fn(keys=[Keys.KEY, Keys.KEY_1], values=[42, None]) + assert res + assert Keys.KEY in res and Keys.KEY_1 in res - await fn("self", keys=[pytest.KEY, pytest.KEY_1]) - assert await cache.exists("fn_" + pytest.KEY + "_ES") is True - assert await cache.exists("fn_" + pytest.KEY_1 + "_ES") is True + assert await cache.exists(Keys.KEY) + assert await cache.get(Keys.KEY) == res[Keys.KEY] + assert not await cache.exists(Keys.KEY_1) - @pytest.mark.asyncio async def test_fn_with_args(self, cache): @multi_cached("keys") async def fn(keys, *args): assert len(args) == 1 - return {pytest.KEY: 1} + return {Keys.KEY: 1} - await fn([pytest.KEY], "arg") - assert await cache.exists(pytest.KEY) is True + await fn([Keys.KEY], "arg") + assert await cache.exists(Keys.KEY) is True - @pytest.mark.asyncio async def test_double_decorator(self, cache): def dummy_d(fn): async def wrapper(*args, **kwargs): @@ -153,7 +205,7 @@ async def wrapper(*args, **kwargs): @dummy_d @multi_cached("keys") async def fn(keys): - return {pytest.KEY: 1} + return {Keys.KEY: 1} - await fn([pytest.KEY]) - assert await cache.exists(pytest.KEY) is True + await fn([Keys.KEY]) + assert await cache.exists(Keys.KEY) is True diff --git a/tests/acceptance/test_factory.py b/tests/acceptance/test_factory.py index 4bb77d5e3..4a3bb4f8c 100644 --- a/tests/acceptance/test_factory.py +++ b/tests/acceptance/test_factory.py @@ -1,41 +1,52 @@ import pytest -from aiocache import Cache, SimpleMemoryCache, RedisCache, MemcachedCache +from aiocache import Cache +from aiocache.backends.memory import SimpleMemoryCache class TestCache: - def test_from_url_memory(self): - cache = Cache.from_url("memory://") - - assert isinstance(cache, SimpleMemoryCache) + async def test_from_url_memory(self): + async with Cache.from_url("memory://") as cache: + assert isinstance(cache, SimpleMemoryCache) def test_from_url_memory_no_endpoint(self): with pytest.raises(TypeError): - Cache.from_url("memory://endpoint:10") - - def test_from_url_redis(self): - cache = Cache.from_url( - "redis://endpoint:1000/0/?password=pass&pool_min_size=40" - "&pool_max_size=50&create_connection_timeout=20" - ) - - assert isinstance(cache, RedisCache) - assert cache.endpoint == "endpoint" - assert cache.port == 1000 - assert cache.password == "pass" - assert cache.pool_min_size == 40 - assert cache.pool_max_size == 50 - assert cache.create_connection_timeout == 20 - - def test_from_url_memcached(self): - cache = Cache.from_url("memcached://endpoint:1000?pool_size=10") - - assert isinstance(cache, MemcachedCache) - assert cache.endpoint == "endpoint" - assert cache.port == 1000 - assert cache.pool_size == 10 - - @pytest.mark.parametrize("scheme", ["memory", "redis", "memcached"]) + Cache.from_url("memory://host:10") + + @pytest.mark.redis + async def test_from_url_redis(self): + from aiocache.backends.redis import RedisCache + + url = ("redis://endpoint:1000/0/?password=pass" + + "&max_connections=50&socket_connect_timeout=20") + + async with Cache.from_url(url) as cache: + assert isinstance(cache, RedisCache) + connection_args = cache.client.connection_pool.connection_kwargs + assert connection_args["host"] == "endpoint" + assert connection_args["port"] == 1000 + assert connection_args["password"] == "pass" + assert cache.client.connection_pool.max_connections == 50 + assert connection_args["socket_connect_timeout"] == 20 + + @pytest.mark.memcached + async def test_from_url_memcached(self): + from aiocache.backends.memcached import MemcachedCache + + url = "memcached://endpoint:1000?pool_size=10" + + async with Cache.from_url(url) as cache: + assert isinstance(cache, MemcachedCache) + assert cache.host == "endpoint" + assert cache.port == 1000 + assert cache.pool_size == 10 + + @pytest.mark.parametrize( + "scheme", + (pytest.param("redis", marks=pytest.mark.redis), + "memory", + pytest.param("memcached", marks=pytest.mark.memcached), + )) def test_from_url_unexpected_param(self, scheme): with pytest.raises(TypeError): Cache.from_url("{}://?arg1=arg1".format(scheme)) diff --git a/tests/acceptance/test_lock.py b/tests/acceptance/test_lock.py index e7ea889f4..3e5a53792 100644 --- a/tests/acceptance/test_lock.py +++ b/tests/acceptance/test_lock.py @@ -1,80 +1,95 @@ import asyncio + import pytest -from aiocache.lock import RedLock, OptimisticLock, OptimisticLockError +from aiocache.lock import OptimisticLock, OptimisticLockError, RedLock from aiocache.serializers import StringSerializer +from ..utils import KEY_LOCK, Keys @pytest.fixture def lock(cache): - return RedLock(cache, pytest.KEY, 20) + return RedLock(cache, Keys.KEY, 20) + + +def build_key(key, namespace=None): + return "custom_key" + + +def build_key_bytes(key, namespace=None): + return b"custom_key" + + +@pytest.fixture +def custom_redis_cache(mocker, redis_cache, build_key=build_key): + mocker.patch.object(redis_cache, "build_key", new=build_key) + yield redis_cache + + +@pytest.fixture +def custom_memory_cache(mocker, memory_cache, build_key=build_key): + mocker.patch.object(memory_cache, "build_key", new=build_key) + yield memory_cache + + +@pytest.fixture +def custom_memcached_cache(mocker, memcached_cache, build_key=build_key_bytes): + mocker.patch.object(memcached_cache, "build_key", new=build_key) + yield memcached_cache class TestRedLock: - @pytest.mark.asyncio async def test_acquire(self, cache, lock): cache.serializer = StringSerializer() async with lock: - assert await cache.get(pytest.KEY + "-lock") == lock._value + assert await cache.get(KEY_LOCK) == lock._value - @pytest.mark.asyncio async def test_release_does_nothing_when_no_lock(self, lock): assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_acquire_release(self, cache, lock): async with lock: pass - assert await cache.get(pytest.KEY + "-lock") is None + assert await cache.get(KEY_LOCK) is None - @pytest.mark.asyncio async def test_locking_dogpile(self, mocker, cache): mocker.spy(cache, "get") mocker.spy(cache, "set") mocker.spy(cache, "_add") async def dummy(): - res = await cache.get(pytest.KEY) - if res is not None: - return res + res = await cache.get(Keys.KEY) + assert res is None - async with RedLock(cache, pytest.KEY, lease=5): - res = await cache.get(pytest.KEY) + async with RedLock(cache, Keys.KEY, lease=5): + res = await cache.get(Keys.KEY) if res is not None: - return res + return await asyncio.sleep(0.1) - await cache.set(pytest.KEY, "value") + await cache.set(Keys.KEY, "value") await asyncio.gather(dummy(), dummy(), dummy(), dummy()) assert cache._add.call_count == 4 assert cache.get.call_count == 8 - assert cache.set.call_count == 1 + assert cache.set.call_count == 1, cache.set.call_args_list - @pytest.mark.asyncio - async def test_locking_dogpile_lease_expiration(self, mocker, cache): - mocker.spy(cache, "get") - mocker.spy(cache, "set") - - async def dummy(): - res = await cache.get(pytest.KEY) - if res is not None: - return res + async def test_locking_dogpile_lease_expiration(self, cache): + async def dummy() -> None: + res = await cache.get(Keys.KEY) + assert res is None - async with RedLock(cache, pytest.KEY, lease=1): - res = await cache.get(pytest.KEY) - if res is not None: - return res + # Lease should expire before cache is set, so res is still None. + async with RedLock(cache, Keys.KEY, lease=1): + res = await cache.get(Keys.KEY) + assert res is None await asyncio.sleep(1.1) - await cache.set(pytest.KEY, "value") + await cache.set(Keys.KEY, "value") await asyncio.gather(dummy(), dummy(), dummy(), dummy()) - assert cache.get.call_count == 8 - assert cache.set.call_count == 4 - @pytest.mark.asyncio async def test_locking_dogpile_propagates_exceptions(self, cache): async def dummy(): - async with RedLock(cache, pytest.KEY, lease=1): + async with RedLock(cache, Keys.KEY, lease=1): raise ValueError() with pytest.raises(ValueError): @@ -84,73 +99,97 @@ async def dummy(): class TestMemoryRedLock: @pytest.fixture def lock(self, memory_cache): - return RedLock(memory_cache, pytest.KEY, 20) + return RedLock(memory_cache, Keys.KEY, 20) + + async def test_acquire_key_builder(self, custom_memory_cache, lock): + async with lock: + assert await custom_memory_cache.get(KEY_LOCK) == lock._value + + async def test_acquire_release_key_builder(self, custom_memory_cache, lock): + async with lock: + assert await custom_memory_cache.get(KEY_LOCK) is not None + assert await custom_memory_cache.get(KEY_LOCK) is None - @pytest.mark.asyncio async def test_release_wrong_token_fails(self, lock): await lock.__aenter__() lock._value = "random" assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_release_wrong_client_fails(self, memory_cache, lock): - wrong_lock = RedLock(memory_cache, pytest.KEY, 20) + wrong_lock = RedLock(memory_cache, Keys.KEY, 20) await lock.__aenter__() assert await wrong_lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_float_lease(self, memory_cache): - lock = RedLock(memory_cache, pytest.KEY, 0.1) + lock = RedLock(memory_cache, Keys.KEY, 0.1) await lock.__aenter__() await asyncio.sleep(0.2) assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None +@pytest.mark.redis class TestRedisRedLock: @pytest.fixture def lock(self, redis_cache): - return RedLock(redis_cache, pytest.KEY, 20) + return RedLock(redis_cache, Keys.KEY, 20) + + async def test_acquire_key_builder(self, custom_redis_cache, lock): + custom_redis_cache.serializer = StringSerializer() + async with lock: + assert await custom_redis_cache.get(KEY_LOCK) == lock._value + + async def test_acquire_release_key_builder(self, custom_redis_cache, lock): + custom_redis_cache.serializer = StringSerializer() + async with lock: + assert await custom_redis_cache.get(KEY_LOCK) is not None + assert await custom_redis_cache.get(KEY_LOCK) is None - @pytest.mark.asyncio async def test_release_wrong_token_fails(self, lock): await lock.__aenter__() lock._value = "random" assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_release_wrong_client_fails(self, redis_cache, lock): - wrong_lock = RedLock(redis_cache, pytest.KEY, 20) + wrong_lock = RedLock(redis_cache, Keys.KEY, 20) await lock.__aenter__() assert await wrong_lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_float_lease(self, redis_cache): - lock = RedLock(redis_cache, pytest.KEY, 0.1) + lock = RedLock(redis_cache, Keys.KEY, 0.1) await lock.__aenter__() await asyncio.sleep(0.2) assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None +@pytest.mark.memcached class TestMemcachedRedLock: @pytest.fixture def lock(self, memcached_cache): - return RedLock(memcached_cache, pytest.KEY, 20) + return RedLock(memcached_cache, Keys.KEY, 20) + + async def test_acquire_key_builder(self, custom_memcached_cache, lock): + custom_memcached_cache.serializer = StringSerializer() + async with lock: + assert await custom_memcached_cache.get(KEY_LOCK) == lock._value + + async def test_acquire_release_key_builder(self, custom_memcached_cache, lock): + custom_memcached_cache.serializer = StringSerializer() + async with lock: + assert await custom_memcached_cache.get(KEY_LOCK) is not None + assert await custom_memcached_cache.get(KEY_LOCK) is None - @pytest.mark.asyncio async def test_release_wrong_token_succeeds_meh(self, lock): await lock.__aenter__() lock._value = "random" assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_release_wrong_client_succeeds_meh(self, memcached_cache, lock): - wrong_lock = RedLock(memcached_cache, pytest.KEY, 20) + wrong_lock = RedLock(memcached_cache, Keys.KEY, 20) await lock.__aenter__() assert await wrong_lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_float_lease(self, memcached_cache): - lock = RedLock(memcached_cache, pytest.KEY, 0.1) + lock = RedLock(memcached_cache, Keys.KEY, 0.1) with pytest.raises(TypeError): await lock.__aenter__() @@ -158,77 +197,83 @@ async def test_float_lease(self, memcached_cache): class TestOptimisticLock: @pytest.fixture def lock(self, cache): - return OptimisticLock(cache, pytest.KEY) + return OptimisticLock(cache, Keys.KEY) - @pytest.mark.asyncio async def test_acquire(self, cache, lock): - await cache.set(pytest.KEY, "value") + await cache.set(Keys.KEY, "value") async with lock: - assert lock._token == await cache._gets(cache._build_key(pytest.KEY)) + assert lock._token == await cache._gets(cache.build_key(Keys.KEY)) - @pytest.mark.asyncio async def test_release_does_nothing(self, lock): assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None - @pytest.mark.asyncio async def test_check_and_set_not_existing_never_fails(self, cache, lock): async with lock as locked: - await cache.set(pytest.KEY, "conflicting_value") + await cache.set(Keys.KEY, "conflicting_value") await locked.cas("value") - assert await cache.get(pytest.KEY) == "value" + assert await cache.get(Keys.KEY) == "value" - @pytest.mark.asyncio async def test_check_and_set(self, cache, lock): - await cache.set(pytest.KEY, "previous_value") + await cache.set(Keys.KEY, "previous_value") async with lock as locked: await locked.cas("value") - assert await cache.get(pytest.KEY) == "value" + assert await cache.get(Keys.KEY) == "value" - @pytest.mark.asyncio async def test_check_and_set_fail(self, cache, lock): - await cache.set(pytest.KEY, "previous_value") + await cache.set(Keys.KEY, "previous_value") with pytest.raises(OptimisticLockError): async with lock as locked: - await cache.set(pytest.KEY, "conflicting_value") + await cache.set(Keys.KEY, "conflicting_value") await locked.cas("value") - @pytest.mark.asyncio async def test_check_and_set_with_int_ttl(self, cache, lock): - await cache.set(pytest.KEY, "previous_value") + await cache.set(Keys.KEY, "previous_value") async with lock as locked: await locked.cas("value", ttl=1) await asyncio.sleep(1) - assert await cache.get(pytest.KEY) is None + assert await cache.get(Keys.KEY) is None class TestMemoryOptimisticLock: @pytest.fixture def lock(self, memory_cache): - return OptimisticLock(memory_cache, pytest.KEY) + return OptimisticLock(memory_cache, Keys.KEY) + + async def test_acquire_key_builder(self, custom_memory_cache, lock): + await custom_memory_cache.set(Keys.KEY, "value") + async with lock: + assert await custom_memory_cache.get(KEY_LOCK) == lock._token + await custom_memory_cache.delete(Keys.KEY, "value") - @pytest.mark.asyncio async def test_check_and_set_with_float_ttl(self, memory_cache, lock): - await memory_cache.set(pytest.KEY, "previous_value") + await memory_cache.set(Keys.KEY, "previous_value") async with lock as locked: await locked.cas("value", ttl=0.1) await asyncio.sleep(1) - assert await memory_cache.get(pytest.KEY) is None + assert await memory_cache.get(Keys.KEY) is None +@pytest.mark.redis class TestRedisOptimisticLock: @pytest.fixture def lock(self, redis_cache): - return OptimisticLock(redis_cache, pytest.KEY) + return OptimisticLock(redis_cache, Keys.KEY) + + async def test_acquire_key_builder(self, custom_redis_cache, lock): + custom_redis_cache.serializer = StringSerializer() + await custom_redis_cache.set(Keys.KEY, "value") + async with lock: + assert await custom_redis_cache.get(KEY_LOCK) == lock._token + await custom_redis_cache.delete(Keys.KEY, "value") - @pytest.mark.asyncio async def test_check_and_set_with_float_ttl(self, redis_cache, lock): - await redis_cache.set(pytest.KEY, "previous_value") + await redis_cache.set(Keys.KEY, "previous_value") async with lock as locked: await locked.cas("value", ttl=0.1) await asyncio.sleep(1) - assert await redis_cache.get(pytest.KEY) is None + assert await redis_cache.get(Keys.KEY) is None diff --git a/tests/acceptance/test_plugins.py b/tests/acceptance/test_plugins.py index 3ebb4ffd1..ec9743146 100644 --- a/tests/acceptance/test_plugins.py +++ b/tests/acceptance/test_plugins.py @@ -1,11 +1,9 @@ import pytest -from aiocache.backends.memory import SimpleMemoryBackend from aiocache.plugins import HitMissRatioPlugin, TimingPlugin class TestHitMissRatioPlugin: - @pytest.mark.asyncio @pytest.mark.parametrize( "data, ratio", [ @@ -18,7 +16,7 @@ class TestHitMissRatioPlugin: async def test_get_hit_miss_ratio(self, memory_cache, data, ratio): keys = ["a", "b", "c", "d", "e", "f"] memory_cache.plugins = [HitMissRatioPlugin()] - SimpleMemoryBackend._cache = data + memory_cache._cache = data for key in keys: await memory_cache.get(key) @@ -30,7 +28,6 @@ async def test_get_hit_miss_ratio(self, memory_cache, data, ratio): == len(hits) / memory_cache.hit_miss_ratio["total"] ) - @pytest.mark.asyncio @pytest.mark.parametrize( "data, ratio", [ @@ -43,7 +40,7 @@ async def test_get_hit_miss_ratio(self, memory_cache, data, ratio): async def test_multi_get_hit_miss_ratio(self, memory_cache, data, ratio): keys = ["a", "b", "c", "d", "e", "f"] memory_cache.plugins = [HitMissRatioPlugin()] - SimpleMemoryBackend._cache = data + memory_cache._cache = data for key in keys: await memory_cache.multi_get([key]) @@ -55,7 +52,6 @@ async def test_multi_get_hit_miss_ratio(self, memory_cache, data, ratio): == len(hits) / memory_cache.hit_miss_ratio["total"] ) - @pytest.mark.asyncio async def test_set_and_get_using_namespace(self, memory_cache): memory_cache.plugins = [HitMissRatioPlugin()] key = "A" @@ -67,7 +63,6 @@ async def test_set_and_get_using_namespace(self, memory_cache): class TestTimingPlugin: - @pytest.mark.asyncio @pytest.mark.parametrize( "data, ratio", [ @@ -80,7 +75,7 @@ class TestTimingPlugin: async def test_get_avg_min_max(self, memory_cache, data, ratio): keys = ["a", "b", "c", "d", "e", "f"] memory_cache.plugins = [TimingPlugin()] - SimpleMemoryBackend._cache = data + memory_cache._cache = data for key in keys: await memory_cache.get(key) diff --git a/tests/acceptance/test_serializers.py b/tests/acceptance/test_serializers.py index 4cbb19c7b..694f0a8b6 100644 --- a/tests/acceptance/test_serializers.py +++ b/tests/acceptance/test_serializers.py @@ -1,24 +1,23 @@ -import pytest +import pickle import random +from typing import Any + +import pytest +from marshmallow import Schema, fields, post_load try: - import ujson as json -except ImportError: - import json -try: - import cPickle as pickle + import ujson as json # noqa: I900 except ImportError: - import pickle - -from marshmallow import fields, Schema, post_load + import json # type: ignore[no-redef] from aiocache.serializers import ( BaseSerializer, + JsonSerializer, NullSerializer, - StringSerializer, PickleSerializer, - JsonSerializer, + StringSerializer, ) +from ..utils import Keys class MyType: @@ -31,15 +30,8 @@ def __eq__(self, obj): return self.__dict__ == obj.__dict__ -class MyTypeSchema(Schema, BaseSerializer): +class MySchema(Schema): r = fields.Integer() - encoding = "utf-8" - - def dumps(self, *args, **kwargs): - return super().dumps(*args, **kwargs) - - def loads(self, *args, **kwargs): - return super().loads(*args, **kwargs) @post_load def build_my_type(self, data, **kwargs): @@ -49,134 +41,116 @@ class Meta: strict = True -def dumps(x): - if x == "value": - return "v4lu3" - return 100 +class MyTypeSchema(BaseSerializer): + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.schema = MySchema() + def dumps(self, value: Any) -> str: + return self.schema.dumps(value) -def loads(x): - if x == "v4lu3": - return "value" - return 200 + def loads(self, value: str) -> Any: + return self.schema.loads(value) class TestNullSerializer: - - TYPES = [1, 2.0, "hi", True, ["1", 1], {"key": "value"}, MyType()] + TYPES = (1, 2.0, "hi", True, ["1", 1], {"key": "value"}, MyType()) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_set_get_types(self, memory_cache, obj): memory_cache.serializer = NullSerializer() - assert await memory_cache.set(pytest.KEY, obj) is True - assert await memory_cache.get(pytest.KEY) is obj + assert await memory_cache.set(Keys.KEY, obj) is True + assert await memory_cache.get(Keys.KEY) is obj @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_add_get_types(self, memory_cache, obj): memory_cache.serializer = NullSerializer() - assert await memory_cache.add(pytest.KEY, obj) is True - assert await memory_cache.get(pytest.KEY) is obj + assert await memory_cache.add(Keys.KEY, obj) is True + assert await memory_cache.get(Keys.KEY) is obj @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_multi_set_multi_get_types(self, memory_cache, obj): memory_cache.serializer = NullSerializer() - assert await memory_cache.multi_set([(pytest.KEY, obj)]) is True - assert (await memory_cache.multi_get([pytest.KEY]))[0] is obj + assert await memory_cache.multi_set([(Keys.KEY, obj)]) is True + assert (await memory_cache.multi_get([Keys.KEY]))[0] is obj class TestStringSerializer: - - TYPES = [1, 2.0, "hi", True, ["1", 1], {"key": "value"}, MyType()] + TYPES = (1, 2.0, "hi", True, ["1", 1], {"key": "value"}, MyType()) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_set_get_types(self, cache, obj): cache.serializer = StringSerializer() - assert await cache.set(pytest.KEY, obj) is True - assert await cache.get(pytest.KEY) == str(obj) + assert await cache.set(Keys.KEY, obj) is True + assert await cache.get(Keys.KEY) == str(obj) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_add_get_types(self, cache, obj): cache.serializer = StringSerializer() - assert await cache.add(pytest.KEY, obj) is True - assert await cache.get(pytest.KEY) == str(obj) + assert await cache.add(Keys.KEY, obj) is True + assert await cache.get(Keys.KEY) == str(obj) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_multi_set_multi_get_types(self, cache, obj): cache.serializer = StringSerializer() - assert await cache.multi_set([(pytest.KEY, obj)]) is True - assert await cache.multi_get([pytest.KEY]) == [str(obj)] + assert await cache.multi_set([(Keys.KEY, obj)]) is True + assert await cache.multi_get([Keys.KEY]) == [str(obj)] class TestJsonSerializer: - - TYPES = [1, 2.0, "hi", True, ["1", 1], {"key": "value"}] + TYPES = (1, 2.0, "hi", True, ["1", 1], {"key": "value"}) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_set_get_types(self, cache, obj): cache.serializer = JsonSerializer() - assert await cache.set(pytest.KEY, obj) is True - assert await cache.get(pytest.KEY) == json.loads(json.dumps(obj)) + assert await cache.set(Keys.KEY, obj) is True + assert await cache.get(Keys.KEY) == json.loads(json.dumps(obj)) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_add_get_types(self, cache, obj): cache.serializer = JsonSerializer() - assert await cache.add(pytest.KEY, obj) is True - assert await cache.get(pytest.KEY) == json.loads(json.dumps(obj)) + assert await cache.add(Keys.KEY, obj) is True + assert await cache.get(Keys.KEY) == json.loads(json.dumps(obj)) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_multi_set_multi_get_types(self, cache, obj): cache.serializer = JsonSerializer() - assert await cache.multi_set([(pytest.KEY, obj)]) is True - assert await cache.multi_get([pytest.KEY]) == [json.loads(json.dumps(obj))] + assert await cache.multi_set([(Keys.KEY, obj)]) is True + assert await cache.multi_get([Keys.KEY]) == [json.loads(json.dumps(obj))] class TestPickleSerializer: - - TYPES = [1, 2.0, "hi", True, ["1", 1], {"key": "value"}, MyType()] + TYPES = (1, 2.0, "hi", True, ["1", 1], {"key": "value"}, MyType()) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_set_get_types(self, cache, obj): cache.serializer = PickleSerializer() - assert await cache.set(pytest.KEY, obj) is True - assert await cache.get(pytest.KEY) == pickle.loads(pickle.dumps(obj)) + assert await cache.set(Keys.KEY, obj) is True + assert await cache.get(Keys.KEY) == pickle.loads(pickle.dumps(obj)) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_add_get_types(self, cache, obj): cache.serializer = PickleSerializer() - assert await cache.add(pytest.KEY, obj) is True - assert await cache.get(pytest.KEY) == pickle.loads(pickle.dumps(obj)) + assert await cache.add(Keys.KEY, obj) is True + assert await cache.get(Keys.KEY) == pickle.loads(pickle.dumps(obj)) @pytest.mark.parametrize("obj", TYPES) - @pytest.mark.asyncio async def test_multi_set_multi_get_types(self, cache, obj): cache.serializer = PickleSerializer() - assert await cache.multi_set([(pytest.KEY, obj)]) is True - assert await cache.multi_get([pytest.KEY]) == [pickle.loads(pickle.dumps(obj))] + assert await cache.multi_set([(Keys.KEY, obj)]) is True + assert await cache.multi_get([Keys.KEY]) == [pickle.loads(pickle.dumps(obj))] class TestAltSerializers: - @pytest.mark.asyncio async def test_get_set_alt_serializer_functions(self, cache): cache.serializer = StringSerializer() - await cache.set(pytest.KEY, "value", dumps_fn=dumps) - assert await cache.get(pytest.KEY) == "v4lu3" - assert await cache.get(pytest.KEY, loads_fn=loads) == "value" + await cache.set(Keys.KEY, "value", dumps_fn=lambda _: "v4lu3") + assert await cache.get(Keys.KEY) == "v4lu3" + assert await cache.get(Keys.KEY, loads_fn=lambda _: "value") == "value" - @pytest.mark.asyncio async def test_get_set_alt_serializer_class(self, cache): my_serializer = MyTypeSchema() my_obj = MyType() cache.serializer = my_serializer - await cache.set(pytest.KEY, my_obj) - assert await cache.get(pytest.KEY) == my_serializer.loads(my_serializer.dumps(my_obj)) + await cache.set(Keys.KEY, my_obj) + assert await cache.get(Keys.KEY) == my_serializer.loads(my_serializer.dumps(my_obj)) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..4482701d1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +import pytest + + +@pytest.fixture() +def max_conns(): + return None + + +@pytest.fixture() +def decode_responses(): + return False + + +@pytest.fixture +async def redis_client(max_conns, decode_responses): + import redis.asyncio as redis + + async with redis.Redis( + host="127.0.0.1", + port=6379, + db=0, + password=None, + decode_responses=decode_responses, + socket_connect_timeout=None, + max_connections=max_conns + ) as r: + yield r diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/performance/conftest.py b/tests/performance/conftest.py index 1c4a610ae..c7a1ccd2d 100644 --- a/tests/performance/conftest.py +++ b/tests/performance/conftest.py @@ -1,20 +1,18 @@ import pytest from aiocache import Cache -from aiocache.backends.redis import RedisBackend @pytest.fixture -def redis_cache(event_loop): - cache = Cache(Cache.REDIS, namespace="test", pool_max_size=1) - yield cache - - for _, pool in RedisBackend.pools.items(): - pool.close() - event_loop.run_until_complete(pool.wait_closed()) +@pytest.mark.parametrize("max_conns", 1) +async def redis_cache(redis_client): + # redis connection pool raises ConnectionError but doesn't wait for conn reuse + # when exceeding max pool size. + async with Cache(Cache.REDIS, namespace="test", client=redis_client) as cache: + yield cache @pytest.fixture -def memcached_cache(): - cache = Cache(Cache.MEMCACHED, namespace="test", pool_size=1) - yield cache +async def memcached_cache(): + async with Cache(Cache.MEMCACHED, namespace="test", pool_size=1) as cache: + yield cache diff --git a/tests/performance/server.py b/tests/performance/server.py index 55b1a11b8..679fdd232 100644 --- a/tests/performance/server.py +++ b/tests/performance/server.py @@ -1,25 +1,33 @@ import asyncio -import argparse import logging import uuid -from aiocache import Cache +import redis.asyncio as redis from aiohttp import web +from aiocache import Cache logging.getLogger("aiohttp.access").propagate = False -AIOCACHE_BACKENDS = { - "memory": Cache(Cache.MEMORY), - "redis": Cache(Cache.REDIS), - "memcached": Cache(Cache.MEMCACHED), -} - - class CacheManager: - def __init__(self, backend): - self.cache = AIOCACHE_BACKENDS.get(backend) + def __init__(self, backend: str): + backends = { + "memory": Cache.MEMORY, + "redis": Cache.REDIS, + "memcached": Cache.MEMCACHED, + } + if backend == "redis": + cache_kwargs = {"client": redis.Redis( + host="127.0.0.1", + port=6379, + db=0, + password=None, + decode_responses=False, + )} + else: + cache_kwargs = dict() + self.cache = Cache(backends[backend], **cache_kwargs) async def get(self, key): return await self.cache.get(key, timeout=0.1) @@ -27,6 +35,9 @@ async def get(self, key): async def set(self, key, value): return await self.cache.set(key, value, timeout=0.1) + async def close(self, *_): + await self.cache.close() + async def handler_get(req): try: @@ -41,19 +52,9 @@ async def handler_get(req): return web.Response(text=str(data)) -def run_server(backend, loop=None): - if loop: - asyncio.set_event_loop(loop) +def run_server(backend: str) -> None: app = web.Application() app["cache"] = CacheManager(backend) + app.on_shutdown.append(app["cache"].close) app.router.add_route("GET", "/", handler_get) web.run_app(app) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-b", dest="backend", required=True, choices=["memory", "redis", "memcached"] - ) - args = parser.parse_args() - run_server(args.backend) diff --git a/tests/performance/test_concurrency.py b/tests/performance/test_concurrency.py index 59f83e9a3..5112e0b9c 100644 --- a/tests/performance/test_concurrency.py +++ b/tests/performance/test_concurrency.py @@ -1,65 +1,41 @@ +import platform import re -import time -import pytest import subprocess - +import time from multiprocessing import Process -from server import run_server - - -@pytest.fixture -def redis_server(): - p = Process(target=run_server, args=["redis"]) - p.start() - yield - p.terminate() - time.sleep(2) - +import pytest -@pytest.fixture -def memcached_server(): - p = Process(target=run_server, args=["memcached"]) - p.start() - yield - p.terminate() - time.sleep(2) +from .server import run_server -@pytest.fixture -def memory_server(): - p = Process(target=run_server, args=["memory"]) +# TODO: Fix and readd "memcached" (currently fails >98% of requests) +@pytest.fixture(params=("memory", "redis")) +def server(request): + p = Process(target=run_server, args=(request.param,)) p.start() + time.sleep(1) yield p.terminate() - time.sleep(2) - - -@pytest.fixture(params=["memcached_server", "memory_server", "redis_server"]) -def server(request): - return request.getfuncargvalue(request.param) + p.join(timeout=15) +@pytest.mark.skipif(platform.python_implementation() == "PyPy", reason="Not working currently.") def test_concurrency_error_rates(server): + """Test with Apache benchmark tool.""" + total_requests = 1500 - result = subprocess.run( - ["ab", "-n", str(total_requests), "-c", "500", "http://127.0.0.1:8080/"], - stdout=subprocess.PIPE, - ) + # On some platforms, it's required to enlarge number of "open file descriptors" + # with "ulimit -n number" before doing the benchmark. + cmd = ("ab", "-n", str(total_requests), "-c", "500", "http://127.0.0.1:8080/") + result = subprocess.run(cmd, capture_output=True, check=True, encoding="utf-8") - failed_requests = total_requests - m = re.search(r"Failed requests:\s+([0-9]+)", str(result.stdout)) - if m: - failed_requests = int(m.group(1)) + m = re.search(r"Failed requests:\s+([0-9]+)", result.stdout) + assert m, "Missing output from ab: " + result.stdout + failed_requests = int(m.group(1)) - non_200 = 0 - m = re.search(r"Non-2xx responses:\s+([0-9]+)", str(result.stdout)) - if m: - non_200 = int(m.group(1)) + m = re.search(r"Non-2xx responses:\s+([0-9]+)", result.stdout) + non_200 = int(m.group(1)) if m else 0 - print("Failed requests: {}%".format(failed_requests / total_requests * 100)) - print("Non 200 requests: {}%".format(non_200 / total_requests * 100)) - assert ( - failed_requests / total_requests < 0.75 - ) # aioredis is the problem here, need to improve it - assert non_200 / total_requests < 0.75 + assert failed_requests / total_requests < 0.75, result.stdout + assert non_200 / total_requests < 0.75, result.stdout diff --git a/tests/performance/test_footprint.py b/tests/performance/test_footprint.py index ed4950964..9595759f5 100644 --- a/tests/performance/test_footprint.py +++ b/tests/performance/test_footprint.py @@ -1,32 +1,33 @@ -import pytest +import platform import time +from typing import AsyncIterator, cast -import aioredis import aiomcache +import pytest +import redis.asyncio as redis @pytest.fixture -def aioredis_pool(event_loop): - return event_loop.run_until_complete(aioredis.create_pool(("127.0.0.1", 6379), maxsize=1)) +async def redis_client() -> AsyncIterator["redis.Redis[str]"]: + async with cast("redis.Redis[str]", + redis.Redis(host="127.0.0.1", port=6379, max_connections=1)) as r: + yield r +@pytest.mark.skipif(platform.python_implementation() == "PyPy", reason="Too slow") class TestRedis: - @pytest.mark.asyncio - async def test_redis_getsetdel(self, aioredis_pool, redis_cache): + async def test_redis_getsetdel(self, redis_client, redis_cache): N = 10000 - aioredis_total_time = 0 - for n in range(N): + redis_total_time = 0 + for _n in range(N): start = time.time() - with await aioredis_pool as redis: - await redis.set("hi", "value") - with await aioredis_pool as redis: - await redis.get("hi") - with await aioredis_pool as redis: - await redis.delete("hi") - aioredis_total_time += time.time() - start + await redis_client.set("hi", "value") + await redis_client.get("hi") + await redis_client.delete("hi") + redis_total_time += time.time() - start aiocache_total_time = 0 - for n in range(N): + for _n in range(N): start = time.time() await redis_cache.set("hi", "value", timeout=0) await redis_cache.get("hi", timeout=0) @@ -35,31 +36,27 @@ async def test_redis_getsetdel(self, aioredis_pool, redis_cache): print( "\n{:0.2f}/{:0.2f}: {:0.2f}".format( - aiocache_total_time, aioredis_total_time, aiocache_total_time / aioredis_total_time + aiocache_total_time, redis_total_time, aiocache_total_time / redis_total_time ) ) print("aiocache avg call: {:0.5f}s".format(aiocache_total_time / N)) - print("aioredis avg call: {:0.5f}s".format(aioredis_total_time / N)) - assert aiocache_total_time / aioredis_total_time < 1.30 + print("redis avg call: {:0.5f}s".format(redis_total_time / N)) + assert aiocache_total_time / redis_total_time < 1.35 - @pytest.mark.asyncio - async def test_redis_multigetsetdel(self, aioredis_pool, redis_cache): + async def test_redis_multigetsetdel(self, redis_client, redis_cache): N = 5000 - aioredis_total_time = 0 + redis_total_time = 0 values = ["a", "b", "c", "d", "e", "f"] - for n in range(N): + for _n in range(N): start = time.time() - with await aioredis_pool as redis: - await redis.mset(*[x for x in values * 2]) - with await aioredis_pool as redis: - await redis.mget(*values) + await redis_client.mset({x: x for x in values}) + await redis_client.mget(values) for k in values: - with await aioredis_pool as redis: - await redis.delete(k) - aioredis_total_time += time.time() - start + await redis_client.delete(k) + redis_total_time += time.time() - start aiocache_total_time = 0 - for n in range(N): + for _n in range(N): start = time.time() await redis_cache.multi_set([(x, x) for x in values], timeout=0) await redis_cache.multi_get(values, timeout=0) @@ -69,25 +66,27 @@ async def test_redis_multigetsetdel(self, aioredis_pool, redis_cache): print( "\n{:0.2f}/{:0.2f}: {:0.2f}".format( - aiocache_total_time, aioredis_total_time, aiocache_total_time / aioredis_total_time + aiocache_total_time, redis_total_time, aiocache_total_time / redis_total_time ) ) print("aiocache avg call: {:0.5f}s".format(aiocache_total_time / N)) - print("aioredis avg call: {:0.5f}s".format(aioredis_total_time / N)) - assert aiocache_total_time / aioredis_total_time < 1.35 + print("redis_client avg call: {:0.5f}s".format(redis_total_time / N)) + assert aiocache_total_time / redis_total_time < 1.35 @pytest.fixture -def aiomcache_pool(): - yield aiomcache.Client("127.0.0.1", 11211, pool_size=1) +async def aiomcache_pool(): + client = aiomcache.Client("127.0.0.1", 11211, pool_size=1) + yield client + await client.close() +@pytest.mark.skipif(platform.python_implementation() == "PyPy", reason="Too slow") class TestMemcached: - @pytest.mark.asyncio async def test_memcached_getsetdel(self, aiomcache_pool, memcached_cache): N = 10000 aiomcache_total_time = 0 - for n in range(N): + for _n in range(N): start = time.time() await aiomcache_pool.set(b"hi", b"value") await aiomcache_pool.get(b"hi") @@ -95,7 +94,7 @@ async def test_memcached_getsetdel(self, aiomcache_pool, memcached_cache): aiomcache_total_time += time.time() - start aiocache_total_time = 0 - for n in range(N): + for _n in range(N): start = time.time() await memcached_cache.set("hi", "value", timeout=0) await memcached_cache.get("hi", timeout=0) @@ -111,14 +110,13 @@ async def test_memcached_getsetdel(self, aiomcache_pool, memcached_cache): ) print("aiocache avg call: {:0.5f}s".format(aiocache_total_time / N)) print("aiomcache avg call: {:0.5f}s".format(aiomcache_total_time / N)) - assert aiocache_total_time / aiomcache_total_time < 1.30 + assert aiocache_total_time / aiomcache_total_time < 1.40 - @pytest.mark.asyncio async def test_memcached_multigetsetdel(self, aiomcache_pool, memcached_cache): N = 2000 aiomcache_total_time = 0 values = [b"a", b"b", b"c", b"d", b"e", b"f"] - for n in range(N): + for _n in range(N): start = time.time() for k in values: await aiomcache_pool.set(k, k) @@ -128,8 +126,8 @@ async def test_memcached_multigetsetdel(self, aiomcache_pool, memcached_cache): aiomcache_total_time += time.time() - start aiocache_total_time = 0 - values = [b"a", b"b", b"c", b"d", b"e", b"f"] - for n in range(N): + values = ["a", "b", "c", "d", "e", "f"] + for _n in range(N): start = time.time() await memcached_cache.multi_set([(x, x) for x in values], timeout=0) await memcached_cache.multi_get(values, timeout=0) @@ -146,4 +144,4 @@ async def test_memcached_multigetsetdel(self, aiomcache_pool, memcached_cache): ) print("aiocache avg call: {:0.5f}s".format(aiocache_total_time / N)) print("aiomcache avg call: {:0.5f}s".format(aiomcache_total_time / N)) - assert aiocache_total_time / aiomcache_total_time < 1.90 + assert aiocache_total_time / aiomcache_total_time < 1.40 diff --git a/tests/ut/backends/__init__.py b/tests/ut/backends/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/ut/backends/test_memcached.py b/tests/ut/backends/test_memcached.py index d9d42c5a9..f0de04cb4 100644 --- a/tests/ut/backends/test_memcached.py +++ b/tests/ut/backends/test_memcached.py @@ -1,19 +1,28 @@ -import pytest -import aiomcache +from unittest.mock import AsyncMock, patch -from asynctest import MagicMock, patch, ANY +import aiomcache +import pytest -from aiocache import MemcachedCache +from aiocache.backends.memcached import MemcachedBackend, MemcachedCache from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer -from aiocache.backends.memcached import MemcachedBackend +from ...utils import Keys, ensure_key @pytest.fixture def memcached(): memcached = MemcachedBackend() - memcached.client = MagicMock(spec=aiomcache.Client) - yield memcached + with patch.object(memcached, "client", autospec=True) as m: + # Autospec messes up the signature on the decorated methods. + for method in ( + "get", "gets", "multi_get", "stats", "set", "cas", "replace", + "append", "prepend", "incr", "decr", "touch", "version", "flush_all" + ): + setattr(m, method, AsyncMock(return_value=None, spec_set=())) + m.add = AsyncMock(return_value=True, spec_set=()) + m.delete = AsyncMock(return_value=True, spec_set=()) + + yield memcached class TestMemcachedBackend: @@ -21,19 +30,19 @@ def test_setup(self): with patch.object(aiomcache, "Client", autospec=True) as aiomcache_client: memcached = MemcachedBackend() - aiomcache_client.assert_called_with("127.0.0.1", 11211, loop=ANY, pool_size=2) + aiomcache_client.assert_called_with("127.0.0.1", 11211, pool_size=2) - assert memcached.endpoint == "127.0.0.1" + assert memcached.host == "127.0.0.1" assert memcached.port == 11211 assert memcached.pool_size == 2 def test_setup_override(self): with patch.object(aiomcache, "Client", autospec=True) as aiomcache_client: - memcached = MemcachedBackend(endpoint="127.0.0.2", port=2, pool_size=10) + memcached = MemcachedBackend(host="127.0.0.2", port=2, pool_size=10) - aiomcache_client.assert_called_with("127.0.0.2", 2, loop=ANY, pool_size=10) + aiomcache_client.assert_called_with("127.0.0.2", 2, pool_size=10) - assert memcached.endpoint == "127.0.0.2" + assert memcached.host == "127.0.0.2" assert memcached.port == 2 assert memcached.pool_size == 10 @@ -41,211 +50,179 @@ def test_setup_casts(self): with patch.object(aiomcache, "Client", autospec=True) as aiomcache_client: memcached = MemcachedBackend(pool_size="10") - aiomcache_client.assert_called_with("127.0.0.1", 11211, loop=ANY, pool_size=10) + aiomcache_client.assert_called_with("127.0.0.1", 11211, pool_size=10) assert memcached.pool_size == 10 - @pytest.mark.asyncio async def test_get(self, memcached): memcached.client.get.return_value = b"value" - assert await memcached._get(pytest.KEY) == "value" - memcached.client.get.assert_called_with(pytest.KEY) + assert await memcached._get(Keys.KEY) == "value" + memcached.client.get.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_gets(self, memcached): memcached.client.gets.return_value = b"value", 12345 - assert await memcached._gets(pytest.KEY) == 12345 - memcached.client.gets.assert_called_with(pytest.KEY.encode()) + assert await memcached._gets(Keys.KEY) == 12345 + memcached.client.gets.assert_called_with(Keys.KEY.encode()) - @pytest.mark.asyncio async def test_get_none(self, memcached): memcached.client.get.return_value = None - assert await memcached._get(pytest.KEY) is None - memcached.client.get.assert_called_with(pytest.KEY) + assert await memcached._get(Keys.KEY) is None + memcached.client.get.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_get_no_encoding(self, memcached): memcached.client.get.return_value = b"value" - assert await memcached._get(pytest.KEY, encoding=None) == b"value" - memcached.client.get.assert_called_with(pytest.KEY) + assert await memcached._get(Keys.KEY, encoding=None) == b"value" + memcached.client.get.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_set(self, memcached): - await memcached._set(pytest.KEY, "value") - memcached.client.set.assert_called_with(pytest.KEY, b"value", exptime=0) + await memcached._set(Keys.KEY, "value") + memcached.client.set.assert_called_with(Keys.KEY, b"value", exptime=0) - await memcached._set(pytest.KEY, "value", ttl=1) - memcached.client.set.assert_called_with(pytest.KEY, b"value", exptime=1) + await memcached._set(Keys.KEY, "value", ttl=1) + memcached.client.set.assert_called_with(Keys.KEY, b"value", exptime=1) - @pytest.mark.asyncio async def test_set_float_ttl(self, memcached): memcached.client.set.side_effect = aiomcache.exceptions.ValidationException("msg") with pytest.raises(TypeError) as exc_info: - await memcached._set(pytest.KEY, "value", ttl=0.1) + await memcached._set(Keys.KEY, "value", ttl=0.1) assert str(exc_info.value) == "aiomcache error: msg" - @pytest.mark.asyncio async def test_set_cas_token(self, mocker, memcached): mocker.spy(memcached, "_cas") - await memcached._set(pytest.KEY, "value", _cas_token="token") - memcached._cas.assert_called_with(pytest.KEY, b"value", "token", ttl=0, _conn=None) + await memcached._set(Keys.KEY, "value", _cas_token="token") + memcached._cas.assert_called_with(Keys.KEY, b"value", "token", ttl=0, _conn=None) - @pytest.mark.asyncio - async def test_cas(self, mocker, memcached): + async def test_cas(self, memcached): memcached.client.cas.return_value = True - assert await memcached._cas(pytest.KEY, b"value", "token", ttl=0) is True - memcached.client.cas.assert_called_with(pytest.KEY, b"value", "token", exptime=0) + assert await memcached._cas(Keys.KEY, b"value", "token", ttl=0) is True + memcached.client.cas.assert_called_with(Keys.KEY, b"value", "token", exptime=0) - @pytest.mark.asyncio - async def test_cas_fail(self, mocker, memcached): + async def test_cas_fail(self, memcached): memcached.client.cas.return_value = False - assert await memcached._cas(pytest.KEY, b"value", "token", ttl=0) is False - memcached.client.cas.assert_called_with(pytest.KEY, b"value", "token", exptime=0) + assert await memcached._cas(Keys.KEY, b"value", "token", ttl=0) is False + memcached.client.cas.assert_called_with(Keys.KEY, b"value", "token", exptime=0) - @pytest.mark.asyncio async def test_multi_get(self, memcached): memcached.client.multi_get.return_value = [b"value", b"random"] - assert await memcached._multi_get([pytest.KEY, pytest.KEY_1]) == ["value", "random"] - memcached.client.multi_get.assert_called_with(pytest.KEY, pytest.KEY_1) + assert await memcached._multi_get([Keys.KEY, Keys.KEY_1]) == ["value", "random"] + memcached.client.multi_get.assert_called_with(Keys.KEY, Keys.KEY_1) - @pytest.mark.asyncio async def test_multi_get_none(self, memcached): memcached.client.multi_get.return_value = [b"value", None] - assert await memcached._multi_get([pytest.KEY, pytest.KEY_1]) == ["value", None] - memcached.client.multi_get.assert_called_with(pytest.KEY, pytest.KEY_1) + assert await memcached._multi_get([Keys.KEY, Keys.KEY_1]) == ["value", None] + memcached.client.multi_get.assert_called_with(Keys.KEY, Keys.KEY_1) - @pytest.mark.asyncio async def test_multi_get_no_encoding(self, memcached): memcached.client.multi_get.return_value = [b"value", None] - assert await memcached._multi_get([pytest.KEY, pytest.KEY_1], encoding=None) == [ + assert await memcached._multi_get([Keys.KEY, Keys.KEY_1], encoding=None) == [ b"value", None, ] - memcached.client.multi_get.assert_called_with(pytest.KEY, pytest.KEY_1) + memcached.client.multi_get.assert_called_with(Keys.KEY, Keys.KEY_1) - @pytest.mark.asyncio async def test_multi_set(self, memcached): - await memcached._multi_set([(pytest.KEY, "value"), (pytest.KEY_1, "random")]) - memcached.client.set.assert_any_call(pytest.KEY, b"value", exptime=0) - memcached.client.set.assert_any_call(pytest.KEY_1, b"random", exptime=0) + await memcached._multi_set([(Keys.KEY, "value"), (Keys.KEY_1, "random")]) + memcached.client.set.assert_any_call(Keys.KEY, b"value", exptime=0) + memcached.client.set.assert_any_call(Keys.KEY_1, b"random", exptime=0) assert memcached.client.set.call_count == 2 - await memcached._multi_set([(pytest.KEY, "value"), (pytest.KEY_1, "random")], ttl=1) - memcached.client.set.assert_any_call(pytest.KEY, b"value", exptime=1) - memcached.client.set.assert_any_call(pytest.KEY_1, b"random", exptime=1) + await memcached._multi_set([(Keys.KEY, "value"), (Keys.KEY_1, "random")], ttl=1) + memcached.client.set.assert_any_call(Keys.KEY, b"value", exptime=1) + memcached.client.set.assert_any_call(Keys.KEY_1, b"random", exptime=1) assert memcached.client.set.call_count == 4 - @pytest.mark.asyncio async def test_multi_set_float_ttl(self, memcached): memcached.client.set.side_effect = aiomcache.exceptions.ValidationException("msg") with pytest.raises(TypeError) as exc_info: - await memcached._multi_set([(pytest.KEY, "value"), (pytest.KEY_1, "random")], ttl=0.1) + await memcached._multi_set([(Keys.KEY, "value"), (Keys.KEY_1, "random")], ttl=0.1) assert str(exc_info.value) == "aiomcache error: msg" - @pytest.mark.asyncio async def test_add(self, memcached): - await memcached._add(pytest.KEY, "value") - memcached.client.add.assert_called_with(pytest.KEY, b"value", exptime=0) + await memcached._add(Keys.KEY, "value") + memcached.client.add.assert_called_with(Keys.KEY, b"value", exptime=0) - await memcached._add(pytest.KEY, "value", ttl=1) - memcached.client.add.assert_called_with(pytest.KEY, b"value", exptime=1) + await memcached._add(Keys.KEY, "value", ttl=1) + memcached.client.add.assert_called_with(Keys.KEY, b"value", exptime=1) - @pytest.mark.asyncio async def test_add_existing(self, memcached): memcached.client.add.return_value = False with pytest.raises(ValueError): - await memcached._add(pytest.KEY, "value") + await memcached._add(Keys.KEY, "value") - @pytest.mark.asyncio async def test_add_float_ttl(self, memcached): memcached.client.add.side_effect = aiomcache.exceptions.ValidationException("msg") with pytest.raises(TypeError) as exc_info: - await memcached._add(pytest.KEY, "value", 0.1) + await memcached._add(Keys.KEY, "value", 0.1) assert str(exc_info.value) == "aiomcache error: msg" - @pytest.mark.asyncio async def test_exists(self, memcached): - await memcached._exists(pytest.KEY) - memcached.client.append.assert_called_with(pytest.KEY, b"") + await memcached._exists(Keys.KEY) + memcached.client.append.assert_called_with(Keys.KEY, b"") - @pytest.mark.asyncio async def test_increment(self, memcached): - await memcached._increment(pytest.KEY, 2) - memcached.client.incr.assert_called_with(pytest.KEY, 2) + await memcached._increment(Keys.KEY, 2) + memcached.client.incr.assert_called_with(Keys.KEY, 2) - @pytest.mark.asyncio async def test_increment_negative(self, memcached): - await memcached._increment(pytest.KEY, -2) - memcached.client.decr.assert_called_with(pytest.KEY, 2) + await memcached._increment(Keys.KEY, -2) + memcached.client.decr.assert_called_with(Keys.KEY, 2) - @pytest.mark.asyncio async def test_increment_missing(self, memcached): memcached.client.incr.side_effect = aiomcache.exceptions.ClientException("NOT_FOUND") - await memcached._increment(pytest.KEY, 2) - memcached.client.incr.assert_called_with(pytest.KEY, 2) - memcached.client.set.assert_called_with(pytest.KEY, b"2", exptime=0) + await memcached._increment(Keys.KEY, 2) + memcached.client.incr.assert_called_with(Keys.KEY, 2) + memcached.client.set.assert_called_with(Keys.KEY, b"2", exptime=0) - @pytest.mark.asyncio async def test_increment_missing_negative(self, memcached): memcached.client.decr.side_effect = aiomcache.exceptions.ClientException("NOT_FOUND") - await memcached._increment(pytest.KEY, -2) - memcached.client.decr.assert_called_with(pytest.KEY, 2) - memcached.client.set.assert_called_with(pytest.KEY, b"-2", exptime=0) + await memcached._increment(Keys.KEY, -2) + memcached.client.decr.assert_called_with(Keys.KEY, 2) + memcached.client.set.assert_called_with(Keys.KEY, b"-2", exptime=0) - @pytest.mark.asyncio async def test_increment_typerror(self, memcached): memcached.client.incr.side_effect = aiomcache.exceptions.ClientException("msg") with pytest.raises(TypeError) as exc_info: - await memcached._increment(pytest.KEY, 2) + await memcached._increment(Keys.KEY, 2) assert str(exc_info.value) == "aiomcache error: msg" - @pytest.mark.asyncio async def test_expire(self, memcached): - await memcached._expire(pytest.KEY, 1) - memcached.client.touch.assert_called_with(pytest.KEY, 1) + await memcached._expire(Keys.KEY, 1) + memcached.client.touch.assert_called_with(Keys.KEY, 1) - @pytest.mark.asyncio async def test_delete(self, memcached): - assert await memcached._delete(pytest.KEY) == 1 - memcached.client.delete.assert_called_with(pytest.KEY) + assert await memcached._delete(Keys.KEY) == 1 + memcached.client.delete.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_delete_missing(self, memcached): memcached.client.delete.return_value = False - assert await memcached._delete(pytest.KEY) == 0 - memcached.client.delete.assert_called_with(pytest.KEY) + assert await memcached._delete(Keys.KEY) == 0 + memcached.client.delete.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_clear(self, memcached): await memcached._clear() memcached.client.flush_all.assert_called_with() - @pytest.mark.asyncio async def test_clear_with_namespace(self, memcached): with pytest.raises(ValueError): await memcached._clear("nm") - @pytest.mark.asyncio async def test_raw(self, memcached): - await memcached._raw("get", pytest.KEY) - await memcached._raw("set", pytest.KEY, 1) - memcached.client.get.assert_called_with(pytest.KEY) - memcached.client.set.assert_called_with(pytest.KEY, 1) + await memcached._raw("get", Keys.KEY) + await memcached._raw("set", Keys.KEY, 1) + memcached.client.get.assert_called_with(Keys.KEY) + memcached.client.set.assert_called_with(Keys.KEY, 1) - @pytest.mark.asyncio async def test_raw_bytes(self, memcached): - await memcached._raw("set", pytest.KEY, "asd") - await memcached._raw("get", pytest.KEY, encoding=None) - memcached.client.get.assert_called_with(pytest.KEY) - memcached.client.set.assert_called_with(pytest.KEY, "asd") + await memcached._raw("set", Keys.KEY, "asd") + await memcached._raw("get", Keys.KEY, encoding=None) + memcached.client.get.assert_called_with(Keys.KEY) + memcached.client.set.assert_called_with(Keys.KEY, "asd") - @pytest.mark.asyncio async def test_redlock_release(self, mocker, memcached): mocker.spy(memcached, "_delete") - await memcached._redlock_release(pytest.KEY, "random") - memcached._delete.assert_called_with(pytest.KEY) + await memcached._redlock_release(Keys.KEY, "random") + memcached._delete.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_close(self, memcached): await memcached._close() assert memcached.client.close.call_count == 1 @@ -272,13 +249,13 @@ def test_parse_uri_path(self): @pytest.mark.parametrize( "namespace, expected", - ([None, "test" + pytest.KEY], ["", pytest.KEY], ["my_ns", "my_ns" + pytest.KEY]), + ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key_bytes(self, set_test_namespace, memcached_cache, namespace, expected): - assert memcached_cache.build_key(pytest.KEY, namespace=namespace) == expected.encode() + assert memcached_cache.build_key(Keys.KEY, namespace) == expected.encode() def test_build_key_no_namespace(self, memcached_cache): - assert memcached_cache.build_key(pytest.KEY, namespace=None) == pytest.KEY.encode() + assert memcached_cache.build_key(Keys.KEY, namespace=None) == Keys.KEY.encode() def test_build_key_no_spaces(self, memcached_cache): assert memcached_cache.build_key("hello world") == b"hello_world" diff --git a/tests/ut/backends/test_memory.py b/tests/ut/backends/test_memory.py index ecc1b8fd6..ea99d5606 100644 --- a/tests/ut/backends/test_memory.py +++ b/tests/ut/backends/test_memory.py @@ -1,216 +1,187 @@ -import pytest import asyncio +from unittest.mock import ANY, MagicMock, create_autospec, patch -from unittest.mock import MagicMock, ANY, patch +import pytest -from aiocache import SimpleMemoryCache +from aiocache.backends.memory import SimpleMemoryBackend, SimpleMemoryCache from aiocache.base import BaseCache from aiocache.serializers import NullSerializer -from aiocache.backends.memory import SimpleMemoryBackend +from ...utils import Keys @pytest.fixture def memory(mocker): - SimpleMemoryBackend._handlers = {} - SimpleMemoryBackend._cache = {} - mocker.spy(SimpleMemoryBackend, "_cache") - return SimpleMemoryBackend() + memory = SimpleMemoryBackend() + mocker.spy(memory, "_cache") + return memory class TestSimpleMemoryBackend: - @pytest.mark.asyncio async def test_get(self, memory): - await memory._get(pytest.KEY) - SimpleMemoryBackend._cache.get.assert_called_with(pytest.KEY) + await memory._get(Keys.KEY) + memory._cache.get.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_gets(self, mocker, memory): mocker.spy(memory, "_get") - await memory._gets(pytest.KEY) - memory._get.assert_called_with(pytest.KEY, encoding="utf-8", _conn=ANY) + await memory._gets(Keys.KEY) + memory._get.assert_called_with(Keys.KEY, encoding="utf-8", _conn=ANY) - @pytest.mark.asyncio async def test_set(self, memory): - await memory._set(pytest.KEY, "value") - SimpleMemoryBackend._cache.__setitem__.assert_called_with(pytest.KEY, "value") + await memory._set(Keys.KEY, "value") + memory._cache.__setitem__.assert_called_with(Keys.KEY, "value") - @pytest.mark.asyncio async def test_set_no_ttl_no_handle(self, memory): - await memory._set(pytest.KEY, "value", ttl=0) - assert pytest.KEY not in memory._handlers + await memory._set(Keys.KEY, "value", ttl=0) + assert Keys.KEY not in memory._handlers - await memory._set(pytest.KEY, "value") - assert pytest.KEY not in memory._handlers + await memory._set(Keys.KEY, "value") + assert Keys.KEY not in memory._handlers - @pytest.mark.asyncio - async def test_set_cancel_previous_ttl_handle(self, memory, mocker): - with patch("asyncio.get_event_loop"): - await memory._set(pytest.KEY, "value", ttl=0.1) - memory._handlers[pytest.KEY].cancel.assert_not_called() + async def test_set_cancel_previous_ttl_handle(self, memory): + with patch("asyncio.get_running_loop", autospec=True): + await memory._set(Keys.KEY, "value", ttl=0.1) + memory._handlers[Keys.KEY].cancel.assert_not_called() - await memory._set(pytest.KEY, "new_value", ttl=0.1) - memory._handlers[pytest.KEY].cancel.assert_called_once_with() + await memory._set(Keys.KEY, "new_value", ttl=0.1) + memory._handlers[Keys.KEY].cancel.assert_called_once_with() - @pytest.mark.asyncio async def test_set_ttl_handle(self, memory): - await memory._set(pytest.KEY, "value", ttl=100) - assert pytest.KEY in memory._handlers - assert isinstance(memory._handlers[pytest.KEY], asyncio.Handle) + await memory._set(Keys.KEY, "value", ttl=100) + assert Keys.KEY in memory._handlers + assert isinstance(memory._handlers[Keys.KEY], asyncio.Handle) - @pytest.mark.asyncio - async def test_set_cas_token(self, mocker, memory): + async def test_set_cas_token(self, memory): memory._cache.get.return_value = "old_value" - assert await memory._set(pytest.KEY, "value", _cas_token="old_value") == 1 - SimpleMemoryBackend._cache.__setitem__.assert_called_with(pytest.KEY, "value") + assert await memory._set(Keys.KEY, "value", _cas_token="old_value") == 1 + memory._cache.__setitem__.assert_called_with(Keys.KEY, "value") - @pytest.mark.asyncio - async def test_set_cas_fail(self, mocker, memory): + async def test_set_cas_fail(self, memory): memory._cache.get.return_value = "value" - assert await memory._set(pytest.KEY, "value", _cas_token="old_value") == 0 - assert SimpleMemoryBackend._cache.__setitem__.call_count == 0 + assert await memory._set(Keys.KEY, "value", _cas_token="old_value") == 0 + assert memory._cache.__setitem__.call_count == 0 - @pytest.mark.asyncio async def test_multi_get(self, memory): - await memory._multi_get([pytest.KEY, pytest.KEY_1]) - SimpleMemoryBackend._cache.get.assert_any_call(pytest.KEY) - SimpleMemoryBackend._cache.get.assert_any_call(pytest.KEY_1) + await memory._multi_get([Keys.KEY, Keys.KEY_1]) + memory._cache.get.assert_any_call(Keys.KEY) + memory._cache.get.assert_any_call(Keys.KEY_1) - @pytest.mark.asyncio async def test_multi_set(self, memory): - await memory._multi_set([(pytest.KEY, "value"), (pytest.KEY_1, "random")]) - SimpleMemoryBackend._cache.__setitem__.assert_any_call(pytest.KEY, "value") - SimpleMemoryBackend._cache.__setitem__.assert_any_call(pytest.KEY_1, "random") + await memory._multi_set([(Keys.KEY, "value"), (Keys.KEY_1, "random")]) + memory._cache.__setitem__.assert_any_call(Keys.KEY, "value") + memory._cache.__setitem__.assert_any_call(Keys.KEY_1, "random") - @pytest.mark.asyncio async def test_add(self, memory, mocker): mocker.spy(memory, "_set") - await memory._add(pytest.KEY, "value") - memory._set.assert_called_with(pytest.KEY, "value", ttl=None) + await memory._add(Keys.KEY, "value") + memory._set.assert_called_with(Keys.KEY, "value", ttl=None) - @pytest.mark.asyncio async def test_add_existing(self, memory): - SimpleMemoryBackend._cache.__contains__.return_value = True + memory._cache.__contains__.return_value = True with pytest.raises(ValueError): - await memory._add(pytest.KEY, "value") + await memory._add(Keys.KEY, "value") - @pytest.mark.asyncio async def test_exists(self, memory): - await memory._exists(pytest.KEY) - SimpleMemoryBackend._cache.__contains__.assert_called_with(pytest.KEY) + await memory._exists(Keys.KEY) + memory._cache.__contains__.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_increment(self, memory): - await memory._increment(pytest.KEY, 2) - SimpleMemoryBackend._cache.__contains__.assert_called_with(pytest.KEY) - SimpleMemoryBackend._cache.__setitem__.assert_called_with(pytest.KEY, 2) + await memory._increment(Keys.KEY, 2) + memory._cache.__contains__.assert_called_with(Keys.KEY) + memory._cache.__setitem__.assert_called_with(Keys.KEY, 2) - @pytest.mark.asyncio async def test_increment_missing(self, memory): - SimpleMemoryBackend._cache.__contains__.return_value = True - SimpleMemoryBackend._cache.__getitem__.return_value = 2 - await memory._increment(pytest.KEY, 2) - SimpleMemoryBackend._cache.__getitem__.assert_called_with(pytest.KEY) - SimpleMemoryBackend._cache.__setitem__.assert_called_with(pytest.KEY, 4) + memory._cache.__contains__.return_value = True + memory._cache.__getitem__.return_value = 2 + await memory._increment(Keys.KEY, 2) + memory._cache.__getitem__.assert_called_with(Keys.KEY) + memory._cache.__setitem__.assert_called_with(Keys.KEY, 4) - @pytest.mark.asyncio async def test_increment_typerror(self, memory): - SimpleMemoryBackend._cache.__contains__.return_value = True - SimpleMemoryBackend._cache.__getitem__.return_value = "asd" + memory._cache.__contains__.return_value = True + memory._cache.__getitem__.return_value = "asd" with pytest.raises(TypeError): - await memory._increment(pytest.KEY, 2) + await memory._increment(Keys.KEY, 2) - @pytest.mark.asyncio async def test_expire_no_handle_no_ttl(self, memory): - SimpleMemoryBackend._cache.__contains__.return_value = True - await memory._expire(pytest.KEY, 0) - assert memory._handlers.get(pytest.KEY) is None + memory._cache.__contains__.return_value = True + await memory._expire(Keys.KEY, 0) + assert memory._handlers.get(Keys.KEY) is None - @pytest.mark.asyncio async def test_expire_no_handle_ttl(self, memory): - SimpleMemoryBackend._cache.__contains__.return_value = True - await memory._expire(pytest.KEY, 1) - assert isinstance(memory._handlers.get(pytest.KEY), asyncio.Handle) + memory._cache.__contains__.return_value = True + await memory._expire(Keys.KEY, 1) + assert isinstance(memory._handlers.get(Keys.KEY), asyncio.Handle) - @pytest.mark.asyncio async def test_expire_handle_ttl(self, memory): - fake = MagicMock() - SimpleMemoryBackend._handlers[pytest.KEY] = fake - SimpleMemoryBackend._cache.__contains__.return_value = True - await memory._expire(pytest.KEY, 1) + fake = create_autospec(asyncio.TimerHandle, instance=True) + memory._handlers[Keys.KEY] = fake + memory._cache.__contains__.return_value = True + await memory._expire(Keys.KEY, 1) assert fake.cancel.call_count == 1 - assert isinstance(memory._handlers.get(pytest.KEY), asyncio.Handle) + assert isinstance(memory._handlers.get(Keys.KEY), asyncio.Handle) - @pytest.mark.asyncio async def test_expire_missing(self, memory): - SimpleMemoryBackend._cache.__contains__.return_value = False - assert await memory._expire(pytest.KEY, 1) is False + memory._cache.__contains__.return_value = False + assert await memory._expire(Keys.KEY, 1) is False - @pytest.mark.asyncio async def test_delete(self, memory): - fake = MagicMock() - SimpleMemoryBackend._handlers[pytest.KEY] = fake - await memory._delete(pytest.KEY) + fake = create_autospec(asyncio.TimerHandle, instance=True) + memory._handlers[Keys.KEY] = fake + await memory._delete(Keys.KEY) assert fake.cancel.call_count == 1 - assert pytest.KEY not in SimpleMemoryBackend._handlers - SimpleMemoryBackend._cache.pop.assert_called_with(pytest.KEY, None) + assert Keys.KEY not in memory._handlers + memory._cache.pop.assert_called_with(Keys.KEY, None) - @pytest.mark.asyncio async def test_delete_missing(self, memory): - SimpleMemoryBackend._cache.pop.return_value = None - await memory._delete(pytest.KEY) - SimpleMemoryBackend._cache.pop.assert_called_with(pytest.KEY, None) + memory._cache.pop.return_value = None + await memory._delete(Keys.KEY) + memory._cache.pop.assert_called_with(Keys.KEY, None) - @pytest.mark.asyncio async def test_delete_non_truthy(self, memory): - non_truthy = MagicMock() + non_truthy = MagicMock(spec_set=("__bool__",)) non_truthy.__bool__.side_effect = ValueError("Does not implement truthiness") with pytest.raises(ValueError): bool(non_truthy) - SimpleMemoryBackend._cache.pop.return_value = non_truthy - await memory._delete(pytest.KEY) + memory._cache.pop.return_value = non_truthy + await memory._delete(Keys.KEY) assert non_truthy.__bool__.call_count == 1 - SimpleMemoryBackend._cache.pop.assert_called_with(pytest.KEY, None) + memory._cache.pop.assert_called_with(Keys.KEY, None) - @pytest.mark.asyncio async def test_clear_namespace(self, memory): - SimpleMemoryBackend._cache.__iter__.return_value = iter(["nma", "nmb", "no"]) + memory._cache.__iter__.return_value = iter(["nma", "nmb", "no"]) await memory._clear("nm") - assert SimpleMemoryBackend._cache.pop.call_count == 2 - SimpleMemoryBackend._cache.pop.assert_any_call("nma", None) - SimpleMemoryBackend._cache.pop.assert_any_call("nmb", None) + assert memory._cache.pop.call_count == 2 + memory._cache.pop.assert_any_call("nma", None) + memory._cache.pop.assert_any_call("nmb", None) - @pytest.mark.asyncio async def test_clear_no_namespace(self, memory): - SimpleMemoryBackend._handlers = "asdad" - SimpleMemoryBackend._cache = "asdad" + memory._handlers = "asdad" + memory._cache = "asdad" await memory._clear() - SimpleMemoryBackend._handlers = {} - SimpleMemoryBackend._cache = {} + memory._handlers = {} + memory._cache = {} - @pytest.mark.asyncio async def test_raw(self, memory): - await memory._raw("get", pytest.KEY) - SimpleMemoryBackend._cache.get.assert_called_with(pytest.KEY) + await memory._raw("get", Keys.KEY) + memory._cache.get.assert_called_with(Keys.KEY) - await memory._set(pytest.KEY, "value") - SimpleMemoryBackend._cache.__setitem__.assert_called_with(pytest.KEY, "value") + await memory._set(Keys.KEY, "value") + memory._cache.__setitem__.assert_called_with(Keys.KEY, "value") - @pytest.mark.asyncio async def test_redlock_release(self, memory): - SimpleMemoryBackend._cache.get.return_value = "lock" - assert await memory._redlock_release(pytest.KEY, "lock") == 1 - SimpleMemoryBackend._cache.get.assert_called_with(pytest.KEY) - SimpleMemoryBackend._cache.pop.assert_called_with(pytest.KEY) + memory._cache.get.return_value = "lock" + assert await memory._redlock_release(Keys.KEY, "lock") == 1 + memory._cache.get.assert_called_with(Keys.KEY) + memory._cache.pop.assert_called_with(Keys.KEY) - @pytest.mark.asyncio async def test_redlock_release_nokey(self, memory): - SimpleMemoryBackend._cache.get.return_value = None - assert await memory._redlock_release(pytest.KEY, "lock") == 0 - SimpleMemoryBackend._cache.get.assert_called_with(pytest.KEY) - assert SimpleMemoryBackend._cache.pop.call_count == 0 + memory._cache.get.return_value = None + assert await memory._redlock_release(Keys.KEY, "lock") == 0 + memory._cache.get.assert_called_with(Keys.KEY) + assert memory._cache.pop.call_count == 0 class TestSimpleMemoryCache: diff --git a/tests/ut/backends/test_redis.py b/tests/ut/backends/test_redis.py index 89b3d3868..10e5d2de2 100644 --- a/tests/ut/backends/test_redis.py +++ b/tests/ut/backends/test_redis.py @@ -1,349 +1,179 @@ -import pytest -import aioredis +from unittest.mock import ANY, AsyncMock, create_autospec, patch -from asynctest import CoroutineMock, MagicMock, patch, ANY +import pytest +from redis.asyncio.client import Pipeline +from redis.exceptions import ResponseError -from aiocache import RedisCache +from aiocache.backends.redis import RedisBackend, RedisCache from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer -from aiocache.backends.redis import RedisBackend, conn, AIOREDIS_BEFORE_ONE - - -@pytest.fixture -def redis_connection(): - conn = MagicMock() - conn.__enter__ = MagicMock(return_value=conn) - conn.__exit__ = MagicMock() - conn.get = CoroutineMock() - conn.mget = CoroutineMock() - conn.set = CoroutineMock() - conn.setex = CoroutineMock() - conn.mset = CoroutineMock() - conn.incrby = CoroutineMock() - conn.exists = CoroutineMock() - conn.persist = CoroutineMock() - conn.expire = CoroutineMock() - conn.delete = CoroutineMock() - conn.flushdb = CoroutineMock() - conn.eval = CoroutineMock() - conn.keys = CoroutineMock() - conn.multi_exec = MagicMock(return_value=conn) - conn.execute = CoroutineMock() - return conn - - -@pytest.fixture -def redis_pool(redis_connection): - class FakePool: - def __await__(self): - yield - return redis_connection - - pool = FakePool() - pool._conn = redis_connection - pool.release = CoroutineMock() - pool.clear = CoroutineMock() - pool.acquire = CoroutineMock(return_value=redis_connection) - pool.__call__ = MagicMock(return_value=pool) - - return pool +from ...utils import Keys, ensure_key @pytest.fixture -def redis(redis_pool): - redis = RedisBackend() - redis._pool = redis_pool - yield redis +def redis(redis_client): + redis = RedisBackend(client=redis_client) + with patch.object(redis, "client", autospec=True) as m: + # These methods actually return an awaitable. + for method in ( + "eval", "expire", "get", "psetex", "setex", "execute_command", "exists", + "incrby", "persist", "delete", "keys", "flushdb", + ): + setattr(m, method, AsyncMock(return_value=None, spec_set=())) + m.mget = AsyncMock(return_value=[None], spec_set=()) + m.set = AsyncMock(return_value=True, spec_set=()) + + m.pipeline.return_value = create_autospec(Pipeline, instance=True) + m.pipeline.return_value.__aenter__.return_value = m.pipeline.return_value + yield redis -@pytest.fixture -def create_pool(): - with patch("aiocache.backends.redis.aioredis.create_pool") as create_pool: - yield create_pool - - -@pytest.fixture(autouse=True) -def mock_redis_v1(mocker, redis_connection): - mocker.patch("aiocache.backends.redis.aioredis.Redis", return_value=redis_connection) +class TestRedisBackend: + @pytest.mark.parametrize("decode_responses", [True]) + async def test_redis_backend_requires_client_decode_responses(self, redis_client): + with pytest.raises(ValueError) as ve: + RedisBackend(client=redis_client) -class TestRedisBackend: - def test_setup(self): - redis_backend = RedisBackend() - assert redis_backend.endpoint == "127.0.0.1" - assert redis_backend.port == 6379 - assert redis_backend.db == 0 - assert redis_backend.password is None - assert redis_backend.pool_min_size == 1 - assert redis_backend.pool_max_size == 10 - - def test_setup_override(self): - redis_backend = RedisBackend(db=2, password="pass") - - assert redis_backend.endpoint == "127.0.0.1" - assert redis_backend.port == 6379 - assert redis_backend.db == 2 - assert redis_backend.password == "pass" - - def test_setup_casts(self): - redis_backend = RedisBackend( - db="2", - port="6379", - pool_min_size="1", - pool_max_size="10", - create_connection_timeout="1.5", + assert str(ve.value) == ( + "redis client must be constructed with decode_responses set to False" ) - assert redis_backend.db == 2 - assert redis_backend.port == 6379 - assert redis_backend.pool_min_size == 1 - assert redis_backend.pool_max_size == 10 - assert redis_backend.create_connection_timeout == 1.5 - - @pytest.mark.asyncio - async def test_acquire_conn(self, redis, redis_connection): - assert await redis.acquire_conn() == redis_connection - - @pytest.mark.asyncio - async def test_release_conn(self, redis): - conn = await redis.acquire_conn() - await redis.release_conn(conn) - if AIOREDIS_BEFORE_ONE: - redis._pool.release.assert_called_with(conn) - else: - redis._pool.release.assert_called_with(conn.connection) - - @pytest.mark.asyncio - async def test_get_pool_sets_pool(self, redis, redis_pool, create_pool): - redis._pool = None - await redis._get_pool() - assert redis._pool == create_pool.return_value - - @pytest.mark.asyncio - async def test_get_pool_reuses_existing_pool(self, redis): - redis._pool = "pool" - await redis._get_pool() - assert redis._pool == "pool" - - @pytest.mark.asyncio - async def test_get_pool_locked(self, mocker, redis, create_pool): - redis._pool = None - mocker.spy(redis._pool_lock, "acquire") - mocker.spy(redis._pool_lock, "release") - - assert await redis._get_pool() == create_pool.return_value - assert redis._pool_lock.acquire.call_count == 1 - assert redis._pool_lock.release.call_count == 1 - - @pytest.mark.asyncio - async def test_get_pool_calls_create_pool(self, redis, create_pool): - redis._pool = None - await redis._get_pool() - if AIOREDIS_BEFORE_ONE: - create_pool.assert_called_with( - (redis.endpoint, redis.port), - db=redis.db, - password=redis.password, - loop=redis._loop, - encoding="utf-8", - minsize=redis.pool_min_size, - maxsize=redis.pool_max_size, - ) - else: - create_pool.assert_called_with( - (redis.endpoint, redis.port), - db=redis.db, - password=redis.password, - loop=redis._loop, - encoding="utf-8", - minsize=redis.pool_min_size, - maxsize=redis.pool_max_size, - create_connection_timeout=redis.create_connection_timeout, - ) - - @pytest.mark.asyncio - async def test_get(self, redis, redis_connection): - await redis._get(pytest.KEY) - redis_connection.get.assert_called_with(pytest.KEY, encoding="utf-8") - - @pytest.mark.asyncio - async def test_gets(self, mocker, redis, redis_connection): + async def test_get(self, redis): + redis.client.get.return_value = b"value" + assert await redis._get(Keys.KEY) == "value" + redis.client.get.assert_called_with(Keys.KEY) + + async def test_gets(self, mocker, redis): mocker.spy(redis, "_get") - await redis._gets(pytest.KEY) - redis._get.assert_called_with(pytest.KEY, encoding="utf-8", _conn=ANY) + await redis._gets(Keys.KEY) + redis._get.assert_called_with(Keys.KEY, encoding="utf-8", _conn=ANY) - @pytest.mark.asyncio - async def test_set(self, redis, redis_connection): - await redis._set(pytest.KEY, "value") - redis_connection.set.assert_called_with(pytest.KEY, "value") + async def test_set(self, redis): + await redis._set(Keys.KEY, "value") + redis.client.set.assert_called_with(Keys.KEY, "value") - await redis._set(pytest.KEY, "value", ttl=1) - redis_connection.setex.assert_called_with(pytest.KEY, 1, "value") + await redis._set(Keys.KEY, "value", ttl=1) + redis.client.setex.assert_called_with(Keys.KEY, 1, "value") - @pytest.mark.asyncio - async def test_set_cas_token(self, mocker, redis, redis_connection): + async def test_set_cas_token(self, mocker, redis): mocker.spy(redis, "_cas") - await redis._set(pytest.KEY, "value", _cas_token="old_value", _conn=redis_connection) + await redis._set(Keys.KEY, "value", _cas_token="old_value", _conn=redis.client) redis._cas.assert_called_with( - pytest.KEY, "value", "old_value", ttl=None, _conn=redis_connection + Keys.KEY, "value", "old_value", ttl=None, _conn=redis.client ) - @pytest.mark.asyncio - async def test_cas(self, mocker, redis, redis_connection): + async def test_cas(self, mocker, redis): mocker.spy(redis, "_raw") - await redis._cas(pytest.KEY, "value", "old_value", ttl=10, _conn=redis_connection) + await redis._cas(Keys.KEY, "value", "old_value", ttl=10, _conn=redis.client) redis._raw.assert_called_with( "eval", redis.CAS_SCRIPT, - [pytest.KEY], - ["value", "old_value", "EX", 10], - _conn=redis_connection, + 1, + *[Keys.KEY, "value", "old_value", "EX", 10], + _conn=redis.client, ) - @pytest.mark.asyncio - async def test_cas_float_ttl(self, mocker, redis, redis_connection): + async def test_cas_float_ttl(self, mocker, redis): mocker.spy(redis, "_raw") - await redis._cas(pytest.KEY, "value", "old_value", ttl=0.1, _conn=redis_connection) + await redis._cas(Keys.KEY, "value", "old_value", ttl=0.1, _conn=redis.client) redis._raw.assert_called_with( "eval", redis.CAS_SCRIPT, - [pytest.KEY], - ["value", "old_value", "PX", 100], - _conn=redis_connection, + 1, + *[Keys.KEY, "value", "old_value", "PX", 100], + _conn=redis.client, + ) + + async def test_multi_get(self, redis): + await redis._multi_get([Keys.KEY, Keys.KEY_1]) + redis.client.mget.assert_called_with(Keys.KEY, Keys.KEY_1) + + async def test_multi_set(self, redis): + await redis._multi_set([(Keys.KEY, "value"), (Keys.KEY_1, "random")]) + redis.client.execute_command.assert_called_with( + "MSET", Keys.KEY, "value", Keys.KEY_1, "random" + ) + + async def test_multi_set_with_ttl(self, redis): + await redis._multi_set([(Keys.KEY, "value"), (Keys.KEY_1, "random")], ttl=1) + assert redis.client.pipeline.call_count == 1 + pipeline = redis.client.pipeline.return_value + pipeline.execute_command.assert_called_with( + "MSET", Keys.KEY, "value", Keys.KEY_1, "random" ) + pipeline.expire.assert_any_call(Keys.KEY, time=1) + pipeline.expire.assert_any_call(Keys.KEY_1, time=1) + assert pipeline.execute.call_count == 1 - @pytest.mark.asyncio - async def test_multi_get(self, redis, redis_connection): - await redis._multi_get([pytest.KEY, pytest.KEY_1]) - redis_connection.mget.assert_called_with(pytest.KEY, pytest.KEY_1, encoding="utf-8") - - @pytest.mark.asyncio - async def test_multi_set(self, redis, redis_connection): - await redis._multi_set([(pytest.KEY, "value"), (pytest.KEY_1, "random")]) - redis_connection.mset.assert_called_with(pytest.KEY, "value", pytest.KEY_1, "random") - - @pytest.mark.asyncio - async def test_multi_set_with_ttl(self, redis, redis_connection): - await redis._multi_set([(pytest.KEY, "value"), (pytest.KEY_1, "random")], ttl=1) - assert redis_connection.multi_exec.call_count == 1 - redis_connection.mset.assert_called_with(pytest.KEY, "value", pytest.KEY_1, "random") - redis_connection.expire.assert_any_call(pytest.KEY, timeout=1) - redis_connection.expire.assert_any_call(pytest.KEY_1, timeout=1) - assert redis_connection.execute.call_count == 1 - - @pytest.mark.asyncio - async def test_add(self, redis, redis_connection): - await redis._add(pytest.KEY, "value") - redis_connection.set.assert_called_with(pytest.KEY, "value", exist=ANY, expire=None) - - await redis._add(pytest.KEY, "value", 1) - redis_connection.set.assert_called_with(pytest.KEY, "value", exist=ANY, expire=1) - - @pytest.mark.asyncio - async def test_add_existing(self, redis, redis_connection): - redis_connection.set.return_value = False + async def test_add(self, redis): + await redis._add(Keys.KEY, "value") + redis.client.set.assert_called_with(Keys.KEY, "value", nx=True, ex=None) + + await redis._add(Keys.KEY, "value", 1) + redis.client.set.assert_called_with(Keys.KEY, "value", nx=True, ex=1) + + async def test_add_existing(self, redis): + redis.client.set.return_value = False with pytest.raises(ValueError): - await redis._add(pytest.KEY, "value") - - @pytest.mark.asyncio - async def test_add_float_ttl(self, redis, redis_connection): - await redis._add(pytest.KEY, "value", 0.1) - redis_connection.set.assert_called_with(pytest.KEY, "value", exist=ANY, pexpire=100) - - @pytest.mark.asyncio - async def test_exists(self, redis, redis_connection): - redis_connection.exists.return_value = 1 - await redis._exists(pytest.KEY) - redis_connection.exists.assert_called_with(pytest.KEY) - - @pytest.mark.asyncio - async def test_expire(self, redis, redis_connection): - await redis._expire(pytest.KEY, ttl=1) - redis_connection.expire.assert_called_with(pytest.KEY, 1) - - @pytest.mark.asyncio - async def test_increment(self, redis, redis_connection): - await redis._increment(pytest.KEY, delta=2) - redis_connection.incrby.assert_called_with(pytest.KEY, 2) - - @pytest.mark.asyncio - async def test_increment_typerror(self, redis, redis_connection): - redis_connection.incrby.side_effect = aioredis.errors.ReplyError("msg") + await redis._add(Keys.KEY, "value") + + async def test_add_float_ttl(self, redis): + await redis._add(Keys.KEY, "value", 0.1) + redis.client.set.assert_called_with(Keys.KEY, "value", nx=True, px=100) + + async def test_exists(self, redis): + redis.client.exists.return_value = 1 + await redis._exists(Keys.KEY) + redis.client.exists.assert_called_with(Keys.KEY) + + async def test_increment(self, redis): + await redis._increment(Keys.KEY, delta=2) + redis.client.incrby.assert_called_with(Keys.KEY, 2) + + async def test_increment_typerror(self, redis): + redis.client.incrby.side_effect = ResponseError("msg") with pytest.raises(TypeError): - await redis._increment(pytest.KEY, 2) + await redis._increment(Keys.KEY, delta=2) + redis.client.incrby.assert_called_with(Keys.KEY, 2) + + async def test_expire(self, redis): + await redis._expire(Keys.KEY, 1) + redis.client.expire.assert_called_with(Keys.KEY, 1) + await redis._increment(Keys.KEY, 2) - @pytest.mark.asyncio - async def test_expire_0_ttl(self, redis, redis_connection): - await redis._expire(pytest.KEY, ttl=0) - redis_connection.persist.assert_called_with(pytest.KEY) + async def test_expire_0_ttl(self, redis): + await redis._expire(Keys.KEY, ttl=0) + redis.client.persist.assert_called_with(Keys.KEY) - @pytest.mark.asyncio - async def test_delete(self, redis, redis_connection): - await redis._delete(pytest.KEY) - redis_connection.delete.assert_called_with(pytest.KEY) + async def test_delete(self, redis): + await redis._delete(Keys.KEY) + redis.client.delete.assert_called_with(Keys.KEY) - @pytest.mark.asyncio - async def test_clear(self, redis, redis_connection): - redis_connection.keys.return_value = ["nm:a", "nm:b"] + async def test_clear(self, redis): + redis.client.keys.return_value = ["nm:a", "nm:b"] await redis._clear("nm") - redis_connection.delete.assert_called_with("nm:a", "nm:b") + redis.client.delete.assert_called_with("nm:a", "nm:b") - @pytest.mark.asyncio - async def test_clear_no_keys(self, redis, redis_connection): - redis_connection.keys.return_value = [] + async def test_clear_no_keys(self, redis): + redis.client.keys.return_value = [] await redis._clear("nm") - redis_connection.delete.assert_not_called() + redis.client.delete.assert_not_called() - @pytest.mark.asyncio - async def test_clear_no_namespace(self, redis, redis_connection): + async def test_clear_no_namespace(self, redis): await redis._clear() - assert redis_connection.flushdb.call_count == 1 + assert redis.client.flushdb.call_count == 1 - @pytest.mark.asyncio - async def test_raw(self, redis, redis_connection): - await redis._raw("get", pytest.KEY) - await redis._raw("set", pytest.KEY, 1) - redis_connection.get.assert_called_with(pytest.KEY, encoding=ANY) - redis_connection.set.assert_called_with(pytest.KEY, 1) + async def test_raw(self, redis): + await redis._raw("get", Keys.KEY) + await redis._raw("set", Keys.KEY, 1) + redis.client.get.assert_called_with(Keys.KEY) + redis.client.set.assert_called_with(Keys.KEY, 1) - @pytest.mark.asyncio async def test_redlock_release(self, mocker, redis): mocker.spy(redis, "_raw") - await redis._redlock_release(pytest.KEY, "random") - redis._raw.assert_called_with("eval", redis.RELEASE_SCRIPT, [pytest.KEY], ["random"]) - - @pytest.mark.asyncio - async def test_close_when_connected(self, redis): - await redis._raw("set", pytest.KEY, 1) - await redis._close() - assert redis._pool.clear.call_count == 1 - - @pytest.mark.asyncio - async def test_close_when_not_connected(self, redis, redis_pool): - redis._pool = None - await redis._close() - assert redis_pool.clear.call_count == 0 - - -class TestConn: - async def dummy(self, *args, _conn=None, **kwargs): - pass - - @pytest.mark.asyncio - async def test_conn(self, redis, redis_connection, mocker): - mocker.spy(self, "dummy") - d = conn(self.dummy) - await d(redis, "a", _conn=None) - self.dummy.assert_called_with(redis, "a", _conn=redis_connection) - - @pytest.mark.asyncio - async def test_conn_reuses(self, redis, redis_connection, mocker): - mocker.spy(self, "dummy") - d = conn(self.dummy) - await d(redis, "a", _conn=redis_connection) - self.dummy.assert_called_with(redis, "a", _conn=redis_connection) - await d(redis, "a", _conn=redis_connection) - self.dummy.assert_called_with(redis, "a", _conn=redis_connection) + await redis._redlock_release(Keys.KEY, "random") + redis._raw.assert_called_with("eval", redis.RELEASE_SCRIPT, 1, Keys.KEY, "random") class TestRedisCache: @@ -356,24 +186,24 @@ def set_test_namespace(self, redis_cache): def test_name(self): assert RedisCache.NAME == "redis" - def test_inheritance(self): - assert isinstance(RedisCache(), BaseCache) + def test_inheritance(self, redis_client): + assert isinstance(RedisCache(client=redis_client), BaseCache) - def test_default_serializer(self): - assert isinstance(RedisCache().serializer, JsonSerializer) + def test_default_serializer(self, redis_client): + assert isinstance(RedisCache(client=redis_client).serializer, JsonSerializer) @pytest.mark.parametrize( "path,expected", [("", {}), ("/", {}), ("/1", {"db": "1"}), ("/1/2/3", {"db": "1"})] ) - def test_parse_uri_path(self, path, expected): - assert RedisCache().parse_uri_path(path) == expected + def test_parse_uri_path(self, path, expected, redis_client): + assert RedisCache(client=redis_client).parse_uri_path(path) == expected @pytest.mark.parametrize( "namespace, expected", - ([None, "test:" + pytest.KEY], ["", pytest.KEY], ["my_ns", "my_ns:" + pytest.KEY]), + ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key_double_dot(self, set_test_namespace, redis_cache, namespace, expected): - assert redis_cache.build_key(pytest.KEY, namespace=namespace) == expected + assert redis_cache.build_key(Keys.KEY, namespace) == expected def test_build_key_no_namespace(self, redis_cache): - assert redis_cache.build_key(pytest.KEY, namespace=None) == pytest.KEY + assert redis_cache.build_key(Keys.KEY, namespace=None) == Keys.KEY diff --git a/tests/ut/conftest.py b/tests/ut/conftest.py index 8a3cbbc7f..591f1c44f 100644 --- a/tests/ut/conftest.py +++ b/tests/ut/conftest.py @@ -1,20 +1,11 @@ +from contextlib import ExitStack +from unittest.mock import create_autospec, patch + import pytest -import asynctest -from aiocache.base import BaseCache, API -from aiocache import caches, RedisCache, MemcachedCache +from aiocache import caches from aiocache.plugins import BasePlugin -from aiocache.serializers import BaseSerializer - - -def pytest_configure(): - """ - Before pytest_namespace was being used to set the keys for - testing but the feature was removed - https://docs.pytest.org/en/latest/deprecations.html#pytest-namespace - """ - pytest.KEY = "key" - pytest.KEY_1 = "random" +from ..utils import AbstractBaseCache, ConcreteBaseCache @pytest.fixture(autouse=True) @@ -29,53 +20,49 @@ def reset_caches(): ) -class MockCache(BaseCache): - def __init__(self): - super().__init__() - self._add = asynctest.CoroutineMock() - self._get = asynctest.CoroutineMock() - self._gets = asynctest.CoroutineMock() - self._set = asynctest.CoroutineMock() - self._multi_get = asynctest.CoroutineMock(return_value=["a", "b"]) - self._multi_set = asynctest.CoroutineMock() - self._delete = asynctest.CoroutineMock() - self._exists = asynctest.CoroutineMock() - self._increment = asynctest.CoroutineMock() - self._expire = asynctest.CoroutineMock() - self._clear = asynctest.CoroutineMock() - self._raw = asynctest.CoroutineMock() - self._redlock_release = asynctest.CoroutineMock() - self.acquire_conn = asynctest.CoroutineMock() - self.release_conn = asynctest.CoroutineMock() - self._close = asynctest.CoroutineMock() +@pytest.fixture +def mock_cache(mocker): + return create_autospec(ConcreteBaseCache()) @pytest.fixture -def mock_cache(mocker): - cache = MockCache() - cache.timeout = 0.002 - mocker.spy(cache, "_build_key") - for cmd in API.CMDS: - mocker.spy(cache, cmd.__name__) - mocker.spy(cache, "close") - cache.serializer = asynctest.Mock(spec=BaseSerializer) - cache.serializer.encoding = "utf-8" - cache.plugins = [asynctest.Mock(spec=BasePlugin)] - return cache +def mock_base_cache(): + """Return BaseCache instance with unimplemented methods mocked out.""" + plugin = create_autospec(BasePlugin, instance=True) + cache = ConcreteBaseCache(timeout=0.002, plugins=(plugin,)) + methods = ("_add", "_get", "_gets", "_set", "_multi_get", "_multi_set", "_delete", + "_exists", "_increment", "_expire", "_clear", "_raw", "_close", + "_redlock_release", "acquire_conn", "release_conn") + with ExitStack() as stack: + for f in methods: + stack.enter_context(patch.object(cache, f, autospec=True)) + stack.enter_context(patch.object(cache, "_serializer", autospec=True)) + stack.enter_context(patch.object(cache, "build_key", cache._str_build_key)) + yield cache @pytest.fixture -def base_cache(): - return BaseCache() +def abstract_base_cache(): + return AbstractBaseCache() @pytest.fixture -def redis_cache(): - cache = RedisCache() +def base_cache(): + cache = ConcreteBaseCache() return cache @pytest.fixture -def memcached_cache(): - cache = MemcachedCache() - return cache +async def redis_cache(redis_client): + from aiocache.backends.redis import RedisCache + + async with RedisCache(client=redis_client) as cache: + yield cache + + +@pytest.fixture +async def memcached_cache(): + from aiocache.backends.memcached import MemcachedCache + + async with MemcachedCache() as cache: + yield cache diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index f0612c121..cfda509e0 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -1,17 +1,18 @@ -import os -import pytest import asyncio +import os +from unittest.mock import ANY, AsyncMock, MagicMock, patch -from asynctest import patch, MagicMock, ANY, CoroutineMock +import pytest -from aiocache.base import API, _Conn, BaseCache +from aiocache.base import API, _Conn +from ..utils import AbstractBaseCache, ConcreteBaseCache, Keys, ensure_key class TestAPI: def test_register(self): @API.register def dummy(): - pass + """Dummy function.""" assert dummy in API.CMDS API.unregister(dummy) @@ -19,19 +20,18 @@ def dummy(): def test_unregister(self): @API.register def dummy(): - pass + "Dummy function.""" API.unregister(dummy) assert dummy not in API.CMDS def test_unregister_unexisting(self): def dummy(): - pass + "Dummy function.""" API.unregister(dummy) assert dummy not in API.CMDS - @pytest.mark.asyncio async def test_aiocache_enabled(self): @API.aiocache_enabled() async def dummy(*args, **kwargs): @@ -39,18 +39,16 @@ async def dummy(*args, **kwargs): assert await dummy() is True - @pytest.mark.asyncio async def test_aiocache_enabled_disabled(self): @API.aiocache_enabled(fake_return=[]) async def dummy(*args, **kwargs): - return True + """Dummy function.""" with patch.dict(os.environ, {"AIOCACHE_DISABLE": "1"}): assert await dummy() == [] - @pytest.mark.asyncio async def test_timeout_no_timeout(self): - self = MagicMock() + self = MagicMock(spec_set=("timeout",)) self.timeout = 0 @API.timeout @@ -62,9 +60,8 @@ async def dummy(self): assert self.call_count == 1 assert wait_for.call_count == 0 - @pytest.mark.asyncio async def test_timeout_self(self): - self = MagicMock() + self = MagicMock(spec_set=("timeout",)) self.timeout = 0.002 @API.timeout @@ -74,9 +71,8 @@ async def dummy(self): with pytest.raises(asyncio.TimeoutError): await dummy(self) - @pytest.mark.asyncio async def test_timeout_kwarg_0(self): - self = MagicMock() + self = MagicMock(spec_set=("timeout",)) self.timeout = 0.002 @API.timeout @@ -86,9 +82,8 @@ async def dummy(self): assert await dummy(self, timeout=0) is True - @pytest.mark.asyncio async def test_timeout_kwarg_None(self): - self = MagicMock() + self = MagicMock(spec_set=("timeout",)) self.timeout = 0.002 @API.timeout @@ -98,9 +93,8 @@ async def dummy(self): assert await dummy(self, timeout=None) is True - @pytest.mark.asyncio async def test_timeout_kwarg(self): - self = MagicMock() + self = MagicMock(spec_set=("timeout",)) @API.timeout async def dummy(self): @@ -109,9 +103,8 @@ async def dummy(self): with pytest.raises(asyncio.TimeoutError): await dummy(self, timeout=0.002) - @pytest.mark.asyncio async def test_timeout_self_kwarg(self): - self = MagicMock() + self = MagicMock(spec_set=("timeout",)) self.timeout = 5 @API.timeout @@ -121,16 +114,15 @@ async def dummy(self): with pytest.raises(asyncio.TimeoutError): await dummy(self, timeout=0.003) - @pytest.mark.asyncio async def test_plugins(self): - self = MagicMock() - plugin1 = MagicMock() - plugin1.pre_dummy = CoroutineMock() - plugin1.post_dummy = CoroutineMock() - plugin2 = MagicMock() - plugin2.pre_dummy = CoroutineMock() - plugin2.post_dummy = CoroutineMock() - self.plugins = [plugin1, plugin2] + self = MagicMock(spec_set=("plugins",)) + plugin1 = MagicMock(spec_set=("pre_dummy", "post_dummy")) + plugin1.pre_dummy = AsyncMock(spec_set=()) + plugin1.post_dummy = AsyncMock(spec_set=()) + plugin2 = MagicMock(spec_set=("pre_dummy", "post_dummy")) + plugin2.pre_dummy = AsyncMock(spec_set=()) + plugin2.post_dummy = AsyncMock(spec_set=()) + self.plugins = (plugin1, plugin2) @API.plugins async def dummy(self, *args, **kwargs): @@ -145,217 +137,284 @@ async def dummy(self, *args, **kwargs): class TestBaseCache: def test_str_ttl(self): - cache = BaseCache(ttl="1.5") + cache = AbstractBaseCache(ttl="1.5") assert cache.ttl == 1.5 def test_str_timeout(self): - cache = BaseCache(timeout="1.5") + cache = AbstractBaseCache(timeout="1.5") assert cache.timeout == 1.5 - @pytest.mark.asyncio async def test_add(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._add(pytest.KEY, "value", 0) + await base_cache._add(Keys.KEY, "value", 0) - @pytest.mark.asyncio async def test_get(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._get(pytest.KEY, "utf-8") + await base_cache._get(Keys.KEY, "utf-8") - @pytest.mark.asyncio async def test_set(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._set(pytest.KEY, "value", 0) + await base_cache._set(Keys.KEY, "value", 0) - @pytest.mark.asyncio async def test_multi_get(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._multi_get([pytest.KEY], encoding="utf-8") + await base_cache._multi_get([Keys.KEY], encoding="utf-8") - @pytest.mark.asyncio async def test_multi_set(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._multi_set([(pytest.KEY, "value")], 0) + await base_cache._multi_set([(Keys.KEY, "value")], 0) - @pytest.mark.asyncio async def test_delete(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._delete(pytest.KEY) + await base_cache._delete(Keys.KEY) - @pytest.mark.asyncio async def test_exists(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._exists(pytest.KEY) + await base_cache._exists(Keys.KEY) - @pytest.mark.asyncio async def test_increment(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._increment(pytest.KEY, 2) + await base_cache._increment(Keys.KEY, 2) - @pytest.mark.asyncio async def test_expire(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._expire(pytest.KEY, 0) + await base_cache._expire(Keys.KEY, 0) - @pytest.mark.asyncio async def test_clear(self, base_cache): with pytest.raises(NotImplementedError): await base_cache._clear("namespace") - @pytest.mark.asyncio async def test_raw(self, base_cache): with pytest.raises(NotImplementedError): - await base_cache._raw("get", pytest.KEY) + await base_cache._raw("get", Keys.KEY) - @pytest.mark.asyncio async def test_close(self, base_cache): assert await base_cache._close() is None - @pytest.mark.asyncio async def test_acquire_conn(self, base_cache): assert await base_cache.acquire_conn() == base_cache - @pytest.mark.asyncio async def test_release_conn(self, base_cache): - await base_cache.release_conn("mock") is None + assert await base_cache.release_conn("mock") is None + + def test_abstract_build_key(self, abstract_base_cache): + with pytest.raises(NotImplementedError): + abstract_base_cache.build_key(Keys.KEY) @pytest.fixture def set_test_namespace(self, base_cache): base_cache.namespace = "test" yield - base_cache.namespace = None + base_cache.namespace = "" @pytest.mark.parametrize( "namespace, expected", - ([None, "test" + pytest.KEY], ["", pytest.KEY], ["my_ns", "my_ns" + pytest.KEY]), + ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 + ) + def test_str_build_key(self, set_test_namespace, namespace, expected): + # TODO: Runtime check for namespace=None: Raise ValueError or replace with ""? + cache = AbstractBaseCache(namespace=namespace) + assert cache._str_build_key(Keys.KEY) == expected + + @pytest.mark.parametrize( + "namespace, expected", + ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key(self, set_test_namespace, base_cache, namespace, expected): - assert base_cache.build_key(pytest.KEY, namespace=namespace) == expected + assert base_cache.build_key(Keys.KEY, namespace) == expected def test_alt_build_key(self): - cache = BaseCache(key_builder=lambda key, namespace: "x") - assert cache.build_key(pytest.KEY, "namespace") == "x" + cache = ConcreteBaseCache(key_builder=lambda key, namespace: "x") + assert cache.build_key(Keys.KEY, "namespace") == "x" - @pytest.mark.asyncio - async def test_add_ttl_cache_default(self, base_cache): - base_cache._add = CoroutineMock() + def alt_build_key(self, key, namespace): + """Custom key_builder for cache""" + sep = ":" if namespace else "" + return f"{namespace}{sep}{ensure_key(key)}" + + @pytest.mark.parametrize( + "namespace, expected", + ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # noqa: B950 + ) + def test_alt_build_key_override_namespace(self, namespace, expected): + """Custom key_builder overrides namespace of cache""" + cache = ConcreteBaseCache(key_builder=self.alt_build_key, namespace="test") + assert cache.build_key(Keys.KEY, namespace) == expected + + @pytest.mark.parametrize( + "namespace, expected", + ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["test", "test:" + ensure_key(Keys.KEY)]), # noqa: B950 + ) + async def test_alt_build_key_default_namespace(self, namespace, expected): + """Custom key_builder for cache with or without namespace specified. + + Cache member functions that accept a ``namespace`` parameter + should default to using ``self.namespace`` if the ``namespace`` + argument is ``None``. + + This enables a cache to correctly build keys when the cache is + initialized with both a ``namespace`` and a ``key_builder``, + even when that cache is supplied to a lock or to a decorator + using the ``alias`` argument. + """ + cache = ConcreteBaseCache(key_builder=self.alt_build_key, namespace=namespace) + + # Verify that private members are called with the correct ns_key + await self._assert_add__alt_build_key_default_namespace(cache, expected) + await self._assert_get__alt_build_key_default_namespace(cache, expected) + await self._assert_multi_get__alt_build_key_default_namespace(cache, expected) + await self._assert_set__alt_build_key_default_namespace(cache, expected) + await self._assert_multi_set__alt_build_key_default_namespace(cache, expected) + await self._assert_exists__alt_build_key_default_namespace(cache, expected) + await self._assert_increment__alt_build_key_default_namespace(cache, expected) + await self._assert_delete__alt_build_key_default_namespace(cache, expected) + await self._assert_expire__alt_build_key_default_namespace(cache, expected) + + async def _assert_add__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_add", autospec=True) as _add: + await cache.add(Keys.KEY, "value") + _add.assert_called_once_with(expected, "value", _conn=None, ttl=None) + + async def _assert_get__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_get", autospec=True) as _get: + await cache.get(Keys.KEY) + _get.assert_called_once_with( + expected, _conn=None, encoding=cache.serializer.encoding) + + async def _assert_multi_get__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_multi_get", autospec=True) as _multi_get: + await cache.multi_get([Keys.KEY]) + _multi_get.assert_called_once_with( + [expected], _conn=None, encoding=cache.serializer.encoding) + + async def _assert_set__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_set", autospec=True) as _set: + await cache.set(Keys.KEY, "value") + _set.assert_called_once_with( + expected, "value", _conn=None, ttl=None, _cas_token=None) + + async def _assert_multi_set__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_multi_set", autospec=True) as _multi_set: + await cache.multi_set([(Keys.KEY, "value")]) + _multi_set.assert_called_once_with( + [(expected, "value")], _conn=None, ttl=None) + + async def _assert_exists__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_exists", autospec=True) as _exists: + await cache.exists(Keys.KEY) + _exists.assert_called_once_with(expected, _conn=None) + + async def _assert_increment__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_increment", autospec=True) as _increment: + await cache.increment(Keys.KEY) + _increment.assert_called_once_with(expected, delta=1, _conn=None) + + async def _assert_delete__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_delete", autospec=True) as _delete: + await cache.delete(Keys.KEY) + _delete.assert_called_once_with(expected, _conn=None) + + async def _assert_expire__alt_build_key_default_namespace(self, cache, expected): + with patch.object(cache, "_expire", autospec=True) as _expire: + await cache.expire(Keys.KEY, 0) + _expire.assert_called_once_with(expected, 0, _conn=None) - await base_cache.add(pytest.KEY, "value") + async def test_add_ttl_cache_default(self, base_cache): + with patch.object(base_cache, "_add", autospec=True) as m: + await base_cache.add(Keys.KEY, "value") - base_cache._add.assert_called_once_with(pytest.KEY, "value", _conn=None, ttl=None) + m.assert_called_once_with(Keys.KEY, "value", _conn=None, ttl=None) - @pytest.mark.asyncio async def test_add_ttl_default(self, base_cache): base_cache.ttl = 10 - base_cache._add = CoroutineMock() - - await base_cache.add(pytest.KEY, "value") + with patch.object(base_cache, "_add", autospec=True) as m: + await base_cache.add(Keys.KEY, "value") - base_cache._add.assert_called_once_with(pytest.KEY, "value", _conn=None, ttl=10) + m.assert_called_once_with(Keys.KEY, "value", _conn=None, ttl=10) - @pytest.mark.asyncio async def test_add_ttl_overriden(self, base_cache): base_cache.ttl = 10 - base_cache._add = CoroutineMock() - - await base_cache.add(pytest.KEY, "value", ttl=20) + with patch.object(base_cache, "_add", autospec=True) as m: + await base_cache.add(Keys.KEY, "value", ttl=20) - base_cache._add.assert_called_once_with(pytest.KEY, "value", _conn=None, ttl=20) + m.assert_called_once_with(Keys.KEY, "value", _conn=None, ttl=20) - @pytest.mark.asyncio async def test_add_ttl_none(self, base_cache): base_cache.ttl = 10 - base_cache._add = CoroutineMock() + with patch.object(base_cache, "_add", autospec=True) as m: + await base_cache.add(Keys.KEY, "value", ttl=None) - await base_cache.add(pytest.KEY, "value", ttl=None) + m.assert_called_once_with(Keys.KEY, "value", _conn=None, ttl=None) - base_cache._add.assert_called_once_with(pytest.KEY, "value", _conn=None, ttl=None) - - @pytest.mark.asyncio async def test_set_ttl_cache_default(self, base_cache): - base_cache._set = CoroutineMock() - - await base_cache.set(pytest.KEY, "value") + with patch.object(base_cache, "_set", autospec=True) as m: + await base_cache.set(Keys.KEY, "value") - base_cache._set.assert_called_once_with( - pytest.KEY, "value", _cas_token=None, _conn=None, ttl=None - ) + m.assert_called_once_with( + Keys.KEY, "value", _cas_token=None, _conn=None, ttl=None + ) - @pytest.mark.asyncio async def test_set_ttl_default(self, base_cache): base_cache.ttl = 10 - base_cache._set = CoroutineMock() + with patch.object(base_cache, "_set", autospec=True) as m: + await base_cache.set(Keys.KEY, "value") - await base_cache.set(pytest.KEY, "value") + m.assert_called_once_with( + Keys.KEY, "value", _cas_token=None, _conn=None, ttl=10 + ) - base_cache._set.assert_called_once_with( - pytest.KEY, "value", _cas_token=None, _conn=None, ttl=10 - ) - - @pytest.mark.asyncio async def test_set_ttl_overriden(self, base_cache): base_cache.ttl = 10 - base_cache._set = CoroutineMock() - - await base_cache.set(pytest.KEY, "value", ttl=20) + with patch.object(base_cache, "_set", autospec=True) as m: + await base_cache.set(Keys.KEY, "value", ttl=20) - base_cache._set.assert_called_once_with( - pytest.KEY, "value", _cas_token=None, _conn=None, ttl=20 - ) + m.assert_called_once_with( + Keys.KEY, "value", _cas_token=None, _conn=None, ttl=20 + ) - @pytest.mark.asyncio async def test_set_ttl_none(self, base_cache): base_cache.ttl = 10 - base_cache._set = CoroutineMock() - - await base_cache.set(pytest.KEY, "value", ttl=None) + with patch.object(base_cache, "_set", autospec=True) as m: + await base_cache.set(Keys.KEY, "value", ttl=None) - base_cache._set.assert_called_once_with( - pytest.KEY, "value", _cas_token=None, _conn=None, ttl=None - ) + m.assert_called_once_with( + Keys.KEY, "value", _cas_token=None, _conn=None, ttl=None + ) - @pytest.mark.asyncio async def test_multi_set_ttl_cache_default(self, base_cache): - base_cache._multi_set = CoroutineMock() + with patch.object(base_cache, "_multi_set", autospec=True) as m: + await base_cache.multi_set([[Keys.KEY, "value"], [Keys.KEY_1, "value1"]]) - await base_cache.multi_set([[pytest.KEY, "value"], [pytest.KEY_1, "value1"]]) + m.assert_called_once_with( + [(Keys.KEY, "value"), (Keys.KEY_1, "value1")], _conn=None, ttl=None + ) - base_cache._multi_set.assert_called_once_with( - [(pytest.KEY, "value"), (pytest.KEY_1, "value1")], _conn=None, ttl=None - ) - - @pytest.mark.asyncio async def test_multi_set_ttl_default(self, base_cache): base_cache.ttl = 10 - base_cache._multi_set = CoroutineMock() - - await base_cache.multi_set([[pytest.KEY, "value"], [pytest.KEY_1, "value1"]]) + with patch.object(base_cache, "_multi_set", autospec=True) as m: + await base_cache.multi_set([[Keys.KEY, "value"], [Keys.KEY_1, "value1"]]) - base_cache._multi_set.assert_called_once_with( - [(pytest.KEY, "value"), (pytest.KEY_1, "value1")], _conn=None, ttl=10 - ) + m.assert_called_once_with( + [(Keys.KEY, "value"), (Keys.KEY_1, "value1")], _conn=None, ttl=10 + ) - @pytest.mark.asyncio async def test_multi_set_ttl_overriden(self, base_cache): base_cache.ttl = 10 - base_cache._multi_set = CoroutineMock() - - await base_cache.multi_set([[pytest.KEY, "value"], [pytest.KEY_1, "value1"]], ttl=20) + with patch.object(base_cache, "_multi_set", autospec=True) as m: + await base_cache.multi_set([[Keys.KEY, "value"], [Keys.KEY_1, "value1"]], ttl=20) - base_cache._multi_set.assert_called_once_with( - [(pytest.KEY, "value"), (pytest.KEY_1, "value1")], _conn=None, ttl=20 - ) + m.assert_called_once_with( + [(Keys.KEY, "value"), (Keys.KEY_1, "value1")], _conn=None, ttl=20 + ) - @pytest.mark.asyncio async def test_multi_set_ttl_none(self, base_cache): base_cache.ttl = 10 - base_cache._multi_set = CoroutineMock() + with patch.object(base_cache, "_multi_set", autospec=True) as m: + await base_cache.multi_set([[Keys.KEY, "value"], [Keys.KEY_1, "value1"]], ttl=None) - await base_cache.multi_set([[pytest.KEY, "value"], [pytest.KEY_1, "value1"]], ttl=None) - - base_cache._multi_set.assert_called_once_with( - [(pytest.KEY, "value"), (pytest.KEY_1, "value1")], _conn=None, ttl=None - ) + m.assert_called_once_with( + [(Keys.KEY, "value"), (Keys.KEY_1, "value1")], _conn=None, ttl=None + ) class TestCache: @@ -371,232 +430,205 @@ class TestCache: async def asleep(self, *args, **kwargs): await asyncio.sleep(0.005) - @pytest.mark.asyncio - async def test_get(self, mock_cache): - await mock_cache.get(pytest.KEY) + async def test_get(self, mock_base_cache): + await mock_base_cache.get(Keys.KEY) - mock_cache._get.assert_called_with( - mock_cache._build_key(pytest.KEY), encoding=ANY, _conn=ANY + mock_base_cache._get.assert_called_with( + mock_base_cache.build_key(Keys.KEY), encoding=ANY, _conn=ANY ) - assert mock_cache.plugins[0].pre_get.call_count == 1 - assert mock_cache.plugins[0].post_get.call_count == 1 + assert mock_base_cache.plugins[0].pre_get.call_count == 1 + assert mock_base_cache.plugins[0].post_get.call_count == 1 - @pytest.mark.asyncio - async def test_get_timeouts(self, mock_cache): - mock_cache._get = self.asleep + async def test_get_timeouts(self, mock_base_cache): + mock_base_cache._get = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.get(pytest.KEY) + await mock_base_cache.get(Keys.KEY) - @pytest.mark.asyncio - async def test_get_default(self, mock_cache): - mock_cache._serializer.loads.return_value = None + async def test_get_default(self, mock_base_cache): + mock_base_cache._serializer.loads.return_value = None - assert await mock_cache.get(pytest.KEY, default=1) == 1 + assert await mock_base_cache.get(Keys.KEY, default=1) == 1 - @pytest.mark.asyncio - async def test_get_negative_default(self, mock_cache): - mock_cache._serializer.loads.return_value = False + async def test_get_negative_default(self, mock_base_cache): + mock_base_cache._serializer.loads.return_value = False - assert await mock_cache.get(pytest.KEY) is False + assert await mock_base_cache.get(Keys.KEY) is False - @pytest.mark.asyncio - async def test_set(self, mock_cache): - await mock_cache.set(pytest.KEY, "value", ttl=2) + async def test_set(self, mock_base_cache): + await mock_base_cache.set(Keys.KEY, "value", ttl=2) - mock_cache._set.assert_called_with( - mock_cache._build_key(pytest.KEY), ANY, ttl=2, _cas_token=None, _conn=ANY + mock_base_cache._set.assert_called_with( + mock_base_cache.build_key(Keys.KEY), ANY, ttl=2, _cas_token=None, _conn=ANY ) - assert mock_cache.plugins[0].pre_set.call_count == 1 - assert mock_cache.plugins[0].post_set.call_count == 1 + assert mock_base_cache.plugins[0].pre_set.call_count == 1 + assert mock_base_cache.plugins[0].post_set.call_count == 1 - @pytest.mark.asyncio - async def test_set_timeouts(self, mock_cache): - mock_cache._set = self.asleep + async def test_set_timeouts(self, mock_base_cache): + mock_base_cache._set = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.set(pytest.KEY, "value") + await mock_base_cache.set(Keys.KEY, "value") - @pytest.mark.asyncio - async def test_add(self, mock_cache): - mock_cache._exists = CoroutineMock(return_value=False) - await mock_cache.add(pytest.KEY, "value", ttl=2) + async def test_add(self, mock_base_cache): + mock_base_cache._exists = AsyncMock(return_value=False) + await mock_base_cache.add(Keys.KEY, "value", ttl=2) - mock_cache._add.assert_called_with(mock_cache._build_key(pytest.KEY), ANY, ttl=2, _conn=ANY) - assert mock_cache.plugins[0].pre_add.call_count == 1 - assert mock_cache.plugins[0].post_add.call_count == 1 + key = mock_base_cache.build_key(Keys.KEY) + mock_base_cache._add.assert_called_with(key, ANY, ttl=2, _conn=ANY) + assert mock_base_cache.plugins[0].pre_add.call_count == 1 + assert mock_base_cache.plugins[0].post_add.call_count == 1 - @pytest.mark.asyncio - async def test_add_timeouts(self, mock_cache): - mock_cache._add = self.asleep + async def test_add_timeouts(self, mock_base_cache): + mock_base_cache._add = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.add(pytest.KEY, "value") + await mock_base_cache.add(Keys.KEY, "value") - @pytest.mark.asyncio - async def test_mget(self, mock_cache): - await mock_cache.multi_get([pytest.KEY, pytest.KEY_1]) + async def test_mget(self, mock_base_cache): + await mock_base_cache.multi_get([Keys.KEY, Keys.KEY_1]) - mock_cache._multi_get.assert_called_with( - [mock_cache._build_key(pytest.KEY), mock_cache._build_key(pytest.KEY_1)], + mock_base_cache._multi_get.assert_called_with( + [mock_base_cache.build_key(Keys.KEY), mock_base_cache.build_key(Keys.KEY_1)], encoding=ANY, _conn=ANY, ) - assert mock_cache.plugins[0].pre_multi_get.call_count == 1 - assert mock_cache.plugins[0].post_multi_get.call_count == 1 + assert mock_base_cache.plugins[0].pre_multi_get.call_count == 1 + assert mock_base_cache.plugins[0].post_multi_get.call_count == 1 - @pytest.mark.asyncio - async def test_mget_timeouts(self, mock_cache): - mock_cache._multi_get = self.asleep + async def test_mget_timeouts(self, mock_base_cache): + mock_base_cache._multi_get = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.multi_get(pytest.KEY, "value") + await mock_base_cache.multi_get(Keys.KEY, "value") - @pytest.mark.asyncio - async def test_mset(self, mock_cache): - await mock_cache.multi_set([[pytest.KEY, "value"], [pytest.KEY_1, "value1"]], ttl=2) + async def test_mset(self, mock_base_cache): + await mock_base_cache.multi_set([[Keys.KEY, "value"], [Keys.KEY_1, "value1"]], ttl=2) - mock_cache._multi_set.assert_called_with( - [(mock_cache._build_key(pytest.KEY), ANY), (mock_cache._build_key(pytest.KEY_1), ANY)], - ttl=2, - _conn=ANY, - ) - assert mock_cache.plugins[0].pre_multi_set.call_count == 1 - assert mock_cache.plugins[0].post_multi_set.call_count == 1 + key = mock_base_cache.build_key(Keys.KEY) + key1 = mock_base_cache.build_key(Keys.KEY_1) + mock_base_cache._multi_set.assert_called_with( + [(key, ANY), (key1, ANY)], ttl=2, _conn=ANY) + assert mock_base_cache.plugins[0].pre_multi_set.call_count == 1 + assert mock_base_cache.plugins[0].post_multi_set.call_count == 1 - @pytest.mark.asyncio - async def test_mset_timeouts(self, mock_cache): - mock_cache._multi_set = self.asleep + async def test_mset_timeouts(self, mock_base_cache): + mock_base_cache._multi_set = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.multi_set([[pytest.KEY, "value"], [pytest.KEY_1, "value1"]]) + await mock_base_cache.multi_set([[Keys.KEY, "value"], [Keys.KEY_1, "value1"]]) - @pytest.mark.asyncio - async def test_exists(self, mock_cache): - await mock_cache.exists(pytest.KEY) + async def test_exists(self, mock_base_cache): + await mock_base_cache.exists(Keys.KEY) - mock_cache._exists.assert_called_with(mock_cache._build_key(pytest.KEY), _conn=ANY) - assert mock_cache.plugins[0].pre_exists.call_count == 1 - assert mock_cache.plugins[0].post_exists.call_count == 1 + mock_base_cache._exists.assert_called_with(mock_base_cache.build_key(Keys.KEY), _conn=ANY) + assert mock_base_cache.plugins[0].pre_exists.call_count == 1 + assert mock_base_cache.plugins[0].post_exists.call_count == 1 - @pytest.mark.asyncio - async def test_exists_timeouts(self, mock_cache): - mock_cache._exists = self.asleep + async def test_exists_timeouts(self, mock_base_cache): + mock_base_cache._exists = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.exists(pytest.KEY) + await mock_base_cache.exists(Keys.KEY) - @pytest.mark.asyncio - async def test_increment(self, mock_cache): - await mock_cache.increment(pytest.KEY, 2) + async def test_increment(self, mock_base_cache): + await mock_base_cache.increment(Keys.KEY, 2) - mock_cache._increment.assert_called_with(mock_cache._build_key(pytest.KEY), 2, _conn=ANY) - assert mock_cache.plugins[0].pre_increment.call_count == 1 - assert mock_cache.plugins[0].post_increment.call_count == 1 + key = mock_base_cache.build_key(Keys.KEY) + mock_base_cache._increment.assert_called_with(key, 2, _conn=ANY) + assert mock_base_cache.plugins[0].pre_increment.call_count == 1 + assert mock_base_cache.plugins[0].post_increment.call_count == 1 - @pytest.mark.asyncio - async def test_increment_timeouts(self, mock_cache): - mock_cache._increment = self.asleep + async def test_increment_timeouts(self, mock_base_cache): + mock_base_cache._increment = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.increment(pytest.KEY) + await mock_base_cache.increment(Keys.KEY) - @pytest.mark.asyncio - async def test_delete(self, mock_cache): - await mock_cache.delete(pytest.KEY) + async def test_delete(self, mock_base_cache): + await mock_base_cache.delete(Keys.KEY) - mock_cache._delete.assert_called_with(mock_cache._build_key(pytest.KEY), _conn=ANY) - assert mock_cache.plugins[0].pre_delete.call_count == 1 - assert mock_cache.plugins[0].post_delete.call_count == 1 + mock_base_cache._delete.assert_called_with(mock_base_cache.build_key(Keys.KEY), _conn=ANY) + assert mock_base_cache.plugins[0].pre_delete.call_count == 1 + assert mock_base_cache.plugins[0].post_delete.call_count == 1 - @pytest.mark.asyncio - async def test_delete_timeouts(self, mock_cache): - mock_cache._delete = self.asleep + async def test_delete_timeouts(self, mock_base_cache): + mock_base_cache._delete = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.delete(pytest.KEY) + await mock_base_cache.delete(Keys.KEY) - @pytest.mark.asyncio - async def test_expire(self, mock_cache): - await mock_cache.expire(pytest.KEY, 1) - mock_cache._expire.assert_called_with(mock_cache._build_key(pytest.KEY), 1, _conn=ANY) - assert mock_cache.plugins[0].pre_expire.call_count == 1 - assert mock_cache.plugins[0].post_expire.call_count == 1 + async def test_expire(self, mock_base_cache): + await mock_base_cache.expire(Keys.KEY, 1) + key = mock_base_cache.build_key(Keys.KEY) + mock_base_cache._expire.assert_called_with(key, 1, _conn=ANY) + assert mock_base_cache.plugins[0].pre_expire.call_count == 1 + assert mock_base_cache.plugins[0].post_expire.call_count == 1 - @pytest.mark.asyncio - async def test_expire_timeouts(self, mock_cache): - mock_cache._expire = self.asleep + async def test_expire_timeouts(self, mock_base_cache): + mock_base_cache._expire = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.expire(pytest.KEY, 0) + await mock_base_cache.expire(Keys.KEY, 0) - @pytest.mark.asyncio - async def test_clear(self, mock_cache): - await mock_cache.clear(pytest.KEY) - mock_cache._clear.assert_called_with(mock_cache._build_key(pytest.KEY), _conn=ANY) - assert mock_cache.plugins[0].pre_clear.call_count == 1 - assert mock_cache.plugins[0].post_clear.call_count == 1 + async def test_clear(self, mock_base_cache): + await mock_base_cache.clear(Keys.KEY) + mock_base_cache._clear.assert_called_with(mock_base_cache.build_key(Keys.KEY), _conn=ANY) + assert mock_base_cache.plugins[0].pre_clear.call_count == 1 + assert mock_base_cache.plugins[0].post_clear.call_count == 1 - @pytest.mark.asyncio - async def test_clear_timeouts(self, mock_cache): - mock_cache._clear = self.asleep + async def test_clear_timeouts(self, mock_base_cache): + mock_base_cache._clear = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.clear(pytest.KEY) + await mock_base_cache.clear(Keys.KEY) - @pytest.mark.asyncio - async def test_raw(self, mock_cache): - await mock_cache.raw("get", pytest.KEY) - mock_cache._raw.assert_called_with( - "get", mock_cache._build_key(pytest.KEY), encoding=ANY, _conn=ANY + async def test_raw(self, mock_base_cache): + await mock_base_cache.raw("get", Keys.KEY) + mock_base_cache._raw.assert_called_with( + "get", mock_base_cache.build_key(Keys.KEY), encoding=ANY, _conn=ANY ) - assert mock_cache.plugins[0].pre_raw.call_count == 1 - assert mock_cache.plugins[0].post_raw.call_count == 1 + assert mock_base_cache.plugins[0].pre_raw.call_count == 1 + assert mock_base_cache.plugins[0].post_raw.call_count == 1 - @pytest.mark.asyncio - async def test_raw_timeouts(self, mock_cache): - mock_cache._raw = self.asleep + async def test_raw_timeouts(self, mock_base_cache): + mock_base_cache._raw = self.asleep with pytest.raises(asyncio.TimeoutError): - await mock_cache.raw("clear") + await mock_base_cache.raw("clear") - @pytest.mark.asyncio - async def test_close(self, mock_cache): - await mock_cache.close() - assert mock_cache._close.call_count == 1 + async def test_close(self, mock_base_cache): + await mock_base_cache.close() + assert mock_base_cache._close.call_count == 1 - @pytest.mark.asyncio - async def test_get_connection(self, mock_cache): - async with mock_cache.get_connection(): + async def test_get_connection(self, mock_base_cache): + async with mock_base_cache.get_connection(): pass - assert mock_cache.acquire_conn.call_count == 1 - assert mock_cache.release_conn.call_count == 1 + assert mock_base_cache.acquire_conn.call_count == 1 + assert mock_base_cache.release_conn.call_count == 1 @pytest.fixture -def conn(mock_cache): - yield _Conn(mock_cache) +def conn(mock_base_cache): + yield _Conn(mock_base_cache) class TestConn: - def test_conn(self, conn, mock_cache): - assert conn._cache == mock_cache + def test_conn(self, conn, mock_base_cache): + assert conn._cache == mock_base_cache - def test_conn_getattr(self, conn, mock_cache): - assert conn.timeout == mock_cache.timeout - assert conn.namespace == conn.namespace - assert conn.serializer is mock_cache.serializer + def test_conn_getattr(self, conn, mock_base_cache): + assert conn.timeout == mock_base_cache.timeout + assert conn.namespace == mock_base_cache.namespace + assert conn.serializer is mock_base_cache.serializer - @pytest.mark.asyncio async def test_conn_context_manager(self, conn): async with conn: assert conn._cache.acquire_conn.call_count == 1 conn._cache.release_conn.assert_called_with(conn._cache.acquire_conn.return_value) - @pytest.mark.asyncio async def test_inject_conn(self, conn): conn._conn = "connection" - conn._cache.dummy = CoroutineMock() - + conn._cache.dummy = AsyncMock(spec_set=()) await _Conn._inject_conn("dummy")(conn, "a", b="b") conn._cache.dummy.assert_called_with("a", _conn=conn._conn, b="b") diff --git a/tests/ut/test_decorators.py b/tests/ut/test_decorators.py index 18c697056..cfa81e1b3 100644 --- a/tests/ut/test_decorators.py +++ b/tests/ut/test_decorators.py @@ -1,15 +1,17 @@ import asyncio -import sys -import pytest -import random import inspect +import random +import sys +from unittest.mock import ANY, create_autospec, patch -from asynctest import MagicMock, CoroutineMock, ANY, patch +import pytest -from aiocache.base import BaseCache, SENTINEL -from aiocache import cached, cached_stampede, multi_cached, SimpleMemoryCache -from aiocache.lock import RedLock +from aiocache import cached, cached_stampede, multi_cached +from aiocache.backends.memory import SimpleMemoryCache +from aiocache.base import SENTINEL from aiocache.decorators import _get_args_dict +from aiocache.lock import RedLock +from ..utils import AbstractBaseCache async def stub(*args, value=None, seconds=0, **kwargs): @@ -21,8 +23,8 @@ async def stub(*args, value=None, seconds=0, **kwargs): class TestCached: @pytest.fixture - def decorator(self, mocker, mock_cache): - with patch("aiocache.decorators._get_cache", return_value=mock_cache): + def decorator(self, mock_cache): + with patch("aiocache.decorators._get_cache", autospec=True, return_value=mock_cache): yield cached() @pytest.fixture @@ -38,33 +40,33 @@ def spy_stub(self, mocker): def test_init(self): c = cached( ttl=1, - key="key", - key_builder="fn", + key_builder=lambda *args, **kw: "key", cache=SimpleMemoryCache, plugins=None, alias=None, noself=False, namespace="test", + unused_kwarg="unused", ) assert c.ttl == 1 - assert c.key == "key" - assert c.key_builder == "fn" + assert c.key_builder() == "key" assert c.cache is None assert c._cache == SimpleMemoryCache assert c._serializer is None - assert c._kwargs == {"namespace": "test"} + assert c._namespace == "test" + assert c._kwargs == {"unused_kwarg": "unused"} def test_fails_at_instantiation(self): with pytest.raises(TypeError): @cached(wrong_param=1) - async def fn(n): - return n + async def fn() -> None: + """Dummy function.""" def test_alias_takes_precedence(self, mock_cache): with patch( - "aiocache.decorators.caches.get", MagicMock(return_value=mock_cache) + "aiocache.decorators.caches.get", autospec=True, return_value=mock_cache ) as mock_get: c = cached(alias="default", cache=SimpleMemoryCache, namespace="test") c(stub) @@ -73,8 +75,7 @@ def test_alias_takes_precedence(self, mock_cache): assert c.cache is mock_cache def test_get_cache_key_with_key(self, decorator): - decorator.key = "key" - decorator.key_builder = "fn" + decorator.key_builder = lambda *args, **kw: "key" assert decorator.get_cache_key(stub, (1, 2), {"a": 1, "b": 2}) == "key" def test_get_cache_key_without_key_and_attr(self, decorator): @@ -94,9 +95,8 @@ def test_get_cache_key_with_key_builder(self, decorator): decorator.key_builder = lambda *args, **kwargs: kwargs["market"].upper() assert decorator.get_cache_key(stub, (), {"market": "es"}) == "ES" - @pytest.mark.asyncio async def test_calls_get_and_returns(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=1) + decorator.cache.get.return_value = 1 await decorator_call() @@ -104,7 +104,6 @@ async def test_calls_get_and_returns(self, decorator, decorator_call): assert decorator.cache.set.call_count == 0 assert stub.call_count == 0 - @pytest.mark.asyncio async def test_cache_read_disabled(self, decorator, decorator_call): await decorator_call(cache_read=False) @@ -112,9 +111,8 @@ async def test_cache_read_disabled(self, decorator, decorator_call): assert decorator.cache.set.call_count == 1 assert stub.call_count == 1 - @pytest.mark.asyncio async def test_cache_write_disabled(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=None) + decorator.cache.get.return_value = None await decorator_call(cache_write=False) @@ -122,34 +120,29 @@ async def test_cache_write_disabled(self, decorator, decorator_call): assert decorator.cache.set.call_count == 0 assert stub.call_count == 1 - @pytest.mark.asyncio async def test_disable_params_not_propagated(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=None) + decorator.cache.get.return_value = None await decorator_call(cache_read=False, cache_write=False) stub.assert_called_once_with() - @pytest.mark.asyncio async def test_get_from_cache_returns(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=1) + decorator.cache.get.return_value = 1 assert await decorator.get_from_cache("key") == 1 - @pytest.mark.asyncio async def test_get_from_cache_exception(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(side_effect=Exception) + decorator.cache.get.side_effect = Exception assert await decorator.get_from_cache("key") is None - @pytest.mark.asyncio async def test_get_from_cache_none(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=None) + decorator.cache.get.return_value = None assert await decorator.get_from_cache("key") is None - @pytest.mark.asyncio async def test_calls_fn_set_when_get_none(self, mocker, decorator, decorator_call): mocker.spy(decorator, "get_from_cache") mocker.spy(decorator, "set_in_cache") - decorator.cache.get = CoroutineMock(return_value=None) + decorator.cache.get.return_value = None await decorator_call(value="value") @@ -157,52 +150,43 @@ async def test_calls_fn_set_when_get_none(self, mocker, decorator, decorator_cal decorator.set_in_cache.assert_called_with("stub()[('value', 'value')]", "value") stub.assert_called_once_with(value="value") - @pytest.mark.asyncio - async def test_calls_fn_raises_exception(self, mocker, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=None) - stub.side_effect = Exception() - with pytest.raises(Exception): + async def test_calls_fn_raises_exception(self, decorator, decorator_call): + decorator.cache.get.return_value = None + stub.side_effect = Exception("foo") + with pytest.raises(Exception, match="foo"): assert await decorator_call() - @pytest.mark.asyncio - async def test_cache_write_waits_for_future(self, mocker, decorator, decorator_call): - decorator.get_from_cache = CoroutineMock(return_value=None) - decorator.set_in_cache = CoroutineMock() - await decorator_call() + async def test_cache_write_waits_for_future(self, decorator, decorator_call): + with patch.object(decorator, "get_from_cache", autospec=True, return_value=None) as m: + await decorator_call() - decorator.set_in_cache.assert_awaited() + m.assert_awaited() - @pytest.mark.asyncio async def test_cache_write_doesnt_wait_for_future(self, mocker, decorator, decorator_call): - decorator.get_from_cache = CoroutineMock(return_value=None) - decorator.set_in_cache = CoroutineMock() - - with patch("aiocache.decorators.asyncio.ensure_future"): - await decorator_call(aiocache_wait_for_write=False, value="value") + mocker.spy(decorator, "set_in_cache") + with patch.object(decorator, "get_from_cache", autospec=True, return_value=None): + with patch("aiocache.decorators.asyncio.ensure_future", autospec=True): + await decorator_call(aiocache_wait_for_write=False, value="value") decorator.set_in_cache.assert_not_awaited() decorator.set_in_cache.assert_called_once_with("stub()[('value', 'value')]", "value") - @pytest.mark.asyncio async def test_set_calls_set(self, decorator, decorator_call): await decorator.set_in_cache("key", "value") decorator.cache.set.assert_called_with("key", "value", ttl=SENTINEL) - @pytest.mark.asyncio async def test_set_calls_set_ttl(self, decorator, decorator_call): decorator.ttl = 10 await decorator.set_in_cache("key", "value") decorator.cache.set.assert_called_with("key", "value", ttl=decorator.ttl) - @pytest.mark.asyncio async def test_set_catches_exception(self, decorator, decorator_call): - decorator.cache.set = CoroutineMock(side_effect=Exception) + decorator.cache.set.side_effect = Exception assert await decorator.set_in_cache("key", "value") is None - @pytest.mark.asyncio async def test_decorate(self, mock_cache): - mock_cache.get = CoroutineMock(return_value=None) - with patch("aiocache.decorators._get_cache", return_value=mock_cache): + mock_cache.get.return_value = None + with patch("aiocache.decorators._get_cache", autospec=True, return_value=mock_cache): @cached() async def fn(n): @@ -212,27 +196,25 @@ async def fn(n): assert await fn(2) == 2 assert fn.cache == mock_cache - @pytest.mark.asyncio async def test_keeps_signature(self, mock_cache): - with patch("aiocache.decorators._get_cache", return_value=mock_cache): + with patch("aiocache.decorators._get_cache", autospec=True, return_value=mock_cache): @cached() async def what(self, a, b): - return "1" + """Dummy function.""" assert what.__name__ == "what" assert str(inspect.signature(what)) == "(self, a, b)" assert inspect.getfullargspec(what.__wrapped__).args == ["self", "a", "b"] - @pytest.mark.asyncio async def test_reuses_cache_instance(self): - with patch("aiocache.decorators._get_cache") as get_c: - cache = MagicMock(spec=BaseCache) + with patch("aiocache.decorators._get_cache", autospec=True) as get_c: + cache = create_autospec(AbstractBaseCache, instance=True) get_c.side_effect = [cache, None] @cached() async def what(): - pass + """Dummy function.""" await what() await what() @@ -240,23 +222,22 @@ async def what(): assert get_c.call_count == 1 assert cache.get.call_count == 2 - @pytest.mark.asyncio async def test_cache_per_function(self): @cached() async def foo(): - pass + """First function.""" @cached() async def bar(): - pass + """Second function.""" assert foo.cache != bar.cache class TestCachedStampede: @pytest.fixture - def decorator(self, mocker, mock_cache): - with patch("aiocache.decorators._get_cache", return_value=mock_cache): + def decorator(self, mock_cache): + with patch("aiocache.decorators._get_cache", autospec=True, return_value=mock_cache): yield cached_stampede() @pytest.fixture @@ -275,27 +256,26 @@ def test_init(self): c = cached_stampede( lease=3, ttl=1, - key="key", - key_builder="fn", + key_builder=lambda *args, **kw: "key", cache=SimpleMemoryCache, plugins=None, alias=None, noself=False, namespace="test", + unused_kwarg="unused", ) assert c.ttl == 1 - assert c.key == "key" - assert c.key_builder == "fn" + assert c.key_builder() == "key" assert c.cache is None assert c._cache == SimpleMemoryCache assert c._serializer is None assert c.lease == 3 - assert c._kwargs == {"namespace": "test"} + assert c._namespace == "test" + assert c._kwargs == {"unused_kwarg": "unused"} - @pytest.mark.asyncio async def test_calls_get_and_returns(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=1) + decorator.cache.get.return_value = 1 await decorator_call() @@ -303,19 +283,17 @@ async def test_calls_get_and_returns(self, decorator, decorator_call): assert decorator.cache.set.call_count == 0 assert stub.call_count == 0 - @pytest.mark.asyncio - async def test_calls_fn_raises_exception(self, mocker, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=None) - stub.side_effect = Exception() - with pytest.raises(Exception): + async def test_calls_fn_raises_exception(self, decorator, decorator_call): + decorator.cache.get.return_value = None + stub.side_effect = Exception("foo") + with pytest.raises(Exception, match="foo"): assert await decorator_call() - @pytest.mark.asyncio async def test_calls_redlock(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(return_value=None) - lock = MagicMock(spec=RedLock) + decorator.cache.get.return_value = None + lock = create_autospec(RedLock, instance=True) - with patch("aiocache.decorators.RedLock", return_value=lock): + with patch("aiocache.decorators.RedLock", autospec=True, return_value=lock): await decorator_call(value="value") assert decorator.cache.get.call_count == 2 @@ -326,14 +304,13 @@ async def test_calls_redlock(self, decorator, decorator_call): ) stub.assert_called_once_with(value="value") - @pytest.mark.asyncio async def test_calls_locked_client(self, decorator, decorator_call): - decorator.cache.get = CoroutineMock(side_effect=[None, None, None, "value"]) - decorator.cache._add = CoroutineMock(side_effect=[True, ValueError]) - lock1 = MagicMock(spec=RedLock) - lock2 = MagicMock(spec=RedLock) + decorator.cache.get.side_effect = [None, None, None, "value"] + decorator.cache._add.side_effect = [True, ValueError] + lock1 = create_autospec(RedLock, instance=True) + lock2 = create_autospec(RedLock, instance=True) - with patch("aiocache.decorators.RedLock", side_effect=[lock1, lock2]): + with patch("aiocache.decorators.RedLock", autospec=True, side_effect=[lock1, lock2]): await asyncio.gather(decorator_call(value="value"), decorator_call(value="value")) assert decorator.cache.get.call_count == 4 @@ -354,8 +331,8 @@ async def stub_dict(*args, keys=None, **kwargs): class TestMultiCached: @pytest.fixture - def decorator(self, mocker, mock_cache): - with patch("aiocache.decorators._get_cache", return_value=mock_cache): + def decorator(self, mock_cache): + with patch("aiocache.decorators._get_cache", autospec=True, return_value=mock_cache): yield multi_cached(keys_from_attr="keys") @pytest.fixture @@ -378,26 +355,31 @@ def test_init(self): plugins=None, alias=None, namespace="test", + unused_kwarg="unused", ) + def f(): + """Dummy function. Not called.""" + assert mc.ttl == 1 - assert mc.key_builder("key", lambda x: x) == "key" + assert mc.key_builder("key", f) == "key" assert mc.keys_from_attr == "keys" assert mc.cache is None assert mc._cache == SimpleMemoryCache assert mc._serializer is None - assert mc._kwargs == {"namespace": "test"} + assert mc._namespace == "test" + assert mc._kwargs == {"unused_kwarg": "unused"} def test_fails_at_instantiation(self): with pytest.raises(TypeError): @multi_cached(wrong_param=1) - async def fn(n): - return n + async def fn() -> None: + """Dummy function.""" def test_alias_takes_precedence(self, mock_cache): with patch( - "aiocache.decorators.caches.get", MagicMock(return_value=mock_cache) + "aiocache.decorators.caches.get", autospec=True, return_value=mock_cache ) as mock_get: mc = multi_cached( keys_from_attr="keys", alias="default", cache=SimpleMemoryCache, namespace="test" @@ -408,80 +390,74 @@ def test_alias_takes_precedence(self, mock_cache): assert mc.cache is mock_cache def test_get_cache_keys(self, decorator): - assert decorator.get_cache_keys(stub_dict, (), {"keys": ["a", "b"]}) == (["a", "b"], [], -1) + keys = decorator.get_cache_keys(stub_dict, (), {"keys": ["a", "b"]}) + assert keys == (["a", "b"], ["a", "b"], [], -1) def test_get_cache_keys_empty_list(self, decorator): - assert decorator.get_cache_keys(stub_dict, (), {"keys": []}) == ([], [], -1) + assert decorator.get_cache_keys(stub_dict, (), {"keys": []}) == ([], [], [], -1) def test_get_cache_keys_missing_kwarg(self, decorator): - assert decorator.get_cache_keys(stub_dict, (), {}) == ([], [], -1) + assert decorator.get_cache_keys(stub_dict, (), {}) == ([], [], [], -1) def test_get_cache_keys_arg_key_from_attr(self, decorator): def fake(keys, a=1, b=2): - pass + """Dummy function.""" - assert decorator.get_cache_keys(fake, (["a"]), {}) == (["a"], [["a"]], 0) + assert decorator.get_cache_keys(fake, (["a"],), {}) == (["a"], ["a"], [["a"]], 0) def test_get_cache_keys_with_none(self, decorator): - assert decorator.get_cache_keys(stub_dict, (), {"keys": None}) == ([], [], -1) + assert decorator.get_cache_keys(stub_dict, (), {"keys": None}) == ([], [], [], -1) def test_get_cache_keys_with_key_builder(self, decorator): decorator.key_builder = lambda key, *args, **kwargs: kwargs["market"] + "_" + key.upper() assert decorator.get_cache_keys(stub_dict, (), {"keys": ["a", "b"], "market": "ES"}) == ( + ["a", "b"], ["ES_A", "ES_B"], [], -1, ) - @pytest.mark.asyncio async def test_get_from_cache(self, decorator, decorator_call): - decorator.cache.multi_get = CoroutineMock(return_value=[1, 2, 3]) + decorator.cache.multi_get.return_value = [1, 2, 3] assert await decorator.get_from_cache("a", "b", "c") == [1, 2, 3] decorator.cache.multi_get.assert_called_with(("a", "b", "c")) - @pytest.mark.asyncio async def test_get_from_cache_no_keys(self, decorator, decorator_call): assert await decorator.get_from_cache() == [] assert decorator.cache.multi_get.call_count == 0 - @pytest.mark.asyncio async def test_get_from_cache_exception(self, decorator, decorator_call): - decorator.cache.multi_get = CoroutineMock(side_effect=Exception) + decorator.cache.multi_get.side_effect = Exception assert await decorator.get_from_cache("a", "b", "c") == [None, None, None] decorator.cache.multi_get.assert_called_with(("a", "b", "c")) - @pytest.mark.asyncio async def test_get_from_cache_conn(self, decorator, decorator_call): - decorator._conn._conn = MagicMock() - decorator.cache.multi_get = CoroutineMock(return_value=[1, 2, 3]) + decorator.cache.multi_get.return_value = [1, 2, 3] assert await decorator.get_from_cache("a", "b", "c") == [1, 2, 3] decorator.cache.multi_get.assert_called_with(("a", "b", "c")) - @pytest.mark.asyncio async def test_calls_no_keys(self, decorator, decorator_call): await decorator_call(keys=[]) assert decorator.cache.multi_get.call_count == 0 assert stub_dict.call_count == 1 - @pytest.mark.asyncio async def test_returns_from_multi_set(self, mocker, decorator, decorator_call): mocker.spy(decorator, "get_from_cache") mocker.spy(decorator, "set_in_cache") - decorator.cache.multi_get = CoroutineMock(return_value=[1, 2]) + decorator.cache.multi_get.return_value = [1, 2] assert await decorator_call(1, keys=["a", "b"]) == {"a": 1, "b": 2} decorator.get_from_cache.assert_called_once_with("a", "b") assert decorator.set_in_cache.call_count == 0 assert stub_dict.call_count == 0 - @pytest.mark.asyncio async def test_calls_fn_multi_set_when_multi_get_none(self, mocker, decorator, decorator_call): mocker.spy(decorator, "get_from_cache") mocker.spy(decorator, "set_in_cache") - decorator.cache.multi_get = CoroutineMock(return_value=[None, None]) + decorator.cache.multi_get.return_value = [None, None] ret = await decorator_call(1, keys=["a", "b"], value="value") @@ -489,43 +465,38 @@ async def test_calls_fn_multi_set_when_multi_get_none(self, mocker, decorator, d decorator.set_in_cache.assert_called_with(ret, stub_dict, ANY, ANY) stub_dict.assert_called_once_with(1, keys=["a", "b"], value="value") - @pytest.mark.asyncio - async def test_cache_write_waits_for_future(self, decorator, decorator_call): - decorator.get_from_cache = CoroutineMock(return_value=[None, None]) - decorator.set_in_cache = CoroutineMock() - await decorator_call(1, keys=["a", "b"], value="value") - - decorator.set_in_cache.assert_awaited() + async def test_cache_write_waits_for_future(self, mocker, decorator, decorator_call): + mocker.spy(decorator, "set_in_cache") + with patch.object(decorator, "get_from_cache", autospec=True, return_value=[None, None]): + await decorator_call(1, keys=["a", "b"], value="value") - @pytest.mark.asyncio - async def test_cache_write_doesnt_wait_for_future(self, decorator, decorator_call): - decorator.get_from_cache = CoroutineMock(return_value=[None, None]) - decorator.set_in_cache = CoroutineMock() + decorator.set_in_cache.assert_awaited() - with patch("aiocache.decorators.asyncio.ensure_future"): - await decorator_call(1, keys=["a", "b"], value="value", aiocache_wait_for_write=False) + async def test_cache_write_doesnt_wait_for_future(self, mocker, decorator, decorator_call): + mocker.spy(decorator, "set_in_cache") + with patch.object(decorator, "get_from_cache", autospec=True, return_value=[None, None]): + with patch("aiocache.decorators.asyncio.ensure_future", autospec=True): + await decorator_call(1, keys=["a", "b"], value="value", + aiocache_wait_for_write=False) decorator.set_in_cache.assert_not_awaited() decorator.set_in_cache.assert_called_once_with({"a": ANY, "b": ANY}, stub_dict, ANY, ANY) - @pytest.mark.asyncio async def test_calls_fn_with_only_missing_keys(self, mocker, decorator, decorator_call): mocker.spy(decorator, "set_in_cache") - decorator.cache.multi_get = CoroutineMock(return_value=[1, None]) + decorator.cache.multi_get.return_value = [1, None] assert await decorator_call(1, keys=["a", "b"], value="value") == {"a": ANY, "b": ANY} decorator.set_in_cache.assert_called_once_with({"a": ANY, "b": ANY}, stub_dict, ANY, ANY) stub_dict.assert_called_once_with(1, keys=["b"], value="value") - @pytest.mark.asyncio - async def test_calls_fn_raises_exception(self, mocker, decorator, decorator_call): - decorator.cache.multi_get = CoroutineMock(return_value=[None]) - stub_dict.side_effect = Exception() - with pytest.raises(Exception): + async def test_calls_fn_raises_exception(self, decorator, decorator_call): + decorator.cache.multi_get.return_value = [None] + stub_dict.side_effect = Exception("foo") + with pytest.raises(Exception, match="foo"): assert await decorator_call(keys=[]) - @pytest.mark.asyncio async def test_cache_read_disabled(self, decorator, decorator_call): await decorator_call(1, keys=["a", "b"], cache_read=False) @@ -533,9 +504,8 @@ async def test_cache_read_disabled(self, decorator, decorator_call): assert decorator.cache.multi_set.call_count == 1 assert stub_dict.call_count == 1 - @pytest.mark.asyncio async def test_cache_write_disabled(self, decorator, decorator_call): - decorator.cache.multi_get = CoroutineMock(return_value=[None, None]) + decorator.cache.multi_get.return_value = [None, None] await decorator_call(1, keys=["a", "b"], cache_write=False) @@ -543,15 +513,13 @@ async def test_cache_write_disabled(self, decorator, decorator_call): assert decorator.cache.multi_set.call_count == 0 assert stub_dict.call_count == 1 - @pytest.mark.asyncio async def test_disable_params_not_propagated(self, decorator, decorator_call): - decorator.cache.multi_get = CoroutineMock(return_value=[None, None]) + decorator.cache.multi_get.return_value = [None, None] await decorator_call(1, keys=["a", "b"], cache_read=False, cache_write=False) stub_dict.assert_called_once_with(1, keys=["a", "b"]) - @pytest.mark.asyncio async def test_set_in_cache(self, decorator, decorator_call): await decorator.set_in_cache({"a": 1, "b": 2}, stub_dict, (), {}) @@ -560,23 +528,20 @@ async def test_set_in_cache(self, decorator, decorator_call): assert ("b", 2) in call_args assert decorator.cache.multi_set.call_args[1]["ttl"] is SENTINEL - @pytest.mark.asyncio async def test_set_in_cache_with_ttl(self, decorator, decorator_call): decorator.ttl = 10 await decorator.set_in_cache({"a": 1, "b": 2}, stub_dict, (), {}) assert decorator.cache.multi_set.call_args[1]["ttl"] == decorator.ttl - @pytest.mark.asyncio async def test_set_in_cache_exception(self, decorator, decorator_call): - decorator.cache.multi_set = CoroutineMock(side_effect=Exception) + decorator.cache.multi_set.side_effect = Exception assert await decorator.set_in_cache({"a": 1, "b": 2}, stub_dict, (), {}) is None - @pytest.mark.asyncio async def test_decorate(self, mock_cache): - mock_cache.multi_get = CoroutineMock(return_value=[None]) - with patch("aiocache.decorators._get_cache", return_value=mock_cache): + mock_cache.multi_get.return_value = [None] + with patch("aiocache.decorators._get_cache", autospec=True, return_value=mock_cache): @multi_cached(keys_from_attr="keys") async def fn(keys=None): @@ -586,20 +551,18 @@ async def fn(keys=None): assert await fn(["test"]) == {"test": 1} assert fn.cache == mock_cache - @pytest.mark.asyncio async def test_keeps_signature(self): @multi_cached(keys_from_attr="keys") async def what(self, keys=None, what=1): - return "1" + """Dummy function.""" assert what.__name__ == "what" assert str(inspect.signature(what)) == "(self, keys=None, what=1)" assert inspect.getfullargspec(what.__wrapped__).args == ["self", "keys", "what"] - @pytest.mark.asyncio async def test_reuses_cache_instance(self): - with patch("aiocache.decorators._get_cache") as get_c: - cache = MagicMock(spec=BaseCache) + with patch("aiocache.decorators._get_cache", autospec=True) as get_c: + cache = create_autospec(AbstractBaseCache, instance=True) cache.multi_get.return_value = [None] get_c.side_effect = [cache, None] @@ -613,22 +576,31 @@ async def what(keys=None): assert get_c.call_count == 1 assert cache.multi_get.call_count == 2 - @pytest.mark.asyncio async def test_cache_per_function(self): @multi_cached("keys") async def foo(): - pass + """First function.""" @multi_cached("keys") async def bar(): - pass + """Second function.""" assert foo.cache != bar.cache + async def test_key_builder(self): + @multi_cached("keys", key_builder=lambda key, _, keys: key + 1) + async def f(keys=None): + return {k: k * 3 for k in keys} + + assert await f(keys=(1,)) == {1: 3} + cached_value = await f.cache.get(2) + assert cached_value == 3 + assert not await f.cache.exists(1) + def test_get_args_dict(): def fn(a, b, *args, keys=None, **kwargs): - pass + """Dummy function.""" args_dict = _get_args_dict(fn, ("a", "b", "c", "d"), {"what": "what"}) assert args_dict == {"a": "a", "b": "b", "keys": None, "what": "what"} diff --git a/tests/ut/test_factory.py b/tests/ut/test_factory.py index 69696198a..7b33b8b36 100644 --- a/tests/ut/test_factory.py +++ b/tests/ut/test_factory.py @@ -1,35 +1,49 @@ +from unittest.mock import Mock, patch + import pytest -from unittest.mock import patch, Mock -from aiocache import SimpleMemoryCache, RedisCache, MemcachedCache, caches, Cache, AIOCACHE_CACHES -from aiocache.factory import _class_from_string, _create_cache +from aiocache import AIOCACHE_CACHES, Cache, caches +from aiocache.backends.memory import SimpleMemoryCache from aiocache.exceptions import InvalidCacheType +from aiocache.factory import _class_from_string, _create_cache +from aiocache.plugins import HitMissRatioPlugin, TimingPlugin from aiocache.serializers import JsonSerializer, PickleSerializer -from aiocache.plugins import TimingPlugin, HitMissRatioPlugin -def test_class_from_string(): - assert _class_from_string("aiocache.RedisCache") == RedisCache +CACHE_NAMES = [Cache.MEMORY.NAME] +try: + from aiocache.backends.memcached import MemcachedCache +except ImportError: + MemcachedCache = None +else: + assert Cache.MEMCACHED is not None + CACHE_NAMES.append(Cache.MEMCACHED.NAME) -def test_create_simple_cache(): - redis = _create_cache(RedisCache, endpoint="127.0.0.10", port=6378) +try: + from aiocache.backends.redis import RedisCache +except ImportError: + RedisCache = None +else: + assert Cache.REDIS is not None + CACHE_NAMES.append(Cache.REDIS.NAME) - assert isinstance(redis, RedisCache) - assert redis.endpoint == "127.0.0.10" - assert redis.port == 6378 + +@pytest.mark.redis +def test_class_from_string(): + assert _class_from_string("aiocache.RedisCache") == RedisCache def test_create_cache_with_everything(): - redis = _create_cache( - RedisCache, + cache = _create_cache( + SimpleMemoryCache, serializer={"class": PickleSerializer, "encoding": "encoding"}, plugins=[{"class": "aiocache.plugins.TimingPlugin"}], ) - assert isinstance(redis.serializer, PickleSerializer) - assert redis.serializer.encoding == "encoding" - assert isinstance(redis.plugins[0], TimingPlugin) + assert isinstance(cache.serializer, PickleSerializer) + assert cache.serializer.encoding == "encoding" + assert isinstance(cache.plugins[0], TimingPlugin) class TestCache: @@ -38,10 +52,8 @@ def test_cache_types(self): assert Cache.REDIS == RedisCache assert Cache.MEMCACHED == MemcachedCache - @pytest.mark.parametrize( - "cache_type", [Cache.MEMORY.NAME, Cache.REDIS.NAME, Cache.MEMCACHED.NAME] - ) - def test_new(self, cache_type): + @pytest.mark.parametrize("cache_type", CACHE_NAMES) + async def test_new(self, cache_type): kwargs = {"a": 1, "b": 2} cache_class = Cache.get_scheme_class(cache_type) @@ -60,7 +72,7 @@ def test_new_invalid_cache_raises(self): list(AIOCACHE_CACHES.keys()) ) - @pytest.mark.parametrize("scheme", [Cache.MEMORY.NAME, Cache.REDIS.NAME, Cache.MEMCACHED.NAME]) + @pytest.mark.parametrize("scheme", CACHE_NAMES) def test_get_scheme_class(self, scheme): assert Cache.get_scheme_class(scheme) == AIOCACHE_CACHES[scheme] @@ -68,7 +80,7 @@ def test_get_scheme_class_invalid(self): with pytest.raises(InvalidCacheType): Cache.get_scheme_class("http") - @pytest.mark.parametrize("scheme", [Cache.MEMORY.NAME, Cache.REDIS.NAME, Cache.MEMCACHED.NAME]) + @pytest.mark.parametrize("scheme", CACHE_NAMES) def test_from_url_returns_cache_from_scheme(self, scheme): assert isinstance(Cache.from_url("{}://".format(scheme)), Cache.get_scheme_class(scheme)) @@ -76,38 +88,39 @@ def test_from_url_returns_cache_from_scheme(self, scheme): "url,expected_args", [ ("redis://", {}), - ("redis://localhost", {"endpoint": "localhost"}), - ("redis://localhost/", {"endpoint": "localhost"}), - ("redis://localhost:6379", {"endpoint": "localhost", "port": 6379}), + ("redis://localhost", {"host": "localhost"}), + ("redis://localhost/", {"host": "localhost"}), + ("redis://localhost:6379", {"host": "localhost", "port": 6379}), ( "redis://localhost/?arg1=arg1&arg2=arg2", - {"endpoint": "localhost", "arg1": "arg1", "arg2": "arg2"}, + {"host": "localhost", "arg1": "arg1", "arg2": "arg2"}, ), ( "redis://localhost:6379/?arg1=arg1&arg2=arg2", - {"endpoint": "localhost", "port": 6379, "arg1": "arg1", "arg2": "arg2"}, + {"host": "localhost", "port": 6379, "arg1": "arg1", "arg2": "arg2"}, ), ("redis:///?arg1=arg1", {"arg1": "arg1"}), ("redis:///?arg2=arg2", {"arg2": "arg2"}), ( "redis://:password@localhost:6379", - {"endpoint": "localhost", "password": "password", "port": 6379}, + {"host": "localhost", "password": "password", "port": 6379}, ), ( "redis://:password@localhost:6379?password=pass", - {"endpoint": "localhost", "password": "password", "port": 6379}, + {"host": "localhost", "password": "password", "port": 6379}, ), ], ) def test_from_url_calls_cache_with_args(self, url, expected_args): - with patch("aiocache.factory.Cache") as mock: + with patch("aiocache.factory.Cache", autospec=True) as mock: Cache.from_url(url) mock.assert_called_once_with(mock.get_scheme_class.return_value, **expected_args) def test_calls_parse_uri_path_from_cache(self): - with patch("aiocache.factory.Cache") as mock: - mock.get_scheme_class.return_value.parse_uri_path = Mock(return_value={"arg1": "arg1"}) + p_mock = Mock(spec_set=(), return_value={"arg1": "arg1"}) + with patch("aiocache.factory.Cache", autospec=True) as mock: + mock.get_scheme_class.return_value.parse_uri_path = p_mock Cache.from_url("redis:///") mock.get_scheme_class.return_value.parse_uri_path.assert_called_once_with("/") @@ -157,37 +170,30 @@ def test_reuse_instance(self): def test_create_not_reuse(self): assert caches.create("default") is not caches.create("default") + @pytest.mark.redis def test_create_extra_args(self): caches.set_config( { "default": { "cache": "aiocache.RedisCache", - "endpoint": "127.0.0.9", + "host": "127.0.0.9", "db": 10, "port": 6378, } } ) - cache = caches.create("default", namespace="whatever", endpoint="127.0.0.10", db=10) + cache = caches.create("default", namespace="whatever", host="127.0.0.10", db=10) assert cache.namespace == "whatever" - assert cache.endpoint == "127.0.0.10" - assert cache.db == 10 - - def test_create_deprecated(self): - with patch("aiocache.factory.warnings.warn") as mock: - caches.create(cache="aiocache.SimpleMemoryCache") - - mock.assert_called_once_with( - "Creating a cache with an explicit config is deprecated, use 'aiocache.Cache'", - DeprecationWarning, - ) + assert cache.client.connection_pool.connection_kwargs["host"] == "127.0.0.10" + assert cache.client.connection_pool.connection_kwargs["db"] == 10 + @pytest.mark.redis def test_retrieve_cache(self): caches.set_config( { "default": { "cache": "aiocache.RedisCache", - "endpoint": "127.0.0.10", + "host": "127.0.0.10", "port": 6378, "ttl": 10, "serializer": { @@ -204,19 +210,20 @@ def test_retrieve_cache(self): cache = caches.get("default") assert isinstance(cache, RedisCache) - assert cache.endpoint == "127.0.0.10" - assert cache.port == 6378 + assert cache.client.connection_pool.connection_kwargs["host"] == "127.0.0.10" + assert cache.client.connection_pool.connection_kwargs["port"] == 6378 assert cache.ttl == 10 assert isinstance(cache.serializer, PickleSerializer) assert cache.serializer.encoding == "encoding" assert len(cache.plugins) == 2 + @pytest.mark.redis def test_retrieve_cache_new_instance(self): caches.set_config( { "default": { "cache": "aiocache.RedisCache", - "endpoint": "127.0.0.10", + "host": "127.0.0.10", "port": 6378, "serializer": { "class": "aiocache.serializers.PickleSerializer", @@ -232,64 +239,19 @@ def test_retrieve_cache_new_instance(self): cache = caches.create("default") assert isinstance(cache, RedisCache) - assert cache.endpoint == "127.0.0.10" - assert cache.port == 6378 + assert cache.client.connection_pool.connection_kwargs["host"] == "127.0.0.10" + assert cache.client.connection_pool.connection_kwargs["port"] == 6378 assert isinstance(cache.serializer, PickleSerializer) assert cache.serializer.encoding == "encoding" assert len(cache.plugins) == 2 - def test_create_cache_str_no_alias(self): - cache = caches.create(cache="aiocache.RedisCache") - - assert isinstance(cache, RedisCache) - assert cache.endpoint == "127.0.0.1" - assert cache.port == 6379 - - def test_create_cache_class_no_alias(self): - cache = caches.create(cache=RedisCache) - - assert isinstance(cache, RedisCache) - assert cache.endpoint == "127.0.0.1" - assert cache.port == 6379 - - def test_create_cache_ensure_alias_or_cache(self): - with pytest.raises(TypeError): - caches.create() - - def test_alias_config_is_reusable(self): - caches.set_config( - { - "default": { - "cache": "aiocache.RedisCache", - "endpoint": "127.0.0.10", - "port": 6378, - "serializer": {"class": "aiocache.serializers.PickleSerializer"}, - "plugins": [ - {"class": "aiocache.plugins.HitMissRatioPlugin"}, - {"class": "aiocache.plugins.TimingPlugin"}, - ], - }, - "alt": {"cache": "aiocache.SimpleMemoryCache"}, - } - ) - - default = caches.create(**caches.get_alias_config("default")) - alt = caches.create(**caches.get_alias_config("alt")) - - assert isinstance(default, RedisCache) - assert default.endpoint == "127.0.0.10" - assert default.port == 6378 - assert isinstance(default.serializer, PickleSerializer) - assert len(default.plugins) == 2 - - assert isinstance(alt, SimpleMemoryCache) - + @pytest.mark.redis def test_multiple_caches(self): caches.set_config( { "default": { "cache": "aiocache.RedisCache", - "endpoint": "127.0.0.10", + "host": "127.0.0.10", "port": 6378, "serializer": {"class": "aiocache.serializers.PickleSerializer"}, "plugins": [ @@ -305,8 +267,8 @@ def test_multiple_caches(self): alt = caches.get("alt") assert isinstance(default, RedisCache) - assert default.endpoint == "127.0.0.10" - assert default.port == 6378 + assert default.client.connection_pool.connection_kwargs["host"] == "127.0.0.10" + assert default.client.connection_pool.connection_kwargs["port"] == 6378 assert isinstance(default.serializer, PickleSerializer) assert len(default.plugins) == 2 @@ -367,7 +329,7 @@ def test_set_config_no_default(self): { "no_default": { "cache": "aiocache.RedisCache", - "endpoint": "127.0.0.10", + "host": "127.0.0.10", "port": 6378, "serializer": {"class": "aiocache.serializers.PickleSerializer"}, "plugins": [ @@ -378,6 +340,7 @@ def test_set_config_no_default(self): } ) + @pytest.mark.redis def test_ensure_plugins_order(self): caches.set_config( { diff --git a/tests/ut/test_lock.py b/tests/ut/test_lock.py index 479499b19..a06ea5678 100644 --- a/tests/ut/test_lock.py +++ b/tests/ut/test_lock.py @@ -1,118 +1,112 @@ import asyncio +from unittest.mock import Mock, patch + import pytest -import asynctest -from aiocache.lock import RedLock, OptimisticLock, OptimisticLockError +from aiocache.lock import OptimisticLock, OptimisticLockError, RedLock +from ..utils import KEY_LOCK, Keys class TestRedLock: @pytest.fixture - def lock(self, mock_cache): + def lock(self, mock_base_cache): RedLock._EVENTS = {} - yield RedLock(mock_cache, pytest.KEY, 20) + yield RedLock(mock_base_cache, Keys.KEY, 20) - @pytest.mark.asyncio - async def test_acquire(self, mock_cache, lock): + async def test_acquire(self, mock_base_cache, lock): await lock._acquire() - mock_cache._add.assert_called_with(pytest.KEY + "-lock", lock._value, ttl=20) - assert lock._EVENTS[pytest.KEY + "-lock"].is_set() is False + mock_base_cache._add.assert_called_with(KEY_LOCK, lock._value, ttl=20) + assert lock._EVENTS[KEY_LOCK].is_set() is False - @pytest.mark.asyncio - async def test_release(self, mock_cache, lock): - mock_cache._redlock_release.return_value = True + async def test_release(self, mock_base_cache, lock): + mock_base_cache._redlock_release.return_value = True await lock._acquire() await lock._release() - mock_cache._redlock_release.assert_called_with(pytest.KEY + "-lock", lock._value) - assert pytest.KEY + "-lock" not in lock._EVENTS + mock_base_cache._redlock_release.assert_called_with(KEY_LOCK, lock._value) + assert KEY_LOCK not in lock._EVENTS - @pytest.mark.asyncio - async def test_release_no_acquire(self, mock_cache, lock): - mock_cache._redlock_release.return_value = False - assert pytest.KEY + "-lock" not in lock._EVENTS + async def test_release_no_acquire(self, mock_base_cache, lock): + mock_base_cache._redlock_release.return_value = False + assert KEY_LOCK not in lock._EVENTS await lock._release() - assert pytest.KEY + "-lock" not in lock._EVENTS + assert KEY_LOCK not in lock._EVENTS - @pytest.mark.asyncio - async def test_context_manager(self, mock_cache, lock): + async def test_context_manager(self, mock_base_cache, lock): async with lock: pass - mock_cache._add.assert_called_with(pytest.KEY + "-lock", lock._value, ttl=20) - mock_cache._redlock_release.assert_called_with(pytest.KEY + "-lock", lock._value) + mock_base_cache._add.assert_called_with(KEY_LOCK, lock._value, ttl=20) + mock_base_cache._redlock_release.assert_called_with(KEY_LOCK, lock._value) - @pytest.mark.asyncio - async def test_raises_exceptions(self, mock_cache, lock): - mock_cache._redlock_release.return_value = True + async def test_raises_exceptions(self, mock_base_cache, lock): + mock_base_cache._redlock_release.return_value = True with pytest.raises(ValueError): async with lock: raise ValueError - @pytest.mark.asyncio - async def test_acquire_block_timeouts(self, mock_cache, lock): + async def test_acquire_block_timeouts(self, mock_base_cache, lock): await lock._acquire() - with asynctest.patch("asyncio.wait_for", side_effect=asyncio.TimeoutError): - mock_cache._add.side_effect = ValueError - assert await lock._acquire() is None - @pytest.mark.asyncio - async def test_wait_for_release_no_acquire(self, mock_cache, lock): - mock_cache._add.side_effect = ValueError + # Mock .wait() to avoid unawaited coroutine warning. + with patch.object(RedLock._EVENTS[lock.key], "wait", Mock(spec_set=())): + with patch("asyncio.wait_for", autospec=True, side_effect=asyncio.TimeoutError): + mock_base_cache._add.side_effect = ValueError + result = await lock._acquire() + assert result is None + + async def test_wait_for_release_no_acquire(self, mock_base_cache, lock): + mock_base_cache._add.side_effect = ValueError assert await lock._acquire() is None - @pytest.mark.asyncio - async def test_multiple_locks_lock(self, mock_cache, lock): - lock_1 = RedLock(mock_cache, pytest.KEY, 20) - lock_2 = RedLock(mock_cache, pytest.KEY, 20) - mock_cache._add.side_effect = [True, ValueError(), ValueError()] + async def test_multiple_locks_lock(self, mock_base_cache, lock): + lock_1 = RedLock(mock_base_cache, Keys.KEY, 20) + lock_2 = RedLock(mock_base_cache, Keys.KEY, 20) + mock_base_cache._add.side_effect = [True, ValueError(), ValueError()] await lock._acquire() - event = lock._EVENTS[pytest.KEY + "-lock"] + event = lock._EVENTS[KEY_LOCK] - assert pytest.KEY + "-lock" in lock._EVENTS - assert pytest.KEY + "-lock" in lock_1._EVENTS - assert pytest.KEY + "-lock" in lock_2._EVENTS + assert KEY_LOCK in lock._EVENTS + assert KEY_LOCK in lock_1._EVENTS + assert KEY_LOCK in lock_2._EVENTS assert not event.is_set() await asyncio.gather(lock_1._acquire(), lock._release(), lock_2._acquire()) - assert pytest.KEY + "-lock" not in lock._EVENTS - assert pytest.KEY + "-lock" not in lock_1._EVENTS - assert pytest.KEY + "-lock" not in lock_2._EVENTS + assert KEY_LOCK not in lock._EVENTS + assert KEY_LOCK not in lock_1._EVENTS + assert KEY_LOCK not in lock_2._EVENTS assert event.is_set() class TestOptimisticLock: @pytest.fixture - def lock(self, mock_cache): - yield OptimisticLock(mock_cache, pytest.KEY) + def lock(self, mock_base_cache): + yield OptimisticLock(mock_base_cache, Keys.KEY) - def test_init(self, mock_cache, lock): - assert lock.client == mock_cache + def test_init(self, mock_base_cache, lock): + assert lock.client == mock_base_cache assert lock._token is None - assert lock.key == pytest.KEY - assert lock.ns_key == mock_cache._build_key(pytest.KEY) + assert lock.key == Keys.KEY + assert lock.ns_key == mock_base_cache.build_key(Keys.KEY) - @pytest.mark.asyncio async def test_aenter_returns_lock(self, lock): assert await lock.__aenter__() is lock - @pytest.mark.asyncio async def test_aexit_not_crashing(self, lock): async with lock: pass - @pytest.mark.asyncio async def test_acquire_calls_get(self, lock): await lock._acquire() - lock.client._gets.assert_called_with(pytest.KEY) + lock.client._gets.assert_called_with(Keys.KEY) assert lock._token == lock.client._gets.return_value - @pytest.mark.asyncio - async def test_cas_calls_set_with_token(self, lock): + async def test_cas_calls_set_with_token(self, lock, mocker): + m = mocker.spy(lock.client, "set") await lock._acquire() await lock.cas("value") - lock.client.set.assert_called_with(pytest.KEY, "value", _cas_token=lock._token) + m.assert_called_with(Keys.KEY, "value", _cas_token=lock._token) - @pytest.mark.asyncio - async def test_wrong_token_raises_error(self, mock_cache, lock): - mock_cache._set.return_value = 0 + async def test_wrong_token_raises_error(self, mock_base_cache, lock): + mock_base_cache._set.return_value = 0 with pytest.raises(OptimisticLockError): await lock.cas("value") diff --git a/tests/ut/test_plugins.py b/tests/ut/test_plugins.py index 410dab980..38cc5b2be 100644 --- a/tests/ut/test_plugins.py +++ b/tests/ut/test_plugins.py @@ -1,26 +1,26 @@ -import pytest +from unittest.mock import create_autospec -from unittest.mock import MagicMock +import pytest -from aiocache.plugins import BasePlugin, TimingPlugin, HitMissRatioPlugin from aiocache.base import API, BaseCache +from aiocache.plugins import BasePlugin, HitMissRatioPlugin, TimingPlugin +from ..utils import Keys class TestBasePlugin: - @pytest.mark.asyncio async def test_interface_methods(self): for method in API.CMDS: - assert await getattr(BasePlugin, "pre_{}".format(method.__name__))(MagicMock()) is None - assert await getattr(BasePlugin, "post_{}".format(method.__name__))(MagicMock()) is None + pre = await getattr(BasePlugin, "pre_{}".format(method.__name__))(None) + assert pre is None + post = await getattr(BasePlugin, "post_{}".format(method.__name__))(None) + assert post is None - @pytest.mark.asyncio async def test_do_nothing(self): assert await BasePlugin().do_nothing() is None class TestTimingPlugin: - @pytest.mark.asyncio - async def test_save_time(mock_cache): + async def test_save_time(self, mock_cache): do_save_time = TimingPlugin().save_time("get") await do_save_time("self", mock_cache, took=1) await do_save_time("self", mock_cache, took=2) @@ -30,8 +30,7 @@ async def test_save_time(mock_cache): assert mock_cache.profiling["get_min"] == 1 assert mock_cache.profiling["get_avg"] == 1.5 - @pytest.mark.asyncio - async def test_save_time_post_set(mock_cache): + async def test_save_time_post_set(self, mock_cache): await TimingPlugin().post_set(mock_cache, took=1) await TimingPlugin().post_set(mock_cache, took=2) @@ -40,7 +39,6 @@ async def test_save_time_post_set(mock_cache): assert mock_cache.profiling["set_min"] == 1 assert mock_cache.profiling["set_avg"] == 1.5 - @pytest.mark.asyncio async def test_interface_methods(self): for method in API.CMDS: assert hasattr(TimingPlugin, "pre_{}".format(method.__name__)) @@ -52,30 +50,28 @@ class TestHitMissRatioPlugin: def plugin(self): return HitMissRatioPlugin() - @pytest.mark.asyncio async def test_post_get(self, plugin): - client = MagicMock(spec=BaseCache) - await plugin.post_get(client, pytest.KEY) + client = create_autospec(BaseCache, instance=True) + await plugin.post_get(client, Keys.KEY) assert client.hit_miss_ratio["hits"] == 0 assert client.hit_miss_ratio["total"] == 1 assert client.hit_miss_ratio["hit_ratio"] == 0 - await plugin.post_get(client, pytest.KEY, ret="value") + await plugin.post_get(client, Keys.KEY, ret="value") assert client.hit_miss_ratio["hits"] == 1 assert client.hit_miss_ratio["total"] == 2 assert client.hit_miss_ratio["hit_ratio"] == 0.5 - @pytest.mark.asyncio async def test_post_multi_get(self, plugin): - client = MagicMock(spec=BaseCache) - await plugin.post_multi_get(client, [pytest.KEY, pytest.KEY_1], ret=[None, None]) + client = create_autospec(BaseCache, instance=True) + await plugin.post_multi_get(client, [Keys.KEY, Keys.KEY_1], ret=[None, None]) assert client.hit_miss_ratio["hits"] == 0 assert client.hit_miss_ratio["total"] == 2 assert client.hit_miss_ratio["hit_ratio"] == 0 - await plugin.post_multi_get(client, [pytest.KEY, pytest.KEY_1], ret=["value", "random"]) + await plugin.post_multi_get(client, [Keys.KEY, Keys.KEY_1], ret=["value", "random"]) assert client.hit_miss_ratio["hits"] == 2 assert client.hit_miss_ratio["total"] == 4 assert client.hit_miss_ratio["hit_ratio"] == 0.5 diff --git a/tests/ut/test_serializers.py b/tests/ut/test_serializers.py index d88a4095e..33835531b 100644 --- a/tests/ut/test_serializers.py +++ b/tests/ut/test_serializers.py @@ -1,16 +1,16 @@ -import pytest import pickle - from collections import namedtuple from unittest import mock +import pytest + from aiocache.serializers import ( BaseSerializer, - NullSerializer, - StringSerializer, - PickleSerializer, JsonSerializer, MsgPackSerializer, + NullSerializer, + PickleSerializer, + StringSerializer, ) @@ -20,33 +20,18 @@ JSON_TYPES = [1, 2.0, "hi", True, ["1", 1], {"key": "value"}] -class TestBaseSerializer: +class TestNullSerializer: def test_init(self): - serializer = BaseSerializer() + serializer = NullSerializer() + assert isinstance(serializer, BaseSerializer) assert serializer.DEFAULT_ENCODING == "utf-8" assert serializer.encoding == "utf-8" def test_init_encoding(self): - serializer = BaseSerializer(encoding="whatever") + serializer = NullSerializer(encoding="whatever") assert serializer.DEFAULT_ENCODING == "utf-8" assert serializer.encoding == "whatever" - def test_dumps(self): - with pytest.raises(NotImplementedError): - BaseSerializer().dumps("") - - def test_loads(self): - with pytest.raises(NotImplementedError): - BaseSerializer().loads("") - - -class TestNullSerializer: - def test_init(self): - serializer = NullSerializer() - assert isinstance(serializer, BaseSerializer) - assert serializer.DEFAULT_ENCODING == "utf-8" - assert serializer.encoding == "utf-8" - @pytest.mark.parametrize("obj", TYPES) def test_set_types(self, obj): assert NullSerializer().dumps(obj) is obj @@ -76,7 +61,7 @@ def serializer(self): yield PickleSerializer(protocol=4) def test_init(self, serializer): - assert isinstance(serializer, BaseSerializer) + assert isinstance(serializer, PickleSerializer) assert serializer.DEFAULT_ENCODING is None assert serializer.encoding is None assert serializer.protocol == 4 @@ -90,9 +75,8 @@ def test_set_types(self, obj, serializer): assert serializer.loads(serializer.dumps(obj)) == obj def test_dumps(self, serializer): - assert ( - serializer.dumps("hi") == b"\x80\x04\x95\x06\x00\x00\x00\x00\x00\x00\x00\x8c\x02hi\x94." - ) + expected = b"\x80\x04\x95\x06\x00\x00\x00\x00\x00\x00\x00\x8c\x02hi\x94." + assert serializer.dumps("hi") == expected def test_dumps_with_none(self, serializer): assert isinstance(serializer.dumps(None), bytes) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..12194f83d --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,76 @@ +from enum import Enum +from typing import Optional, Union + +from aiocache.base import BaseCache + + +class Keys(str, Enum): + KEY: str = "key" + KEY_1: str = "random" + + +KEY_LOCK = Keys.KEY + "-lock" + + +def ensure_key(key: Union[str, Enum]) -> str: + if isinstance(key, Enum): + return key.value + else: + return key + + +class AbstractBaseCache(BaseCache[str]): + """BaseCache that can be mocked for NotImplementedError tests""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def build_key(self, key: str, namespace: Optional[str] = None) -> str: + return super().build_key(key, namespace) + + async def _add(self, key, value, ttl, _conn=None): + return await super()._add(key, value, ttl, _conn) + + async def _get(self, key, encoding, _conn=None): + return await super()._get(key, encoding, _conn) + + async def _gets(self, key, encoding="utf-8", _conn=None): + return await super()._gets(key, encoding, _conn) + + async def _multi_get(self, keys, encoding, _conn=None): + return await super()._multi_get(keys, encoding, _conn) + + async def _set(self, key, value, ttl, _cas_token=None, _conn=None): + return await super()._set(key, value, ttl, _cas_token, _conn) + + async def _multi_set(self, pairs, ttl, _conn=None): + return await super()._multi_set(pairs, ttl, _conn) + + async def _delete(self, key, _conn=None): + return await super()._delete(key, _conn) + + async def _exists(self, key, _conn=None): + return await super()._exists(key, _conn) + + async def _increment(self, key, delta, _conn=None): + return await super()._increment(key, delta, _conn) + + async def _expire(self, key, ttl, _conn=None): + return await super()._expire(key, ttl, _conn) + + async def _clear(self, namespace, _conn=None): + return await super()._clear(namespace, _conn) + + async def _raw(self, command, *args, **kwargs): + return await super()._raw(command, *args, **kwargs) + + async def _redlock_release(self, key, value): + return await super()._redlock_release(key, value) + + +class ConcreteBaseCache(AbstractBaseCache): + """BaseCache that can be mocked for tests""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def build_key(self, key: str, namespace: Optional[str] = None) -> str: + return self._str_build_key(key, namespace) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 1517d6659..000000000 --- a/tox.ini +++ /dev/null @@ -1,64 +0,0 @@ -[tox] -envlist = - py{36,37,38,39}-{deps-lowest,deps-devel} - py{36,37,38,39}-ujson - codecov - syntax - docs-html - - -[testenv] -usedevelop = true -whitelist_externals = - make - bash - -deps = - py{38,39}-deps-lowest: aioredis==1.3.0 - py37-deps-lowest: aioredis==1.0.0 - py36-deps-lowest: aioredis==0.3.3 - deps-lowest: aiomcache==0.5.2 - deps-devel: https://github.com/aio-libs/aiomcache/archive/master.tar.gz - deps-devel: https://github.com/aio-libs/aioredis/archive/master.tar.gz - - ujson: ujson - - .[redis] - .[memcached] - .[msgpack] - .[dev] - -commands = - make unit cov-report=false - make acceptance - bash examples/run_all.sh - - -[testenv:syntax] -deps = - flake8 - black -whitelist_externals = make -commands = - make lint - - -[testenv:codecov] -passenv = CI TRAVIS TRAVIS_* -deps = codecov -skip_install = true -commands = - coverage combine - coverage report - codecov - - -[testenv:docs-html] -deps = - .[redis] - .[memcached] - .[msgpack] - sphinx - sphinx-rtd-theme -commands = - sphinx-build -b html -d {envtmpdir}/doctrees docs {envtmpdir}/html