".join([field for field in row.diff]) summary += ( " | ||
change_type | " @@ -132,18 +139,26 @@ def before_import_row(self, row, **kwargs): + " | ".join([str(field) for field in row.values])
- cols_error = lambda row: "".join(
- [
- ""
- + key
- + ""
- + " " - + row.error.message_dict[key][0] - + " " - for key in row.error.message_dict.keys() - ] + cols = lambda row: " | ".join(
+ [str(field) for field in row.values]
)
+
+ def cols_error(row):
+ if hasattr(row.error, "message_dict"):
+ return "".join(
+ [
+ ""
+ + key
+ + ""
+ + " " + + row.error.message_dict[key][0] + + " " + for key in row.error.message_dict.keys() + ] + ) + else: + return "".join(message + " " for message in row.error.messages) + summary += ( " |
row | " + "errors | "
@@ -179,9 +194,13 @@ def before_import_row(self, row, **kwargs):
import_job.save()
-@shared_task(bind=False)
+@shared_task(
+ bind=False,
+ soft_time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_IMPORT_SOFT_TIME_LIMIT", 0),
+ time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_IMPORT_HARD_TIME_LIMIT", 0),
+)
def run_import_job(pk, dry_run=True):
- log.info("Importing %s dry-run %s" % (pk, dry_run))
+ log.info(f"Importing {pk} dry-run {dry_run}")
import_job = models.ImportJob.objects.get(pk=pk)
try:
_run_import_job(import_job, dry_run)
@@ -192,7 +211,11 @@ def run_import_job(pk, dry_run=True):
return
-@shared_task(bind=False)
+@shared_task(
+ bind=False,
+ soft_time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_EXPORT_SOFT_TIME_LIMIT", 0),
+ time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_EXPORT_HARD_TIME_LIMIT", 0),
+)
def run_export_job(pk):
log.info("Exporting %s" % pk)
export_job = models.ExportJob.objects.get(pk=pk)
@@ -211,10 +234,10 @@ def export_resource(self, *args, **kwargs):
change_job_status(
export_job,
"export",
- "Exporting row %s/%s" % (self.row_number, qs_len),
+ f"Exporting row {self.row_number}/{qs_len}",
)
self.row_number += 1
- return super(Resource, self).export_resource(*args, **kwargs)
+ return super().export_resource(*args, **kwargs)
resource = Resource(export_job=export_job)
@@ -232,21 +255,5 @@ def export_resource(self, *args, **kwargs):
serialized = serialized.encode("utf8")
export_job.file.save(filename, ContentFile(serialized))
if export_job.email_on_completion:
- send_mail(
- _("Django: Export job completed"),
- _(
- "Your export job on model {app_label}.{model} has completed. You can download the file at the following link:\n\n{link}"
- ).format(
- app_label=export_job.app_label,
- model=export_job.model,
- link=export_job.site_of_origin
- + reverse(
- "admin:%s_%s_change"
- % (export_job._meta.app_label, export_job._meta.model_name,),
- args=[export_job.pk],
- ),
- ),
- settings.SERVER_EMAIL,
- [export_job.updated_by.email],
- )
+ send_export_job_completion_mail(export_job)
return
diff --git a/import_export_celery/templates/email/export_job_completion.html b/import_export_celery/templates/email/export_job_completion.html
new file mode 100644
index 0000000..845c8c5
--- /dev/null
+++ b/import_export_celery/templates/email/export_job_completion.html
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+ Your export job on model {{app_label}}.{{model}} has completed. You can download the file at the following link: + {{link}} + + diff --git a/import_export_celery/utils.py b/import_export_celery/utils.py new file mode 100644 index 0000000..cbe05a7 --- /dev/null +++ b/import_export_celery/utils.py @@ -0,0 +1,98 @@ +import html2text +from django.core.mail import send_mail +from django.template.loader import get_template +from django.conf import settings +from django.urls import reverse +from import_export.formats.base_formats import DEFAULT_FORMATS + +DEFAULT_EXPORT_JOB_EMAIL_ON_COMPLETION = True +DEFAULT_EXPORT_JOB_COMPLETION_MAIL_SUBJECT = "Django: Export job completed" +DEFAULT_EXPORT_JOB_COMPLETION_MAIL_TEMPLATE = ( + "email/export_job_completion.html" +) +IMPORT_EXPORT_CELERY_EXCLUDED_FORMATS = getattr( + settings, + "IMPORT_EXPORT_CELERY_EXCLUDED_FORMATS", + [], +) + + +def get_formats(): + return [ + format + for format in DEFAULT_FORMATS + if format.TABLIB_MODULE.split(".")[-1].strip("_") + not in IMPORT_EXPORT_CELERY_EXCLUDED_FORMATS + ] + + +def build_html_and_text_message(template_name, context={}): + """ + Render the given template with the context and returns + the data in html and plain text format. + """ + template = get_template(template_name) + html_message = template.render(context) + text_message = html2text.html2text(html_message) + return html_message, text_message + + +def get_export_job_mail_context(export_job): + context = { + "app_label": export_job.app_label, + "model": export_job.model, + "link": export_job.site_of_origin + + reverse( + "admin:%s_%s_change" + % ( + export_job._meta.app_label, + export_job._meta.model_name, + ), + args=[export_job.pk], + ), + } + return context + + +def get_export_job_email_on_completion(): + return getattr( + settings, + "EXPORT_JOB_EMAIL_ON_COMPLETION", + DEFAULT_EXPORT_JOB_EMAIL_ON_COMPLETION, + ) + + +def get_export_job_mail_subject(): + return getattr( + settings, + "EXPORT_JOB_COMPLETION_MAIL_SUBJECT", + DEFAULT_EXPORT_JOB_COMPLETION_MAIL_SUBJECT, + ) + + +def get_export_job_mail_template(): + return getattr( + settings, + "EXPORT_JOB_COMPLETION_MAIL_TEMPLATE", + DEFAULT_EXPORT_JOB_COMPLETION_MAIL_TEMPLATE, + ) + + +def send_export_job_completion_mail(export_job): + """ + Send export job completion mail + """ + subject = get_export_job_mail_subject() + template_name = get_export_job_mail_template() + context = get_export_job_mail_context(export_job) + context.update({"export_job": export_job}) + html_message, text_message = build_html_and_text_message( + template_name, context + ) + send_mail( + subject=subject, + message=text_message, + html_message=html_message, + from_email=settings.SERVER_EMAIL, + recipient_list=[export_job.updated_by.email], + ) diff --git a/requirements_test.txt b/requirements_test.txt new file mode 100644 index 0000000..9a18bf2 --- /dev/null +++ b/requirements_test.txt @@ -0,0 +1,7 @@ +Django +django-import-export +django-author +html2text +celery +psycopg2-binary +django-admin-smoke-tests diff --git a/setup-dev-env.sh b/setup-dev-env.sh new file mode 100755 index 0000000..58bd72f --- /dev/null +++ b/setup-dev-env.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd example +poetry install +poetry run python3 manage.py migrate diff --git a/setup.cfg b/setup.cfg index 987b3e5..8d4f8ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [flake8] max-line-length = 150 exclude = + pyenv/*, env/*, env-dj19/*, *migrations/*, @@ -23,4 +24,4 @@ exclude = *.xml max-complexity = 10 enable-extensions = import-order, blind-except -ignore = C000, W504 +ignore = C000, C416, C901, E731, W503, W504 diff --git a/setup.py b/setup.py index a7a48fa..415e322 100644 --- a/setup.py +++ b/setup.py @@ -1,38 +1,51 @@ +import codecs import os from setuptools import setup, find_packages -import sys import subprocess +import datetime here = os.path.abspath(os.path.dirname(__file__)) -import codecs -requires = ['Django', 'django-import-export', 'django-author'] +requires = ["Django", "django-import-export", "django-author", "html2text"] -version = subprocess.check_output(['git','describe','--abbrev=0','--tags']).decode("utf-8") +try: + version = ( + subprocess.check_output(["git", "describe", "--abbrev=0", "--tags"]) + .decode("utf-8") + .strip() + ) +except subprocess.CalledProcessError: + version = "0.dev" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") setup( - name='django-import-export-celery', - version=version, - author='Timothy Hobbs', - author_email='timothy.hobbs@auto-mat.cz', - url='https://github.com/auto-mat/django-import-export-celery', - download_url="http://pypi.python.org/pypi/django-import-export-celery/", - description="Process long running django imports and exports in celery", - long_description=codecs.open( - os.path.join( - here, 'README.rst'), 'r', 'utf-8').read(), - license='License :: OSI Approved :: GNU Lesser General Public License v3.0 or later (LGPLv3.0+)', - install_requires=requires, - packages=find_packages(), - include_package_data=True, - zip_safe=False, - classifiers=['Topic :: Utilities', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Intended Audience :: Developers', - 'Environment :: Web Environment', - 'Framework :: Django', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5'], + name="django-import-export-celery", + version=version, + author="Timothy Hobbs", + author_email="timothy.hobbs@auto-mat.cz", + url="https://github.com/auto-mat/django-import-export-celery", + download_url="http://pypi.python.org/pypi/django-import-export-celery/", + description="Process long running django imports and exports in celery", + long_description=codecs.open( + os.path.join(here, "README.rst"), "r", "utf-8" + ).read(), + long_description_content_type="text/x-rst", + license=( + "License :: OSI Approved :: GNU Lesser General Public License v3.0 or" + " later (LGPLv3.0+)" + ), + install_requires=requires, + packages=find_packages(), + include_package_data=True, + zip_safe=False, + classifiers=[ + "Topic :: Utilities", + "Natural Language :: English", + "Operating System :: OS Independent", + "Intended Audience :: Developers", + "Environment :: Web Environment", + "Framework :: Django", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], ) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..53ee723 --- /dev/null +++ b/tox.ini @@ -0,0 +1,38 @@ +[tox] +envlist = + py{36,37,38,39,310}-django32 + py{38,39,310}-django40 + py{38,39,310,311}-django41 + py{38,39,310,311,312}-django42 + py{310,311,312}-django50 + py{310,311,312}-django51 + +[testenv] +deps = + -rrequirements_test.txt + coverage + django-coverage-plugin + django32: django>=3.2,<3.3 + django40: django>=4.0,<4.1 + django41: django>=4.1,<4.2 + django42: django>=4.2,<4.3 + django50: django>=5.0,<5.1 + django51: django>=5.1a1,<5.2 + +setenv = + DATABASE_TYPE=sqlite + REDIS_URL=redis://127.0.0.1:6379/0 + +allowlist_externals = coverage + +test-executable = + python --version + python -c "import django; print(django.get_version())" + pip install -r requirements_test.txt + {envbindir}/python -Wall {envbindir}/coverage run --append + +commands = + python example/manage.py migrate + {[testenv]test-executable} example/manage.py test winners + coverage report + coverage xml -o coverage.xml |