From fb6f2e07c97d8b6ad455a269d24f216ac80f5cf9 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Thu, 13 Sep 2018 15:19:25 +0200 Subject: [PATCH] Added a bunch of new features: - Debug mode is now configurable in the configuration file. This way, we don't have to edit versioned files to disable it on production systems. - Recent correspondents filter (enable in configuration file) - Document actions: Edit tags and correspondents on multiple documents at once - Replaced month list filter with date drilldown - Sortable document count columns on Tag and Correspondent admin - Last correspondence column on Correspondent admin - Save and edit next functionality for document editing --- .gitignore | 2 + paperless.conf.example | 10 ++ requirements.txt | 3 + src/documents/actions.py | 108 +++++++++++++ src/documents/admin.py | 142 +++++++++++++----- .../admin/documents/document/change_form.html | 18 ++- .../documents/document/select_object.html | 46 ++++++ src/paperless/settings.py | 14 +- 8 files changed, 301 insertions(+), 42 deletions(-) mode change 100644 => 100755 requirements.txt create mode 100644 src/documents/actions.py create mode 100644 src/documents/templates/admin/documents/document/select_object.html diff --git a/.gitignore b/.gitignore index a16f958a3..439d9df4b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,5 @@ docker-compose.env scripts/import-for-development scripts/nuke +# Static files collected by the collectstatic command +static/ diff --git a/paperless.conf.example b/paperless.conf.example index 15498a26a..05cf81724 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -59,6 +59,11 @@ PAPERLESS_EMAIL_SECRET="" #### Security #### ############################################################################### +# Controls whether django's debug mode is enabled. Disable this on production +# systems. Debug mode is enabled by default. +PAPERLESS_DEBUG="false" + + # Paperless can be instructed to attempt to encrypt your PDF files with GPG # using the PAPERLESS_PASSPHRASE specified below. If however you're not # concerned about encrypting these files (for example if you have disk @@ -203,3 +208,8 @@ PAPERLESS_EMAIL_SECRET="" # positive integer, but if you don't define one in paperless.conf, a default of # 100 will be used. #PAPERLESS_LIST_PER_PAGE=100 + + +# The number of years for which a correspondent will be included in the recent +# correspondents filter. +#PAPERLESS_RECENT_CORRESPONDENT_YEARS=1 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index 0476efef1..89c7e296f --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ idna==2.7 inotify-simple==1.1.8 langdetect==1.0.7 more-itertools==4.3.0 +numpy==1.15.1 pdftotext==2.1.0 pillow==5.2.0 pluggy==0.7.1; python_version != '3.1.*' @@ -45,6 +46,8 @@ pytz==2018.5 regex==2018.8.29 requests==2.19.1 six==1.11.0 +scikit-learn==0.19.2 +scipy==1.1.0 termcolor==1.1.0 text-unidecode==1.2 tzlocal==1.5.1 diff --git a/src/documents/actions.py b/src/documents/actions.py new file mode 100644 index 000000000..3db5cd314 --- /dev/null +++ b/src/documents/actions.py @@ -0,0 +1,108 @@ +from django.contrib import messages +from django.contrib.admin import helpers +from django.contrib.admin.utils import model_ngettext +from django.core.exceptions import PermissionDenied +from django.template.response import TemplateResponse + +from documents.models import Tag, Correspondent + + +def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None): + opts = modeladmin.model._meta + app_label = opts.app_label + + if not modeladmin.has_change_permission(request): + raise PermissionDenied + + if request.POST.get('post'): + n = queryset.count() + selected_object = modelclass.objects.get(id=request.POST.get('obj_id')) + if n: + for document in queryset: + if document_action: + document_action(document, selected_object) + document_display = str(document) + modeladmin.log_change(request, document, document_display) + if queryset_action: + queryset_action(queryset, selected_object) + + modeladmin.message_user(request, success_message % { + "selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n) + }, messages.SUCCESS) + + # Return None to display the change list page again. + return None + + context = dict( + modeladmin.admin_site.each_context(request), + title=title, + queryset=queryset, + opts=opts, + action_checkbox_name=helpers.ACTION_CHECKBOX_NAME, + media=modeladmin.media, + action=action, + objects=modelclass.objects.all(), + itemname=model_ngettext(modelclass, 1) + ) + + request.current_app = modeladmin.admin_site.name + + return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context) + + +def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None): + if not modeladmin.has_change_permission(request): + raise PermissionDenied + + n = queryset.count() + if n: + for document in queryset: + if document_action: + document_action(document) + document_display = str(document) + modeladmin.log_change(request, document, document_display) + if queryset_action: + queryset_action(queryset) + modeladmin.message_user(request, success_message % { + "count": n, "items": model_ngettext(modeladmin.opts, n) + }, messages.SUCCESS) + + # Return None to display the change list page again. + return None + + +def add_tag_to_selected(modeladmin, request, queryset): + return select_action(modeladmin=modeladmin, request=request, queryset=queryset, + title="Add tag to multiple documents", + action="add_tag_to_selected", + modelclass=Tag, + success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.", + document_action=lambda doc, tag: doc.tags.add(tag)) +add_tag_to_selected.short_description = "Add tag to selected documents" + + +def remove_tag_from_selected(modeladmin, request, queryset): + return select_action(modeladmin=modeladmin, request=request, queryset=queryset, + title="Remove tag from multiple documents", + action="remove_tag_from_selected", + modelclass=Tag, + success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.", + document_action=lambda doc, tag: doc.tags.remove(tag)) +remove_tag_from_selected.short_description = "Remove tag from selected documents" + + +def set_correspondent_on_selected(modeladmin, request, queryset): + return select_action(modeladmin=modeladmin, request=request, queryset=queryset, + title="Set correspondent on multiple documents", + action="set_correspondent_on_selected", + modelclass=Correspondent, + success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.", + queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent)) +set_correspondent_on_selected.short_description = "Set correspondent on selected documents" + + +def remove_correspondent_from_selected(modeladmin, request, queryset): + return simple_action(modeladmin=modeladmin, request=request, queryset=queryset, + success_message="Successfully removed correspondent from %(count)d %(items)s.", + queryset_action=lambda qs: qs.update(correspondent=None)) +remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents" diff --git a/src/documents/admin.py b/src/documents/admin.py index 36154b6ba..d545c1c02 100644 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -1,44 +1,25 @@ -from datetime import datetime +from datetime import datetime, timedelta from django.conf import settings -from django.contrib import admin +from django.contrib import admin, messages +from django.contrib.admin.templatetags.admin_urls import add_preserved_filters from django.contrib.auth.models import User, Group +from django.http import HttpResponseRedirect try: from django.core.urlresolvers import reverse except ImportError: from django.urls import reverse from django.templatetags.static import static -from django.utils.safestring import mark_safe from django.utils.html import format_html, format_html_join +from django.utils.http import urlquote +from django.utils.safestring import mark_safe +from django.db import models +from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \ + remove_correspondent_from_selected from .models import Correspondent, Tag, Document, Log -class MonthListFilter(admin.SimpleListFilter): - - title = "Month" - - # Parameter for the filter that will be used in the URL query. - parameter_name = "month" - - def lookups(self, request, model_admin): - r = [] - for document in Document.objects.all(): - r.append(( - document.created.strftime("%Y-%m"), - document.created.strftime("%B %Y") - )) - return sorted(set(r), key=lambda x: x[0], reverse=True) - - def queryset(self, request, queryset): - - if not self.value(): - return None - - year, month = self.value().split("-") - return queryset.filter(created__year=year, created__month=month) - - class FinancialYearFilter(admin.SimpleListFilter): title = "Financial Year" @@ -104,18 +85,43 @@ def queryset(self, request, queryset): created__lte=self._fy_end(end)) +class RecentCorrespondentFilter(admin.RelatedFieldListFilter): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.title = "correspondent (recent)" + + def field_choices(self, field, request, model_admin): + lookups = [] + if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0: + date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS) + for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct(): + lookups.append((c.id, c.name)) + return lookups + + class CommonAdmin(admin.ModelAdmin): list_per_page = settings.PAPERLESS_LIST_PER_PAGE class CorrespondentAdmin(CommonAdmin): - list_display = ("name", "match", "matching_algorithm", "document_count") + list_display = ("name", "match", "matching_algorithm", "document_count", "last_correspondence") list_filter = ("matching_algorithm",) list_editable = ("match", "matching_algorithm") + def get_queryset(self, request): + qs = super(CorrespondentAdmin, self).get_queryset(request) + qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created")) + return qs + def document_count(self, obj): - return obj.documents.count() + return obj.document_count + document_count.admin_order_field = "document_count" + + def last_correspondence(self, obj): + return obj.last_correspondence + last_correspondence.admin_order_field = "last_correspondence" class TagAdmin(CommonAdmin): @@ -125,8 +131,14 @@ class TagAdmin(CommonAdmin): list_filter = ("colour", "matching_algorithm") list_editable = ("colour", "match", "matching_algorithm") + def get_queryset(self, request): + qs = super(TagAdmin, self).get_queryset(request) + qs = qs.annotate(document_count=models.Count("documents")) + return qs + def document_count(self, obj): - return obj.documents.count() + return obj.document_count + document_count.admin_order_field = "document_count" class DocumentAdmin(CommonAdmin): @@ -140,12 +152,18 @@ class Media: readonly_fields = ("added",) list_display = ("title", "created", "added", "thumbnail", "correspondent", "tags_") - list_filter = ("tags", "correspondent", FinancialYearFilter, - MonthListFilter) + list_filter = ("tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter) + filter_horizontal = ("tags",) ordering = ["-created", "correspondent"] + actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected] + + date_hierarchy = 'created' + + document_queue = None + def has_add_permission(self, request): return False @@ -153,6 +171,56 @@ def created_(self, obj): return obj.created.date().strftime("%Y-%m-%d") created_.short_description = "Created" + def changelist_view(self, request, extra_context=None): + response = super().changelist_view(request, extra_context) + + if request.method == 'GET': + cl = self.get_changelist_instance(request) + self.document_queue = [doc.id for doc in cl.queryset] + + return response + + def change_view(self, request, object_id=None, form_url='', extra_context=None): + extra_context = extra_context or {} + doc = Document.objects.get(id=object_id) + if self.document_queue and object_id and int(object_id) in self.document_queue: + # There is a queue of documents + current_index = self.document_queue.index(int(object_id)) + if current_index < len(self.document_queue) - 1: + # ... and there are still documents in the queue + extra_context['next_object'] = self.document_queue[current_index + 1] + return super(DocumentAdmin, self).change_view( + request, object_id, form_url, extra_context=extra_context, + ) + + def response_change(self, request, obj): + + # This is mostly copied from ModelAdmin.response_change() + opts = self.model._meta + preserved_filters = self.get_preserved_filters(request) + + msg_dict = { + 'name': opts.verbose_name, + 'obj': format_html('{}', urlquote(request.path), obj), + } + if "_saveandeditnext" in request.POST: + msg = format_html( + 'The {name} "{obj}" was changed successfully. Editing next object.', + **msg_dict + ) + self.message_user(request, msg, messages.SUCCESS) + redirect_url = reverse('admin:%s_%s_change' % + (opts.app_label, opts.model_name), + args=(request.POST['_next_object'],), + current_app=self.admin_site.name) + redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url) + response = HttpResponseRedirect(redirect_url) + else: + response = super().response_change(request, obj) + + return response + + @mark_safe def thumbnail(self, obj): return self._html_tag( "a", @@ -165,8 +233,8 @@ def thumbnail(self, obj): ), href=obj.download_url ) - thumbnail.allow_tags = True + @mark_safe def tags_(self, obj): r = "" for tag in obj.tags.all(): @@ -183,10 +251,11 @@ def tags_(self, obj): ) } ) - return mark_safe(r) - tags_.allow_tags = True + return r + @mark_safe def document(self, obj): + # TODO: is this method even used anymore? return self._html_tag( "a", self._html_tag( @@ -199,7 +268,6 @@ def document(self, obj): ), href=obj.download_url ) - document.allow_tags = True @staticmethod def _html_tag(kind, inside=None, **kwargs): diff --git a/src/documents/templates/admin/documents/document/change_form.html b/src/documents/templates/admin/documents/document/change_form.html index 7bd0e483f..88fae955d 100644 --- a/src/documents/templates/admin/documents/document/change_form.html +++ b/src/documents/templates/admin/documents/document/change_form.html @@ -1,5 +1,21 @@ {% extends 'admin/change_form.html' %} +{% block content %} + +{{ block.super }} + +{% if next_object %} + +{% endif %} + +{% endblock content %} {% block footer %} @@ -10,4 +26,4 @@ django.jQuery(".field-created input").first().attr("type", "date") -{% endblock footer %} \ No newline at end of file +{% endblock footer %} diff --git a/src/documents/templates/admin/documents/document/select_object.html b/src/documents/templates/admin/documents/document/select_object.html new file mode 100644 index 000000000..1439b5c21 --- /dev/null +++ b/src/documents/templates/admin/documents/document/select_object.html @@ -0,0 +1,46 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls static %} +{% load staticfiles %} + +{% block extrahead %} +{{ block.super }} +{{ media }} + + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
+ {% trans 'Home' %} + › {{ opts.app_config.verbose_name }} + › {{ opts.verbose_name_plural|capfirst }} + › {{title}} +
+{% endblock %} + +{% block content %} +

Please select the {{itemname}}.

+
{% csrf_token %} +
+ {% for obj in queryset %} + + {% endfor %} +

+ +

+ + + +

+ + {% trans "Go back" %} +

+
+
+{% endblock %} diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 956b90a7f..e6f3da0cb 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -22,12 +22,12 @@ load_dotenv("/usr/local/etc/paperless.conf") -def __get_boolean(key): +def __get_boolean(key, default="NO"): """ Return a boolean value based on whatever the user has supplied in the environment based on whether the value "looks like" it's True or not. """ - return bool(os.getenv(key, "NO").lower() in ("yes", "y", "1", "t", "true")) + return bool(os.getenv(key, default).lower() in ("yes", "y", "1", "t", "true")) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -47,7 +47,7 @@ def __get_boolean(key): # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = __get_boolean("PAPERLESS_DEBUG", "YES") LOGIN_URL = "admin:login" @@ -81,7 +81,7 @@ def __get_boolean(key): "rest_framework", "crispy_forms", - "django_filters", + "django_filters" ] @@ -292,3 +292,9 @@ def __get_boolean(key): # Specify the default date order (for autodetected dates) DATE_ORDER = os.getenv("PAPERLESS_DATE_ORDER", "DMY") + +# Specify for how many years a correspondent is considered recent. Recent +# correspondents will be shown in a separate "Recent correspondents" filter as +# well. Set to 0 to disable this filter. +PAPERLESS_RECENT_CORRESPONDENT_YEARS = int(os.getenv( + "PAPERLESS_RECENT_CORRESPONDENT_YEARS", 0))