better sanity checker that logs messages in the log files and does no…

…t fail on warnings.

docs/changelog.rst

@@ -10,8 +10,10 @@ paperless-ng 1.1.2
 
 * Always show top left corner of thumbnails, even for extra wide documents.
 
-* Added a management command for executing the sanity checker directly. See :ref:`utilities-sanity-checker`.
-
+* Added a management command for executing the sanity checker directly.
+  See :ref:`utilities-sanity-checker`.
+  The sanity checker will also report errors in the log files.
+
 * Fixed an issue with the metadata tab not reporting anything in case of missing files.
 
 * Reverted a change from 1.1.0 that caused huge memory usage due to redis caching.

src/documents/management/commands/document_sanity_checker.py

@@ -1,8 +1,5 @@
-import logging
 from django.core.management.base import BaseCommand
-from documents.sanity_checker import check_sanity, SanityError, SanityWarning
-
-logger = logging.getLogger("paperless.management.sanity_checker")
+from documents.sanity_checker import check_sanity
 
 
 class Command(BaseCommand):
@@ -15,13 +12,4 @@ def handle(self, *args, **options):
 
         messages = check_sanity(progress=True)
 
-        if len(messages) == 0:
-            logger.info("No issues found.")
-        else:
-            for msg in messages:
-                if type(msg) == SanityError:
-                    logger.error(str(msg))
-                elif type(msg) == SanityWarning:
-                    logger.warning(str(msg))
-                else:
-                    logger.info((str(msg)))
+        messages.log_messages()

src/documents/sanity_checker.py

@@ -1,4 +1,5 @@
 import hashlib
+import logging
 import os
 
 from django.conf import settings
@@ -7,40 +8,48 @@
 from documents.models import Document
 
 
-class SanityMessage:
-    message = None
+class SanityCheckMessages:
 
+    def __init__(self):
+        self._messages = []
 
-class SanityWarning(SanityMessage):
-    def __init__(self, message):
-        self.message = message
+    def error(self, message):
+        self._messages.append({"level": logging.ERROR, "message": message})
 
-    def __str__(self):
-        return f"Warning: {self.message}"
+    def warning(self, message):
+        self._messages.append({"level": logging.WARNING, "message": message})
 
+    def info(self, message):
+        self._messages.append({"level": logging.INFO, "message": message})
 
-class SanityError(SanityMessage):
-    def __init__(self, message):
-        self.message = message
+    def log_messages(self):
+        logger = logging.getLogger("paperless.sanity_checker")
 
-    def __str__(self):
-        return f"ERROR: {self.message}"
+        if len(self._messages) == 0:
+            logger.info("Sanity checker detected no issues.")
+        else:
+            for msg in self._messages:
+                logger.log(msg['level'], msg['message'])
+
+    def __len__(self):
+        return len(self._messages)
+
+    def __getitem__(self, item):
+        return self._messages[item]
 
+    def has_error(self):
+        return any([msg['level'] == logging.ERROR for msg in self._messages])
 
-class SanityFailedError(Exception):
+    def has_warning(self):
+        return any([msg['level'] == logging.WARNING for msg in self._messages])
 
-    def __init__(self, messages):
-        self.messages = messages
 
-    def __str__(self):
-        message_string = "\n".join([str(m) for m in self.messages])
-        return (
-            f"The following issuse were found by the sanity checker:\n"
-            f"{message_string}\n\n===============\n\n")
+class SanityCheckFailedException(Exception):
+    pass
 
 
 def check_sanity(progress=False):
-    messages = []
+    messages = SanityCheckMessages()
 
     present_files = []
     for root, subdirs, files in os.walk(settings.MEDIA_ROOT):
@@ -59,83 +68,78 @@ def check_sanity(progress=False):
     for doc in docs:
         # Check sanity of the thumbnail
         if not os.path.isfile(doc.thumbnail_path):
-            messages.append(SanityError(
-                f"Thumbnail of document {doc.pk} does not exist."))
+            messages.error(f"Thumbnail of document {doc.pk} does not exist.")
         else:
             if os.path.normpath(doc.thumbnail_path) in present_files:
                 present_files.remove(os.path.normpath(doc.thumbnail_path))
             try:
                 with doc.thumbnail_file as f:
                     f.read()
             except OSError as e:
-                messages.append(SanityError(
+                messages.error(
                     f"Cannot read thumbnail file of document {doc.pk}: {e}"
-                ))
+                )
 
         # Check sanity of the original file
         # TODO: extract method
         if not os.path.isfile(doc.source_path):
-            messages.append(SanityError(
-                f"Original of document {doc.pk} does not exist."))
+            messages.error(f"Original of document {doc.pk} does not exist.")
         else:
             if os.path.normpath(doc.source_path) in present_files:
                 present_files.remove(os.path.normpath(doc.source_path))
             try:
                 with doc.source_file as f:
                     checksum = hashlib.md5(f.read()).hexdigest()
             except OSError as e:
-                messages.append(SanityError(
-                    f"Cannot read original file of document {doc.pk}: {e}"))
+                messages.error(
+                    f"Cannot read original file of document {doc.pk}: {e}")
             else:
                 if not checksum == doc.checksum:
-                    messages.append(SanityError(
+                    messages.error(
                         f"Checksum mismatch of document {doc.pk}. "
                         f"Stored: {doc.checksum}, actual: {checksum}."
-                    ))
+                    )
 
         # Check sanity of the archive file.
         if doc.archive_checksum and not doc.archive_filename:
-            messages.append(SanityError(
+            messages.error(
                 f"Document {doc.pk} has an archive file checksum, but no "
                 f"archive filename."
-            ))
+            )
         elif not doc.archive_checksum and doc.archive_filename:
-            messages.append(SanityError(
+            messages.error(
                 f"Document {doc.pk} has an archive file, but its checksum is "
                 f"missing."
-            ))
+            )
         elif doc.has_archive_version:
             if not os.path.isfile(doc.archive_path):
-                messages.append(SanityError(
+                messages.error(
                     f"Archived version of document {doc.pk} does not exist."
-                ))
+                )
             else:
                 if os.path.normpath(doc.archive_path) in present_files:
                     present_files.remove(os.path.normpath(doc.archive_path))
                 try:
                     with doc.archive_file as f:
                         checksum = hashlib.md5(f.read()).hexdigest()
                 except OSError as e:
-                    messages.append(SanityError(
+                    messages.error(
                         f"Cannot read archive file of document {doc.pk}: {e}"
-                    ))
+                    )
                 else:
                     if not checksum == doc.archive_checksum:
-                        messages.append(SanityError(
+                        messages.error(
                             f"Checksum mismatch of archived document "
                             f"{doc.pk}. "
-                            f"Stored: {doc.checksum}, actual: {checksum}."
-                        ))
+                            f"Stored: {doc.archive_checksum}, "
+                            f"actual: {checksum}."
+                        )
 
         # other document checks
         if not doc.content:
-            messages.append(SanityWarning(
-                f"Document {doc.pk} has no content."
-            ))
+            messages.info(f"Document {doc.pk} has no content.")
 
     for extra_file in present_files:
-        messages.append(SanityWarning(
-            f"Orphaned file in media dir: {extra_file}"
-        ))
+        messages.warning(f"Orphaned file in media dir: {extra_file}")
 
     return messages

src/documents/tasks.py

@@ -9,8 +9,7 @@
 from documents.classifier import DocumentClassifier, load_classifier
 from documents.consumer import Consumer, ConsumerError
 from documents.models import Document, Tag, DocumentType, Correspondent
-from documents.sanity_checker import SanityFailedError
-
+from documents.sanity_checker import SanityCheckFailedException
 
 logger = logging.getLogger("paperless.tasks")
 
@@ -94,8 +93,15 @@ def consume_file(path,
 def sanity_check():
     messages = sanity_checker.check_sanity()
 
-    if len(messages) > 0:
-        raise SanityFailedError(messages)
+    messages.log_messages()
+
+    if messages.has_error():
+        raise SanityCheckFailedException(
+            "Sanity check failed with errors. See log.")
+    elif messages.has_warning():
+        return "Sanity check exited with warnings. See log."
+    elif len(messages) > 0:
+        return "Sanity check exited with infos. See log."
     else:
         return "No issues detected."
 

src/documents/tests/test_management.py

@@ -159,33 +159,20 @@ def test_create_classifier(self, m):
 
 class TestSanityChecker(DirectoriesMixin, TestCase):
 
-    def test_no_errors(self):
+    def test_no_issues(self):
         with self.assertLogs() as capture:
             call_command("document_sanity_checker")
 
         self.assertEqual(len(capture.output), 1)
-        self.assertIn("No issues found.", capture.output[0])
+        self.assertIn("Sanity checker detected no issues.", capture.output[0])
 
-    @mock.patch("documents.management.commands.document_sanity_checker.logger.warning")
-    @mock.patch("documents.management.commands.document_sanity_checker.logger.error")
-    def test_warnings(self, error, warning):
-        doc = Document.objects.create(title="test", filename="test.pdf", checksum="d41d8cd98f00b204e9800998ecf8427e")
-        Path(doc.source_path).touch()
-        Path(doc.thumbnail_path).touch()
-
-        call_command("document_sanity_checker")
-
-        error.assert_not_called()
-        warning.assert_called()
-
-    @mock.patch("documents.management.commands.document_sanity_checker.logger.warning")
-    @mock.patch("documents.management.commands.document_sanity_checker.logger.error")
-    def test_errors(self, error, warning):
+    def test_errors(self):
         doc = Document.objects.create(title="test", content="test", filename="test.pdf", checksum="abc")
         Path(doc.source_path).touch()
         Path(doc.thumbnail_path).touch()
 
-        call_command("document_sanity_checker")
+        with self.assertLogs() as capture:
+            call_command("document_sanity_checker")
 
-        warning.assert_not_called()
-        error.assert_called()
+        self.assertEqual(len(capture.output), 1)
+        self.assertIn("Checksum mismatch of document", capture.output[0])