Use setting to determine if null dates should be replaced (#591)

* Use setting to determine if null dates should be replaced with current time upon import

* Maintain same scan decision `created` text between imports and exports

* Use dateparser and remove print statements

* Fix import order for isort

* Include installation of types-dateparser

miqa/core/migrations/0035_allow_null_decision_creation_times.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.13 on 2022-09-16 13:55
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0034_CT_scan_type'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='scandecision',
+            name='created',
+            field=models.DateTimeField(default=django.utils.timezone.now, null=True),
+        ),
+    ]

miqa/core/models/scan_decision.py

@@ -56,7 +56,7 @@ class Meta:
         ]
 
     id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
-    created = models.DateTimeField(default=timezone.now)
+    created = models.DateTimeField(default=timezone.now, null=True)
     scan = models.ForeignKey('Scan', related_name='decisions', on_delete=models.CASCADE)
     creator = models.ForeignKey(User, on_delete=models.PROTECT, null=True, blank=True)
     decision = models.CharField(max_length=2, choices=DECISION_CHOICES, blank=False)

miqa/core/tasks.py

@@ -1,4 +1,4 @@
-import datetime
+from datetime import datetime
 from io import BytesIO, StringIO
 import json
 from pathlib import Path
@@ -9,9 +9,9 @@
 from botocore import UNSIGNED
 from botocore.client import Config
 from celery import shared_task
+import dateparser
 from django.conf import settings
 from django.contrib.auth.models import User
-from django.utils import timezone
 import pandas
 from rest_framework.exceptions import APIException
 
@@ -213,19 +213,16 @@ def perform_import(import_dict):
                         except User.DoesNotExist:
                             creator = None
                         note = ''
-                        created = timezone.now()
+                        created = (
+                            datetime.now() if settings.REPLACE_NULL_CREATION_DATETIMES else None
+                        )
                         location = {}
                         if last_decision_dict['note']:
                             note = last_decision_dict['note'].replace(';', ',')
                         if last_decision_dict['created']:
-                            try:
-                                datetime.datetime.strptime(
-                                    last_decision_dict['created'], '%Y-%m-%d'
-                                )
-                            except ValueError:
-                                created = timezone.now()
-                            else:
-                                created = last_decision_dict['created']
+                            valid_dt = dateparser.parse(last_decision_dict['created'])
+                            if valid_dt:
+                                created = valid_dt.strftime('%Y-%m-%d %H:$M')
                         if last_decision_dict['location'] and last_decision_dict['location'] != '':
                             slices = [
                                 axis.split('=')[1]
@@ -347,7 +344,11 @@ def perform_export(project_id: Optional[str]):
                 ]
                 # if a last decision exists for the scan, encode that decision on this row
                 # ... U, [email protected], note; with; commas; replaced, artifact_1;artifact_2
-                last_decision = frame_object.scan.decisions.order_by('created').last()
+                last_decision = (
+                    frame_object.scan.decisions.filter(created__isnull=False)
+                    .order_by('created')
+                    .last()
+                )
                 if last_decision:
                     location = ''
                     if last_decision.location:

miqa/settings.py

@@ -38,6 +38,8 @@ class MiqaMixin(ConfigMixin):
     DEMO_MODE = values.BooleanValue(environ=True, default=False)
     # It is recommended to enable the following for demo mode:
     NORMAL_USERS_CAN_CREATE_PROJECTS = values.BooleanValue(environ=True, default=False)
+    # Enable the following to replace null creation times for scan decisions with import time
+    REPLACE_NULL_CREATION_DATETIMES = values.BooleanValue(environ=True, default=False)
 
     # Override default signup sheet to ask new users for first and last name
     ACCOUNT_FORMS = {'signup': 'miqa.core.rest.accounts.AccountSignupForm'}

setup.py

@@ -37,6 +37,7 @@
     include_package_data=True,
     install_requires=[
         'celery',
+        'dateparser',
         'django>=3.2,<4.0',
         'django-allauth',
         'django-auth-style[allauth]',
@@ -67,6 +68,7 @@
             'factory_boy',
             'girder-pytest-pyppeteer==0.0.9',
             'pytest-asyncio',
+            'types-dateparser',
         ],
         'learning': [
             'itk>=5.3rc4',