diff --git a/miqa/core/migrations/0035_allow_null_decision_creation_times.py b/miqa/core/migrations/0035_allow_null_decision_creation_times.py new file mode 100644 index 00000000..eb9f5c03 --- /dev/null +++ b/miqa/core/migrations/0035_allow_null_decision_creation_times.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-09-16 13:55 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0034_CT_scan_type'), + ] + + operations = [ + migrations.AlterField( + model_name='scandecision', + name='created', + field=models.DateTimeField(default=django.utils.timezone.now, null=True), + ), + ] diff --git a/miqa/core/models/scan_decision.py b/miqa/core/models/scan_decision.py index bd8987c1..b1cea128 100644 --- a/miqa/core/models/scan_decision.py +++ b/miqa/core/models/scan_decision.py @@ -56,7 +56,7 @@ class Meta: ] id = models.UUIDField(primary_key=True, default=uuid4, editable=False) - created = models.DateTimeField(default=timezone.now) + created = models.DateTimeField(default=timezone.now, null=True) scan = models.ForeignKey('Scan', related_name='decisions', on_delete=models.CASCADE) creator = models.ForeignKey(User, on_delete=models.PROTECT, null=True, blank=True) decision = models.CharField(max_length=2, choices=DECISION_CHOICES, blank=False) diff --git a/miqa/core/tasks.py b/miqa/core/tasks.py index c902c2cb..06775446 100644 --- a/miqa/core/tasks.py +++ b/miqa/core/tasks.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime from io import BytesIO, StringIO import json from pathlib import Path @@ -9,9 +9,9 @@ from botocore import UNSIGNED from botocore.client import Config from celery import shared_task +import dateparser from django.conf import settings from django.contrib.auth.models import User -from django.utils import timezone import pandas from rest_framework.exceptions import APIException @@ -213,19 +213,16 @@ def perform_import(import_dict): except User.DoesNotExist: creator = None note = '' - created = timezone.now() + created = ( + datetime.now() if settings.REPLACE_NULL_CREATION_DATETIMES else None + ) location = {} if last_decision_dict['note']: note = last_decision_dict['note'].replace(';', ',') if last_decision_dict['created']: - try: - datetime.datetime.strptime( - last_decision_dict['created'], '%Y-%m-%d' - ) - except ValueError: - created = timezone.now() - else: - created = last_decision_dict['created'] + valid_dt = dateparser.parse(last_decision_dict['created']) + if valid_dt: + created = valid_dt.strftime('%Y-%m-%d %H:$M') if last_decision_dict['location'] and last_decision_dict['location'] != '': slices = [ axis.split('=')[1] @@ -347,7 +344,11 @@ def perform_export(project_id: Optional[str]): ] # if a last decision exists for the scan, encode that decision on this row # ... U, reviewer@miqa.dev, note; with; commas; replaced, artifact_1;artifact_2 - last_decision = frame_object.scan.decisions.order_by('created').last() + last_decision = ( + frame_object.scan.decisions.filter(created__isnull=False) + .order_by('created') + .last() + ) if last_decision: location = '' if last_decision.location: diff --git a/miqa/settings.py b/miqa/settings.py index 3722c99e..3581f14c 100644 --- a/miqa/settings.py +++ b/miqa/settings.py @@ -38,6 +38,8 @@ class MiqaMixin(ConfigMixin): DEMO_MODE = values.BooleanValue(environ=True, default=False) # It is recommended to enable the following for demo mode: NORMAL_USERS_CAN_CREATE_PROJECTS = values.BooleanValue(environ=True, default=False) + # Enable the following to replace null creation times for scan decisions with import time + REPLACE_NULL_CREATION_DATETIMES = values.BooleanValue(environ=True, default=False) # Override default signup sheet to ask new users for first and last name ACCOUNT_FORMS = {'signup': 'miqa.core.rest.accounts.AccountSignupForm'} diff --git a/setup.py b/setup.py index 4a8b0ccb..10621007 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ include_package_data=True, install_requires=[ 'celery', + 'dateparser', 'django>=3.2,<4.0', 'django-allauth', 'django-auth-style[allauth]', @@ -67,6 +68,7 @@ 'factory_boy', 'girder-pytest-pyppeteer==0.0.9', 'pytest-asyncio', + 'types-dateparser', ], 'learning': [ 'itk>=5.3rc4',