Merge branch 'master' into manual-validator-workflow

.github/bors.toml

@@ -2,6 +2,7 @@ status = [
     "linting",
     "build-frontend",
     "build-docs",
+    "build-docker",
     "test-windows",
     "test-linux (sqlite)",
     "test-linux (mariadb)",

.github/workflows/main.yml

@@ -142,6 +142,16 @@ jobs:
           cd ESSArch_Core/docs
           make html SPHINXOPTS="-W"
 
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Build
+        run: |
+          cd docker
+          docker-compose build essarch
+
   test-windows:
     runs-on: windows-latest
     env:

ESSArch_Core/WorkflowEngine/util.py

@@ -55,7 +55,7 @@ def _create_step(parent_step, flow, ip, responsible, context=None):
 
             child_s = ProcessStep.objects.create(
                 name=flow_entry['name'],
-                parallel=flow_entry['parallel'],
+                parallel=flow_entry.get('parallel', False),
                 parent_step=parent_step,
                 parent_step_pos=e_idx,
                 eager=parent_step.eager,

ESSArch_Core/config/settings.py

@@ -36,7 +36,7 @@
 
 
 # Set test runner
-TEST_RUNNER = "ESSArch_Core.testing.runner.QuietTestRunner"
+TEST_RUNNER = "ESSArch_Core.testing.runner.ESSArchTestRunner"
 
 ALLOWED_HOSTS = ['*']
 

ESSArch_Core/db/utils.py

@@ -0,0 +1,27 @@
+from django.db.models import Case, CharField, F, IntegerField, Value, When
+from django.db.models.functions import Cast, Length, StrIndex, Substr, Trim
+
+
+def natural_sort(qs, field):
+    return qs.annotate(
+        ns_len=Length(field),
+        ns_split_index=StrIndex(field, Value(' ')),
+        ns_suffix=Trim(Substr(field, F('ns_split_index'), output_field=CharField())),
+    ).annotate(
+        ns_code=Trim(Substr(field, Value(1), 'ns_split_index', output_field=CharField())),
+        ns_weight=Case(
+            When(ns_split_index=0, then=Value(0)),
+            default=Case(
+                When(
+                    ns_suffix__regex=r'^\d+$',
+                    then=Cast(
+                        Substr(field, F('ns_split_index'), output_field=CharField()),
+                        output_field=IntegerField(),
+                    )
+                ),
+                default=Value(1230),
+                output_field=IntegerField()
+            ),
+            output_field=IntegerField(),
+        )
+    ).order_by('ns_code', 'ns_weight', 'ns_len', field)

ESSArch_Core/docs/conf.py

@@ -119,7 +119,7 @@
 # The short X.Y version.
 version = get_versions()['version']
 # The full version, including alpha/beta/rc tags.
-release = get_versions()['full-revisionid']
+release = get_versions()['full-revisionid'] or ''
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

ESSArch_Core/frontend/static/frontend/scripts/controllers/SearchDetailCtrl.js

@@ -237,18 +237,20 @@ export default class SearchDetailCtrl {
     vm.getClassificationStructureChildren = function(id) {
       console.log('Getting children of structure with id "' + id + '"');
       const url = vm.url + 'structures/' + id + '/units/';
-      return $http.get(url, {params: {has_parent: false, pager: 'none'}}).then(function(response) {
-        const data = response.data.map(function(unit) {
-          unit._id = unit.id;
-          unit._is_structure_unit = true;
-          delete unit.parent;
-          return vm.createNode(unit);
+      return $http
+        .get(url, {params: {has_parent: false, ordering: 'reference_code', pager: 'none'}})
+        .then(function(response) {
+          const data = response.data.map(function(unit) {
+            unit._id = unit.id;
+            unit._is_structure_unit = true;
+            delete unit.parent;
+            return vm.createNode(unit);
+          });
+          return {
+            data: data,
+            count: response.headers('Count'),
+          };
         });
-        return {
-          data: data,
-          count: response.headers('Count'),
-        };
-      });
     };
 
     vm.createPlaceholderNode = function() {

ESSArch_Core/frontend/static/frontend/views/administration_media_information.html

@@ -45,16 +45,16 @@ <h4>
       <table class="table table-striped" style="margin-bottom: 0px;">
         <thead>
           <tr>
-            <th class="clickable" st-sort="medium_id" class="column-label">ID</th>
+            <th class="clickable" st-sort="medium_id" st-sort-default="true" class="column-label">ID</th>
             <th class="clickable" st-sort="storage_target" class="column-label">{{'STORAGETARGET' | translate}}</th>
-            <th class="clickable" st-sort="create_date" st-sort-default="reverse" class="column-label">
+            <th class="clickable" st-sort="create_date" class="column-label">
               {{'CREATED' | translate}}
             </th>
             <th class="clickable" st-sort="status" class="column-label">{{'STATUS' | translate}}</th>
             <th class="clickable" st-sort="location" class="column-label">{{'LOCATION' | translate}}</th>
             <th class="clickable" st-sort="location_status" class="column-label">{{'LOCATIONSTATUS' | translate}}</th>
             <th class="clickable" st-sort="used_capacity" class="column-label">{{'USEDCAPACITY' | translate}}</th>
-            <th class="clickable" st-sort="storage_target.max_capacity" class="column-label">
+            <th class="clickable" st-sort="max_capacity" class="column-label">
               {{'MAXCAPACITY' | translate}}
             </th>
           </tr>

ESSArch_Core/storage/filters.py

@@ -23,6 +23,7 @@
 """
 
 from django_filters import rest_framework as filters
+from django_filters.constants import EMPTY_VALUES
 
 from ESSArch_Core.api.filters import CharSuffixRangeFilter, ListFilter
 from ESSArch_Core.configuration.models import StoragePolicy
@@ -36,6 +37,16 @@
 )
 
 
+class StorageMediumOrderingFilter(filters.OrderingFilter):
+    def filter(self, qs, value):
+        if value in EMPTY_VALUES or 'medium_id' in value:
+            return qs.natural_sort()
+        elif '-medium_id' in value:
+            return qs.natural_sort().reverse()
+
+        return super().filter(qs, value)
+
+
 class StorageMediumFilter(filters.FilterSet):
     status = ListFilter(field_name='status', distinct='true')
     medium_type = filters.ChoiceFilter(field_name='storage_target__type', choices=medium_type_CHOICES)
@@ -68,11 +79,12 @@ def filter_migratable(self, queryset, name, value):
         else:
             return queryset.non_migratable()
 
-    ordering = filters.OrderingFilter(
+    ordering = StorageMediumOrderingFilter(
         fields=(
             ('id', 'id'),
             ('medium_id', 'medium_id'),
             ('storage_target__name', 'storage_target'),
+            ('storage_target__max_capacity', 'max_capacity'),
             ('status', 'status'),
             ('location', 'location'),
             ('location_status', 'location_status'),

ESSArch_Core/storage/models.py

@@ -38,6 +38,7 @@
 )
 
 from ESSArch_Core.configuration.models import Parameter, Path
+from ESSArch_Core.db.utils import natural_sort
 from ESSArch_Core.fixity.validation.backends.checksum import ChecksumValidator
 from ESSArch_Core.storage.backends import get_backend
 from ESSArch_Core.storage.copy import copy_file
@@ -508,6 +509,9 @@ def migratable(self):
     def non_migratable(self):
         return self.exclude(pk__in=self.migratable())
 
+    def natural_sort(self):
+        return natural_sort(self, 'medium_id')
+
     def fastest(self):
         container = Case(
             When(storage_target__methods__containers=False, then=Value(1)),

ESSArch_Core/storage/tests/test_views.py

@@ -37,6 +37,72 @@
 User = get_user_model()
 
 
+class StorageMediumListTests(APITestCase):
+    def setUp(self):
+        user = User.objects.create(username='user')
+        self.client = APIClient()
+        self.client.force_authenticate(user=user)
+        self.url = reverse('storagemedium-list')
+
+    def test_sort_medium_id(self):
+        storage_target = StorageTarget.objects.create()
+
+        medium_ids = [
+            'BA 100',
+            'ABC',
+            'X 1',
+            'BA 1001',
+            'XY 1',
+            'XYZ 1',
+            'BA 1',
+            'BA 10',
+            'BA 2',
+            'BA 1002',
+            'BA 1000',
+            'BA 003',
+            'QWE1',
+            'QWE10',
+            'QWE2',
+            'A B 2',
+            'A B 10',
+            'DEF',
+            'A B 1',
+        ]
+
+        for mid in medium_ids:
+            StorageMedium.objects.create(
+                medium_id=mid, storage_target=storage_target,
+                status=20, location_status=50, block_size=1024, format=103,
+            )
+
+        res = self.client.get(self.url, {'ordering': 'medium_id', 'pager': 'none'})
+
+        expected = [
+            'ABC',
+            'DEF',
+            'QWE1',
+            'QWE2',
+            'QWE10',
+            'A B 1',
+            'A B 2',
+            'A B 10',
+            'BA 1',
+            'BA 2',
+            'BA 003',
+            'BA 10',
+            'BA 100',
+            'BA 1000',
+            'BA 1001',
+            'BA 1002',
+            'X 1',
+            'XY 1',
+            'XYZ 1',
+        ]
+        actual = [sm['medium_id'] for sm in res.data]
+
+        self.assertEqual(expected, actual)
+
+
 class StorageMediumDeactivatableTests(TestCase):
     @classmethod
     def setUpTestData(cls):

ESSArch_Core/storage/views.py

@@ -285,14 +285,14 @@ class StorageMediumViewSet(viewsets.ModelViewSet):
     API endpoint for storage medium
     """
     queryset = StorageMedium.objects.all()
+
     serializer_class = StorageMediumSerializer
-    filter_backends = (filters.OrderingFilter, DjangoFilterBackend, SearchFilter)
+    filter_backends = (DjangoFilterBackend, SearchFilter)
     filterset_class = StorageMediumFilter
 
     search_fields = (
         '=id', 'medium_id', 'status', 'location', 'location_status', 'used_capacity', 'create_date',
     )
-    ordering = ('-create_date',)
 
     def get_serializer_class(self):
         if self.request.method in permissions.SAFE_METHODS:

ESSArch_Core/tags/filters.py

@@ -1,5 +1,6 @@
 from django.db.models import OuterRef, Subquery
 from django_filters import rest_framework as filters
+from django_filters.constants import EMPTY_VALUES
 
 from ESSArch_Core.tags.models import Structure, StructureUnit, Tag
 
@@ -26,9 +27,29 @@ class Meta:
         fields = ['type', 'is_template', 'published', 'archive']
 
 
+class StructureUnitOrderingFilter(filters.OrderingFilter):
+    def filter(self, qs, value):
+        if value in EMPTY_VALUES:
+            return qs
+
+        if 'reference_code' in value:
+            return qs.natural_sort()
+        elif '-reference_code' in value:
+            return qs.natural_sort().reverse()
+
+        return super().filter(qs, value)
+
+
 class StructureUnitFilter(filters.FilterSet):
     has_parent = filters.BooleanFilter(field_name='parent', lookup_expr='isnull', exclude=True)
 
+    ordering = StructureUnitOrderingFilter(
+        fields=(
+            ('reference_code', 'reference_code'),
+            ('name', 'name'),
+        ),
+    )
+
     class Meta:
         model = StructureUnit
         fields = ['has_parent', 'structure']

ESSArch_Core/tags/models.py

@@ -11,10 +11,12 @@
 from elasticsearch_dsl.connections import get_connection
 from mptt.managers import TreeManager
 from mptt.models import MPTTModel, TreeForeignKey
+from mptt.querysets import TreeQuerySet
 from relativity.mptt import MPTTSubtree
 
 from ESSArch_Core.agents.models import Agent
 from ESSArch_Core.auth.util import get_objects_for_user
+from ESSArch_Core.db.utils import natural_sort
 from ESSArch_Core.fields import JSONField
 from ESSArch_Core.managers import OrganizationManager
 from ESSArch_Core.profiles.models import SubmissionAgreement
@@ -404,8 +406,16 @@ class Meta:
         unique_together = ('structure_unit_a', 'structure_unit_b', 'type')  # Avoid duplicates within same type
 
 
+class StructureUnitQueryset(TreeQuerySet):
+    def natural_sort(self):
+        return natural_sort(self, 'reference_code')
+
+
 class StructureUnitManager(TreeManager, OrganizationManager):
-    pass
+    def get_queryset(self, *args, **kwargs):
+        return StructureUnitQueryset(self.model, using=self._db).order_by(
+            self.tree_id_attr, self.left_attr
+        )
 
 
 class StructureUnit(MPTTModel):
@@ -905,6 +915,9 @@ def for_user(self, user, perms=None):
         else:
             return qs.filter(Q(Q(security_level=0) | Q(security_level__isnull=True)))
 
+    def natural_sort(self):
+        return natural_sort(self, 'reference_code')
+
 
 class TagVersionManager(OrganizationManager):
     def get_queryset(self):