Merge pull request #241 from OpenImaging/global-import-export

Global import export

client/src/components/ProjectSettings.vue

@@ -17,10 +17,12 @@ export default defineComponent({
 
     const importPath = ref('');
     const exportPath = ref('');
+    const globalImportExport = ref(false);
     watchEffect(() => {
       djangoRest.settings(currentProject.value.id).then((settings) => {
         importPath.value = settings.importPath;
         exportPath.value = settings.exportPath;
+        globalImportExport.value = settings.globalImportExport;
       });
     });
 
@@ -37,6 +39,7 @@ export default defineComponent({
         await djangoRest.setSettings(currentProject.value.id, {
           importPath: importPath.value,
           exportPath: exportPath.value,
+          globalImportExport: globalImportExport.value,
         });
         changed.value = false;
       } catch (e) {
@@ -57,6 +60,7 @@ export default defineComponent({
       currentProject,
       importPath,
       exportPath,
+      globalImportExport,
       changed,
       importPathError,
       exportPathError,
@@ -120,6 +124,29 @@ export default defineComponent({
         />
       </v-flex>
     </v-layout>
+    <v-layout>
+      <v-switch
+        v-model="globalImportExport"
+        @click="changed = true"
+        color="primary"
+        label="Global import/export"
+      />
+      <v-tooltip top>
+        <template v-slot:activator="{ on, attrs }">
+          <v-icon
+            v-bind="attrs"
+            v-on="on"
+            color="primary"
+            small
+            class="mx-1"
+          >
+            mdi-information-outline
+          </v-icon>
+        </template>
+        Global imports/exports will use the project name from the import file, which will
+        potentially modify other projects.
+      </v-tooltip>
+    </v-layout>
     <v-btn
       :disabled="!changed"
       v-if="user.is_superuser"

client/src/types.ts

@@ -57,7 +57,8 @@ interface Project {
 
 interface Settings {
   importPath: string,
-  exportPath: string
+  exportPath: string,
+  globalImportExport: boolean
 }
 
 export {

miqa/core/conversion/import_export_csvs.py

@@ -2,6 +2,8 @@
 
 from schema import And, Schema, SchemaError, Use
 
+from miqa.core.models import Project
+
 IMPORT_CSV_COLUMNS = [
     'project_name',
     'experiment_name',
@@ -30,7 +32,7 @@ def validate_file_locations(input_dict, project):
     return input_dict
 
 
-def validate_import_dict(import_dict, project):
+def validate_import_dict(import_dict, project: Project):
     import_schema = Schema(
         {
             'projects': {
@@ -52,9 +54,13 @@ def validate_import_dict(import_dict, project):
     try:
         import_schema.validate(import_dict)
         import_dict = validate_file_locations(import_dict, project)
-        return import_dict
     except SchemaError:
         raise ValueError(f'Invalid format of import file {project.import_path}')
+    if project.global_import_export:
+        for project_name in import_dict['projects']:
+            if not Project.objects.filter(name=project_name).exists():
+                raise ValueError(f'Project {project_name} does not exist')
+    return import_dict
 
 
 def import_dataframe_to_dict(df):

miqa/core/migrations/0022_project_global_import_export.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.9 on 2021-11-30 22:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0021_update_qc_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='global_import_export',
+            field=models.BooleanField(default=False),
+        ),
+    ]

miqa/core/models/project.py

@@ -37,6 +37,7 @@ class Project(TimeStampedModel, models.Model):
     archived = models.BooleanField(default=False)
     import_path = models.CharField(max_length=500)
     export_path = models.CharField(max_length=500)
+    global_import_export = models.BooleanField(default=False)
     evaluation_models = models.JSONField(default=default_evaluation_model_mapping)
 
     def __str__(self):

miqa/core/rest/project.py

@@ -16,10 +16,11 @@
 class ProjectSettingsSerializer(serializers.ModelSerializer):
     class Meta:
         model = Project
-        fields = ['importPath', 'exportPath', 'permissions']
+        fields = ['importPath', 'exportPath', 'globalImportExport', 'permissions']
 
     importPath = serializers.CharField(source='import_path')  # noqa: N815
     exportPath = serializers.CharField(source='export_path')  # noqa: N815
+    globalImportExport = serializers.BooleanField(source='global_import_export')  # noqa: N815
     permissions = serializers.SerializerMethodField('get_permissions')
 
     def get_permissions(self, obj):
@@ -117,6 +118,7 @@ def settings_(self, request, **kwargs):
 
             project.import_path = request.data['importPath']
             project.export_path = request.data['exportPath']
+            project.global_import_export = request.data['globalImportExport']
             project.full_clean()
             project.save()
         serializer = ProjectSettingsSerializer(project)

miqa/core/tasks.py

@@ -68,11 +68,13 @@ def perform_import(import_dict, project_id):
     # when multi-project imports are supported
     project = Project.objects.get(id=project_id)
 
-    for _project_name, project_data in import_dict['projects'].items():
-        # Switch these lines to support multi-project imports
-        # project_object = Project(name=_project_name)
-        # new_projects.append(project_object)
-        project_object = project
+    for project_name, project_data in import_dict['projects'].items():
+        if project.global_import_export:
+            # A global import uses the project name column to determine which project to import to
+            project_object = Project.objects.get(name=project_name)
+        else:
+            # A normal import ignores the project name column
+            project_object = project
 
         # delete old imports of these projects
         Experiment.objects.filter(
@@ -107,28 +109,49 @@ def perform_import(import_dict, project_id):
 
 
 def export_data(project_id):
-    project_object = Project.objects.get(id=project_id)
-    parent_location = Path(project_object.export_path).parent
+    project = Project.objects.get(id=project_id)
+    parent_location = Path(project.export_path).parent
     if not parent_location.exists():
         raise ValueError(f'No such location {parent_location} to create export file.')
-    perform_export.delay(project_id)
+
+    # In the event of a global export, we only want to export the projects listed in the import
+    # file. Read the import file now and extract the project names.
+    if project.import_path.endswith('.csv'):
+        import_dict = import_dataframe_to_dict(pandas.read_csv(project.import_path))
+    elif project.import_path.endswith('.json'):
+        import_dict = json.load(open(project.import_path))
+    else:
+        raise ValueError(f'Invalid import file {project.import_path}.')
+
+    import_dict = validate_import_dict(import_dict, project)
+    project_names = import_dict['projects'].keys()
+
+    perform_export.delay(project_id, project_names)
 
 
 @shared_task
-def perform_export(project_id):
-    project_object = Project.objects.get(id=project_id)
+def perform_export(project_id, project_names):
+    project = Project.objects.get(id=project_id)
     data = []
 
-    for frame_object in Frame.objects.filter(scan__experiment__project=project_object):
-        data.append(
-            [
-                project_object.name,
-                frame_object.scan.experiment.name,
-                frame_object.scan.name,
-                frame_object.scan.scan_type,
-                frame_object.frame_number,
-                frame_object.raw_path,
-            ]
-        )
+    if project.global_import_export:
+        # A global export should export all projects listed in the import file
+        projects = [Project.objects.get(name=project_name) for project_name in project_names]
+    else:
+        # A normal export should only export the current project
+        projects = [project]
+
+    for project_object in projects:
+        for frame_object in Frame.objects.filter(scan__experiment__project=project_object):
+            data.append(
+                [
+                    project_object.name,
+                    frame_object.scan.experiment.name,
+                    frame_object.scan.name,
+                    frame_object.scan.scan_type,
+                    frame_object.frame_number,
+                    frame_object.raw_path,
+                ]
+            )
     export_df = pandas.DataFrame(data, columns=IMPORT_CSV_COLUMNS)
     export_df.to_csv(project_object.export_path, index=False)

miqa/core/tests/factories.py

@@ -25,6 +25,7 @@ class Meta:
 
     import_path = factory.Faker('file_path')
     export_path = factory.Faker('file_path')
+    global_import_export = False
 
 
 class ExperimentFactory(factory.django.DjangoModelFactory):