Make ignore error option configurable (#236)

kuzudb · Nov 13, 2024 · f1a4b40 · f1a4b40
1 parent ec35331
commit f1a4b40
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 13 deletions.
diff --git a/kuzu b/kuzu
diff --git a/src/components/ImporterView/ImporterMainView.vue b/src/components/ImporterView/ImporterMainView.vue
@@ -300,6 +300,7 @@ export default {
           currentFile.format.ListStart = '[';
           currentFile.format.ListEnd = ']';
           currentFile.format.Parallelism = true;
+          currentFile.format.IgnoreErrors = true;
         }
         currentFile.format.Columns.forEach((c, i) => {
           c.type = DuckDB.convertDuckDBTypeToKuzuType(c.type);
@@ -358,8 +359,9 @@ export default {
       const listBegin = file.format.ListStart;
       const listEnd = file.format.ListEnd;
       const parallelism = file.format.Parallelism ? 'true' : 'false';
+      const ignoreErrors = file.format.IgnoreErrors ? 'true' : 'false';
       this.$refs.csvFormatModal.setFormat(
-        key, delimiter, quote, escape, hasHeader, listBegin, listEnd, parallelism
+        key, delimiter, quote, escape, hasHeader, listBegin, listEnd, parallelism, ignoreErrors
       );
       this.$refs.csvFormatModal.showModal();
     },
@@ -373,6 +375,7 @@ export default {
       const listBegin = format.listBegin;
       const listEnd = format.listEnd;
       const parallelism = format.parallelism;
+      const ignoreErrors = format.ignoreErrors;
 
       const columns = await DuckDB.getCsvHeaderWithCustomSettings(key, delimiter, quote, escape, hasHeader);
       columns.forEach((c, i) => {
@@ -388,6 +391,7 @@ export default {
       file.format.ListEnd = listEnd;
       file.format.Parallelism = parallelism;
       file.format.Columns = columns;
+      file.format.IgnoreErrors = ignoreErrors;
       file.detectedFormat.Columns = columns;
       if (file.type === 'node') {
         if (file.format.Columns[0]) {
@@ -574,6 +578,7 @@ export default {
             listBegin: rawFile.format.ListStart,
             listEnd: rawFile.format.ListEnd,
             parallelism: rawFile.format.Parallelism,
+            ignoreErrors: rawFile.format.IgnoreErrors,
           }
         }
         summary.push(file);

diff --git a/src/components/ImporterView/ImporterViewCsvFormatModal.vue b/src/components/ImporterView/ImporterViewCsvFormatModal.vue
@@ -68,7 +68,22 @@
 
             <div class="input-group mb-3">
               <span class="input-group-text">Read CSV in Parallel?</span>
-              <select class="form-select">
+              <select
+                v-model="parallelism"
+                class="form-select"
+              >
+                <option value="true">
+                  Yes
+                </option>
+                <option value="false">
+                  No
+                </option>
+              </select>
+              <span class="input-group-text">Ignore Errors?</span>
+              <select
+                v-model="ignoreErrors"
+                class="form-select"
+              >
                 <option value="true">
                   Yes
                 </option>
@@ -135,6 +150,7 @@ export default {
     listBegin: null,
     listEnd: null,
     parallelism: null,
+    ignoreErrors: null,
   }),
   computed: {
   },
@@ -149,7 +165,7 @@ export default {
       this.modal.show();
     },
 
-    setFormat(fileKey, delimiter, quote, escape, hasHeader, listBegin, listEnd, parallelism) {
+    setFormat(fileKey, delimiter, quote, escape, hasHeader, listBegin, listEnd, parallelism, ignoreErrors) {
       const convertToEscapedString = (str) => {
         if(str === '"' || str === "'") {
           return str;
@@ -165,6 +181,7 @@ export default {
       this.listBegin = convertToEscapedString(listBegin);
       this.listEnd = convertToEscapedString(listEnd);
       this.parallelism = String(parallelism);
+      this.ignoreErrors = String(ignoreErrors);
     },
 
     getFormat() {
@@ -184,6 +201,7 @@ export default {
         listBegin: processStringWithEscape(this.listBegin),
         listEnd: processStringWithEscape(this.listEnd),
         parallelism: this.parallelism === "true",
+        ignoreErrors: this.ignoreErrors === "true",
       };
 
     },

diff --git a/src/server/Import.js b/src/server/Import.js
@@ -116,6 +116,14 @@ router.post("/:job_id/exec", async (req, res) => {
         throw err;
       }
     }
+    try {
+      await DataImportUtil.deleteTmp(jobId);
+    } catch (err) {
+      // Ignore
+    }
+  } catch (err) {
+    // Ignore 
+  } finally {
     const numberOfWarningsQueryResult = await conn.query("CALL SHOW_WARNINGS() RETURN COUNT(*) AS count;");
     const numberOfWarnings = (await numberOfWarningsQueryResult.getNext()).count;
     job.numberOfWarnings = numberOfWarnings;
@@ -141,12 +149,6 @@ router.post("/:job_id/exec", async (req, res) => {
         };
       });
     }
-    try {
-      await DataImportUtil.deleteTmp(jobId);
-    } catch (err) {
-      // Ignore
-    }
-  } finally {
     database.releaseConnection(conn);
   }
 });

diff --git a/src/utils/DataDefinitionLanguage.js b/src/utils/DataDefinitionLanguage.js
@@ -141,7 +141,7 @@ class DataDefinitionLanguage {
   }
 
   getCsvOptionsSubquery(csvFormatOptions) {
-    const { delimiter, quote, escape, hasHeader, listBegin, listEnd, parallelism } = csvFormatOptions;
+    const { delimiter, quote, escape, hasHeader, listBegin, listEnd, parallelism, ignoreErrors } = csvFormatOptions;
     let csvOptions = [];
     csvOptions.push(`HEADER=${hasHeader}`);
     csvOptions.push(`DELIM="${this._jsonEscapedString(delimiter)}"`);
@@ -150,8 +150,7 @@ class DataDefinitionLanguage {
     // csvOptions.push(`LIST_BEGIN="${this._jsonEscapedString(listBegin)}"`);
     // csvOptions.push(`LIST_END="${this._jsonEscapedString(listEnd)}"`);
     csvOptions.push(`PARALLEL=${parallelism}`);
-    // Always ignore errors for data import UI
-    csvOptions.push(`IGNORE_ERRORS=true`);
+    csvOptions.push(`IGNORE_ERRORS=${ignoreErrors}`);
     const csvOptionsString = `(${csvOptions.join(", ")})`;
     return csvOptionsString;
   }