Merge pull request #75 from ncss-tech/osd-drainage

[OSD] Improve drainage class parsing

DESCRIPTION

@@ -9,7 +9,7 @@ Imports: curl, xml2, jsonlite, rvest, stringi, tibble, dplyr, pdftools, data.tab
 License: GPL-3
 Encoding: UTF-8
 LazyData: true
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.1
 Suggests: 
     knitr,
     rmarkdown,

R/parseOSD_functions.R

@@ -219,27 +219,41 @@
   # drainage classes, in order, lower case
   classes <- c("excessively", "somewhat excessively", "well", "moderately well",
                "somewhat poorly", "poorly", "very poorly", "subaqueous")
+  class_hyphen <- gsub(" ", "[ \\-]", classes)
 
   # combine into capturing REGEX
-  classes.regex <- paste0('(', paste(classes, collapse = '|'), ')', "( (to|or|and) )?",
-                          paste0('(', paste(classes, collapse = '|'), ')'), "? drained")
+  classes.regex <- paste0('(', paste(class_hyphen, collapse = '|'), ')', "([ \\-]drained)?( (to|or|and) )?",
+                          paste0('(', paste(class_hyphen, collapse = '|'), ')'),
+                          "?[ \\-]drained|subaqueous|Drainage[ class]*[:\\-]+ ",
+                          '(', paste(class_hyphen, collapse = '|'), ')', "([ \\-]drained)?( (to|or|and) )?",
+                          paste0('(', paste(class_hyphen, collapse = '|'), ')?'))
 
   # get matches
   m <- stringi::stri_match(text, regex = classes.regex, mode = 'first', opts_regex = list(case_insensitive = TRUE))
+  m <- gsub("Drainage[ Cclass]*[:\\-]+ ", "", m, ignore.case = TRUE)
 
   # fail gracefully in the case of no section data or no matches
   if (nrow(m) < 1) {
     return(NA)
   }
 
   # keep full match and convert to lower case, remove the word "drained"
-  m <- trimws(gsub("drained", "", tolower(m[, 1])))
-
-  # return as an ordered factor
-  # m <- factor(m, levels = classes, ordered = TRUE)
-  # factors cannot be preserved in JSON output, and wont work for multiple classes/ranges of classes
+  m <- trimws(gsub("  ", " ", gsub("-", " ", gsub("drained", "", tolower(m[, 1])))))
+
+  # put classes in order from excessively->subaqueous
+  # interpolate ranges across more than 2 classes, and concatenate with comma
+  m2 <- strsplit(m, "\\b(and|or|to)\\b")
+  m3 <- lapply(m2, function(x) {
+    x <- trimws(x)
+    y <- as.integer(factor(unique(classes[match(x, classes)]),
+                           levels = classes, ordered = TRUE))
+    if (length(y) > 1) {
+      y <- seq(from = min(y, na.rm = TRUE), to = max(y, na.rm = TRUE))
+    }
+    ifelse(is.na(classes[y]), "", classes[y]) # TODO: use zero chars or NA?
+  })
 
-  return(m)
+  return(sapply(m3, paste0, collapse = ", "))
 }
 
 .zerochar_to_na <- function(x) {

inst/extdata/OSD/A/AABERG.json

@@ -65,7 +65,7 @@
     [
       {
         "drainage": "moderately well",
-        "drainage_overview": "moderately well"
+        "drainage_overview": "well, moderately well"
       }
     ]
   ],

inst/extdata/OSD/A/ABAC.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ABAJO.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ABALAN.json

@@ -65,7 +65,7 @@
     [
       {
         "drainage": "well",
-        "drainage_overview": ""
+        "drainage_overview": "well"
       }
     ]
   ],

inst/extdata/OSD/A/ABERONE.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "somewhat excessively or well",
+        "drainage": "somewhat excessively, well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ABGESE.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ABIQUA.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ABREU.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ABSHER.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "well and moderately well",
-        "drainage_overview": "well and moderately well"
+        "drainage": "well, moderately well",
+        "drainage_overview": "well, moderately well"
       }
     ]
   ],

inst/extdata/OSD/A/ACADEMY.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "well to moderately well",
+        "drainage": "well, moderately well",
         "drainage_overview": ""
       }
     ]

inst/extdata/OSD/A/ACO.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "well to somewhat excessively",
-        "drainage_overview": "well to somewhat excessively"
+        "drainage": "somewhat excessively, well",
+        "drainage_overview": "somewhat excessively, well"
       }
     ]
   ],

inst/extdata/OSD/A/ADAMS.json

@@ -65,7 +65,7 @@
     [
       {
         "drainage": "somewhat excessively",
-        "drainage_overview": "excessively and somewhat excessively"
+        "drainage_overview": "excessively, somewhat excessively"
       }
     ]
   ],

inst/extdata/OSD/A/ADGER.json

@@ -65,7 +65,7 @@
     [
       {
         "drainage": "well",
-        "drainage_overview": "well"
+        "drainage_overview": "well, moderately well"
       }
     ]
   ],

inst/extdata/OSD/A/ADILIS.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "well to moderately well",
-        "drainage_overview": "well to moderately well"
+        "drainage": "well, moderately well",
+        "drainage_overview": "well, moderately well"
       }
     ]
   ],

inst/extdata/OSD/A/ADJIDAUMO.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "poorly and very poorly",
-        "drainage_overview": "poorly"
+        "drainage": "poorly, very poorly",
+        "drainage_overview": "poorly, very poorly"
       }
     ]
   ],

inst/extdata/OSD/A/ADOLPH.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "poorly and very poorly",
-        "drainage_overview": "poorly and very poorly"
+        "drainage": "poorly, very poorly",
+        "drainage_overview": "poorly, very poorly"
       }
     ]
   ],

inst/extdata/OSD/A/AGER.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/AGUA_DULCE.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": ""
       }
     ]

inst/extdata/OSD/A/AGUEDA.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "well and moderately well",
+        "drainage": "well, moderately well",
         "drainage_overview": ""
       }
     ]

inst/extdata/OSD/A/AHL.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/AILEY.json

@@ -65,7 +65,7 @@
     [
       {
         "drainage": "well",
-        "drainage_overview": "well"
+        "drainage_overview": "somewhat excessively, well"
       }
     ]
   ],

inst/extdata/OSD/A/AINAHOU.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "poorly or very poorly",
-        "drainage_overview": "poorly or very poorly"
+        "drainage": "poorly, very poorly",
+        "drainage_overview": "poorly, very poorly"
       }
     ]
   ],

inst/extdata/OSD/A/AKAN.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "poorly",
-        "drainage_overview": "poorly"
+        "drainage": "poorly, very poorly",
+        "drainage_overview": "poorly, very poorly"
       }
     ]
   ],

inst/extdata/OSD/A/ALAMOSA.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "poorly to somewhat poorly",
-        "drainage_overview": "poorly to somewhat poorly"
+        "drainage": "somewhat poorly, poorly",
+        "drainage_overview": "somewhat poorly, poorly"
       }
     ]
   ],

inst/extdata/OSD/A/ALANTHUS.json

@@ -65,7 +65,7 @@
     [
       {
         "drainage": "well",
-        "drainage_overview": ""
+        "drainage_overview": "well"
       }
     ]
   ],

inst/extdata/OSD/A/ALBATON.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "poorly or very poorly",
-        "drainage_overview": "poorly or very poorly"
+        "drainage": "poorly, very poorly",
+        "drainage_overview": "poorly, very poorly"
       }
     ]
   ],

inst/extdata/OSD/A/ALBINAS.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ALBION.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "well",
-        "drainage_overview": "somewhat excessively"
+        "drainage": "somewhat excessively, well",
+        "drainage_overview": "somewhat excessively, well"
       }
     ]
   ],

inst/extdata/OSD/A/ALBRIGHTS.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "moderately well to somewhat poorly",
-        "drainage_overview": "moderately well to somewhat poorly"
+        "drainage": "moderately well, somewhat poorly",
+        "drainage_overview": "moderately well, somewhat poorly"
       }
     ]
   ],

inst/extdata/OSD/A/ALCESTER.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "well and moderately well",
-        "drainage_overview": "well and moderately well"
+        "drainage": "well, moderately well",
+        "drainage_overview": "well, moderately well"
       }
     ]
   ],

inst/extdata/OSD/A/ALDA.json

@@ -64,8 +64,8 @@
   "SITE": [
     [
       {
-        "drainage": "somewhat poorly",
-        "drainage_overview": "somewhat poorly"
+        "drainage": "moderately well, somewhat poorly",
+        "drainage_overview": "moderately well, somewhat poorly"
       }
     ]
   ],

inst/extdata/OSD/A/ALDERMAND.json

@@ -65,7 +65,7 @@
     [
       {
         "drainage": "well",
-        "drainage_overview": ""
+        "drainage_overview": "well"
       }
     ]
   ],

inst/extdata/OSD/A/ALET.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ALGERITA.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]

inst/extdata/OSD/A/ALICE.json

@@ -64,7 +64,7 @@
   "SITE": [
     [
       {
-        "drainage": "",
+        "drainage": "well",
         "drainage_overview": "well"
       }
     ]