TADA_CreateParamRef() and TADA_CreateParamUseRef()

DESCRIPTION

@@ -72,7 +72,10 @@ Imports:
     nhdplusTools,
     arcgislayers,
     httr,
-    jsonlite
+    jsonlite, 
+    openxlsx,
+    rATTAINS,
+    DT
 Depends: 
     R (>= 3.5.0)
 Suggests:

NAMESPACE

@@ -7,6 +7,7 @@ export(TADA_AutoClean)
 export(TADA_AutoFilter)
 export(TADA_BigDataRetrieval)
 export(TADA_Boxplot)
+export(TADA_CSVExport)
 export(TADA_CalculateTotalNP)
 export(TADA_CharStringRemoveNA)
 export(TADA_CheckRequiredFields)
@@ -16,6 +17,8 @@ export(TADA_ConvertResultUnits)
 export(TADA_ConvertSpecialChars)
 export(TADA_CreateComparableID)
 export(TADA_CreatePairRef)
+export(TADA_CreateParamRef)
+export(TADA_CreateParamUseRef)
 export(TADA_CreateUnitRef)
 export(TADA_DataRetrieval)
 export(TADA_DepthProfilePlot)
@@ -41,10 +44,12 @@ export(TADA_FlagSpeciation)
 export(TADA_FlaggedSitesMap)
 export(TADA_FormatDelimitedString)
 export(TADA_GetATTAINS)
+export(TADA_GetATTAINSParamUseOrgRef)
 export(TADA_GetActivityTypeRef)
 export(TADA_GetCharacteristicRef)
 export(TADA_GetDetCondRef)
 export(TADA_GetDetLimitRef)
+export(TADA_GetEPA304aRef)
 export(TADA_GetMeasureQualifierCodeRef)
 export(TADA_GetMeasureUnitRef)
 export(TADA_GetMonLocTypeRef)

R/DataDiscoveryRetrieval.R

@@ -18,7 +18,6 @@
 #' characteristicName and Characteristic Group also work as an AND, therefore the
 #' characteristicName must fall within the Characteristic Group when both are entered.
 #'
-#'
 #' Users can reference the \href{https://www.epa.gov/waterdata/storage-and-retrieval-and-water-quality-exchange-domain-services-and-downloads}{WQX domain tables}
 #' to find allowable values for queries, e.g., reference the WQX domain table to find countycode and statecode: https://cdx.epa.gov/wqx/download/DomainValues/County_CSV.zip
 #' Alternatively, you can use the WQP services to find areas where data is available in the US: https://www.waterqualitydata.us/Codes/countycode
@@ -27,7 +26,7 @@
 #' is important for categorizing result value and detection limit data, as well as
 #' harmonizing key columns used in TADA. See ?TADA_AutoClean for more information.
 #'
-#' Note: TADA_DataRetrieval (by leveraging dataRetrieval),  automatically converts
+#' Note: TADA_DataRetrieval (by leveraging dataRetrieval), automatically converts
 #' the date times to UTC. It also automatically converts the data to dates,
 #' datetimes, numerics based on a standard algorithm. See: ?dataRetrieval::readWQPdata
 #'

R/DepthProfile.R

@@ -625,7 +625,7 @@ TADA_IDDepthProfiles <- function(.data, nresults = TRUE, nvalue = 2, aggregates
 #'   TADA_ConvertDepthUnits.
 #'
 #' @param groups A vector of two identifiers from the TADA.ComparableDataIdentifier column.
-#' For example, the groups could be 'DISSOLVED OXYGEN (DO)_NA_NA_UG/L' and 'PH_NA_NA_NA'.
+#' For example, the groups could be 'DISSOLVED OXYGEN (DO)_NA_NA_UG/L' and 'PH_NA_NA_STD UNITS'.
 #' These groups will be specific to your data frame. The TADA_IDDepthProfiles can be
 #' used to identify available groups.
 #'
@@ -661,7 +661,7 @@ TADA_IDDepthProfiles <- function(.data, nresults = TRUE, nvalue = 2, aggregates
 #' data(Data_6Tribes_5y_Harmonized)
 #' # Create a depth profile figure with three parameters for a single monitoring location and date
 #' TADA_DepthProfilePlot(Data_6Tribes_5y_Harmonized,
-#'   groups = c("TEMPERATURE_NA_NA_DEG C", "PH_NA_NA_NA", "DEPTH, SECCHI DISK DEPTH_NA_NA_M"),
+#'   groups = c("TEMPERATURE_NA_NA_DEG C", "PH_NA_NA_STD UNITS", "DEPTH, SECCHI DISK DEPTH_NA_NA_M"),
 #'   location = "REDLAKE_WQX-ANKE",
 #'   activity_date = "2018-10-04"
 #' )

R/ExampleData.R

@@ -108,3 +108,16 @@ NULL
 #' @usage data(Data_R5_TADAPackageDemo)
 #' @format A data frame with 6569 rows and 123 variables
 NULL
+
+#' Data_WV_Mod1_Output
+#'
+#' See Module 1 Workflow.R script in vignettes folder
+#' This is the final data after running a recommended module 1 workflow.
+#' It can be used as a starting point for Modules 2 or 3.
+#'
+#' @docType data
+#' @keywords data frame
+#' @name Data_WV_Mod1_Output
+#' @usage data(Data_WV_Mod1_Output)
+#' @format A data frame with 84 rows and 161 variables
+NULL

R/Figures.R

@@ -915,7 +915,7 @@ TADA_Scatterplot <- function(.data, id_cols = c("TADA.ComparableDataIdentifier")
 #'
 #' @param groups A vector of two identifiers from the id_cols column. For
 #'   example, if the id_cols is 'TADA.ComparableDataIdentifier', the groups could
-#'   be 'DISSOLVED OXYGEN (DO)_NA_NA_UG/L' and 'PH_NA_NA_NA'. These groups will
+#'   be 'DISSOLVED OXYGEN (DO)_NA_NA_UG/L' and 'PH_NA_NA_STD UNITS'. These groups will
 #'   be specific to your dataset. If the id_cols is 'MonitoringLocationName',
 #'   the groups could be 'Upper Red Lake: West' and 'Upper Red Lake: West-Central'.
 #'
@@ -934,7 +934,7 @@ TADA_Scatterplot <- function(.data, id_cols = c("TADA.ComparableDataIdentifier")
 #' # Load example dataset:
 #' data(Data_6Tribes_5y_Harmonized)
 #' # Create a single scatterplot with two specified groups from TADA.ComparableDataIdentifier
-#' TADA_TwoCharacteristicScatterplot(Data_6Tribes_5y_Harmonized, id_cols = "TADA.ComparableDataIdentifier", groups = c("TEMPERATURE_NA_NA_DEG C", "PH_NA_NA_NA"))
+#' TADA_TwoCharacteristicScatterplot(Data_6Tribes_5y_Harmonized, id_cols = "TADA.ComparableDataIdentifier", groups = c("TEMPERATURE_NA_NA_DEG C", "PH_NA_NA_STD UNITS"))
 #'
 TADA_TwoCharacteristicScatterplot <- function(.data, id_cols = "TADA.ComparableDataIdentifier", groups) {
   # check .data is data.frame

R/GeospatialFunctions.R

@@ -130,7 +130,7 @@ TADA_MakeSpatial <- function(.data, crs = 4326) {
 #'   applyautoclean = TRUE
 #' )
 #'
-#' nv_attains_features <- fetchATTAINS(tada_data, catchments_only = FALSE)
+#' nv_attains_features <- EPATADA:::fetchATTAINS(tada_data, catchments_only = FALSE)
 #' }
 fetchATTAINS <- function(.data, catchments_only = FALSE) {
   sf::sf_use_s2(FALSE)
@@ -922,11 +922,15 @@ fetchNHD <- function(.data, resolution = "Hi", features = "catchments") {
 #' TADA_GetATTAINS
 #'
 #' Link catchment-based ATTAINS assessment unit data (EPA snapshot of NHDPlus HR catchments associated with entity submitted assessment unit features - points, lines, and polygons) to Water Quality Portal observations, often imported via `TADA_DataRetrieval()`. This function returns the objects that can be mapped in `TADA_ViewATTAINS()`. Check out the
-#' TADAModule2.Rmd for an example workflow.
+#' TADAModule2.Rmd for an example workflow. Note that approximately 80% of state submitted assessment units in ATTAINS were developed based on high res NHDPlus, so we are using that as the default.
+#' 
+#' The ATTAINS snapshot of NHDPlus HR catchments is not available for areas that do not have existing Assessment Units in ATTAINS. For these areas where there are WQP sites, but no existing ATTAINS assessment units, a user can choose to associate the WQP sites with NHDPlus catchments available from the USGS nhdplusTools package (USGS snapshot) using the optional function param 'fill_catchments'. In theory, if desired by the user, these high res catchments could be created as new assessment unit polygons in ATTAINS (that process is outside of TADA).
 #'
 #' Adds one new column to input dataframe, 'index', which identifies rows that are the same observation but are linked to multiple ATTAINS assessment units. It is possible for a single TADA WQP observation to have multiple ATTAINS assessment units linked to it and subsequently more than one row of data.
 #'
-#' If TADA_MakeSpatial has not yet been run, this function runs it which also adds another new column to the input dataframe, 'geometry', which allows for mapping and additional geospatial capabilities.
+#' If TADA_MakeSpatial has not yet been run, this function runs it which also adds another new column to the input dataframe, 'geometry', which allows for mapping and additional geospatial capabilities. 
+#' 
+#' Please review the output of this function carefully, especially waterbody intersections and lake/ocean coasts where imprecise WQP monitoring location coordinates can be problematic. Note that many WQP locations will not fall within the bounds of NHDPlus (estuaries, oceans). Manual adjustments and quality control checks are strongly encouraged. WQP monitoring location metadata may also be helpful for matching waterbody names with ATTAINS waterbody names instead of relying solely on the geospatial location (lat/long).
 #'
 #' @param .data A dataframe created by `TADA_DataRetrieval()` or the sf equivalent made by `TADA_MakeSpatial()`.
 #' @param fill_catchments Whether the user would like to return NHD catchments for WQP observations not associated with an ATTAINS assessment unit (TRUE or FALSE). When fill_catchments = TRUE, the returned list splits observations into two dataframes: WQP observations with ATTAINS catchment data, and WQP observations without ATTAINS catchment data. Defaults to FALSE.

R/Maintenance.R

@@ -12,6 +12,7 @@ TADA_UpdateAllRefs <- function() {
   TADA_UpdateCharacteristicRef()
   TADA_UpdateMeasureQualifierCodeRef()
   TADA_UpdateMonLocTypeRef()
+  TADA_UpdateEPA304aRef()
 }
 
 # # update tribal layers
@@ -132,6 +133,97 @@ TADA_UpdateExampleData <- function() {
   # save(Data_R5_TADAPackageDemo, file = "data/Data_R5_TADAPackageDemo.rda")
   usethis::use_data(Data_R5_TADAPackageDemo, internal = FALSE, overwrite = TRUE, compress = "xz", version = 3, ascii = FALSE)
   rm(Data_R5_TADAPackageDemo)
+
+  # MODULE 3 VIGNETTE EXAMPLE DATA
+  # Get data
+  Data_WV <- TADA_DataRetrieval(
+    startDate = "2020-03-14",
+    huc = "02070004",
+    applyautoclean = FALSE)
+
+  # Remove non-surface water media
+  # OPTIONAL
+  Data_WV_2 <- TADA_AnalysisDataFilter(
+    Data_WV,
+    clean = TRUE,
+    surface_water = TRUE,
+    ground_water = FALSE,
+    sediment = FALSE)
+
+  # Remove single org duplicates
+  # REQUIRED
+  Data_WV_3 <- TADA_FindPotentialDuplicatesSingleOrg(
+    Data_WV_2)
+
+  Data_WV_4 <- dplyr::filter(
+    Data_WV_3,
+    TADA.SingleOrgDup.Flag == "Unique")
+
+  # Run autoclean
+  # REQUIRED
+  Data_WV_5 <- TADA_AutoClean(Data_WV_4)
+
+  # Prepare censored results
+  # REQUIRED
+  Data_WV_6 <- TADA_SimpleCensoredMethods(
+    Data_WV_5,
+    nd_method = "multiplier",
+    nd_multiplier = 0.5,
+    od_method = "as-is",
+    od_multiplier = "null")
+
+  # Remove multiple org duplicates
+  # OPTIONAL
+  Data_WV_7 <- TADA_FindPotentialDuplicatesMultipleOrgs(
+    Data_WV_6)
+
+  Data_WV_8 <- dplyr::filter(
+    Data_WV_7,
+    TADA.ResultSelectedMultipleOrgs == "Y")
+
+  # Filter out remaining irrelevant data, NA's and empty cols
+  # REQUIRED
+  unique(Data_WV_8$TADA.ResultMeasureValueDataTypes.Flag)
+  sum(is.na(Data_WV_8$TADA.ResultMeasureValue))
+  Data_WV_9 = TADA_AutoFilter(Data_WV_8)
+  unique(Data_WV_9$TADA.ResultMeasureValueDataTypes.Flag)
+  sum(is.na(Data_WV_9$TADA.ResultMeasureValue))
+
+  # Remove results with QC issues
+  # REQUIRED
+  Data_WV_10 <- TADA_RunKeyFlagFunctions(
+    Data_WV_9,
+    clean = TRUE)
+
+  # CM note for team discussion: Should results with NA units be dealt with now as well within TADA_AutoFilter? 
+
+  # Flag above and below threshold. Do not remove
+  # OPTIONAL
+  Data_WV_11 <- TADA_FlagAboveThreshold(Data_WV_10, clean = FALSE, flaggedonly = FALSE)
+  Data_WV_12 <- TADA_FlagBelowThreshold(Data_WV_11, clean = FALSE, flaggedonly = FALSE)
+
+  # Harmonize synonyms
+  # OPTIONAL
+  Data_WV_13 <- TADA_HarmonizeSynonyms(Data_WV_12)
+
+  # Review
+  Data_WV_14 <- dplyr::filter(Data_WV_13, TADA.CharacteristicName %in% c("ZINC", "PH","NITRATE"))
+
+  TADA_FieldValuesTable(Data_WV_14, field = "TADA.ComparableDataIdentifier")
+
+  # Save example data
+  Data_WV_Mod1_Output <- Data_WV_14
+
+  print("Data_WV_Mod1_Output:")
+  print(dim(Data_WV_Mod1_Output))
+
+  usethis::use_data(Data_WV_Mod1_Output,
+                    internal = FALSE,
+                    overwrite = TRUE,
+                    compress = "xz",
+                    version = 3,
+                    ascii = FALSE)
+  rm(Data_WV_Mod1_Output)
 }
 
 ###########################################################

R/RequiredCols.R

@@ -418,124 +418,6 @@ TADA_CheckRequiredFields <- function(.data) {
 }
 
 
-
-#' AutoFilter
-#'
-#' This function removes rows where the result value is not numeric to
-#' prepare a dataframe for quantitative analyses. Ideally, this function should
-#' be run after other data cleaning, QA/QC, and harmonization steps are
-#' completed using other TADA package functions, or manually. Specifically, .
-#' this function removes rows with "Text" and "NA - Not Available"
-#' in the TADA.ResultMeasureValueDataTypes.Flag column, or NA in the
-#' TADA.ResultMeasureValue column.
-#'
-#' This function also removes any columns not required for TADA workflow where
-#' all values are equal to NA.It also provides a warning message identifying
-#' any TADA required columns containing only NA values.
-#'
-#' @param .data TADA dataframe OR TADA sites dataframe
-#'
-#' @return .data with rows removed where result values are not quantitative (NA or text),
-#' or the results have other issues that are not dealt with elsewhere.
-#'
-#' @export
-#'
-#' @examples
-#' # Load example dataset:
-#' data(Data_Nutrients_UT)
-#'
-#' # Remove all:
-#' TADA_filtered <- TADA_AutoFilter(Data_Nutrients_UT)
-#'
-TADA_AutoFilter <- function(.data) {
-  # check .data is data.frame
-  TADA_CheckType(.data, "data.frame", "Input object")
-
-  TADA_CheckColumns(.data, c(
-    "ActivityTypeCode", "MeasureQualifierCode",
-    "TADA.ResultMeasureValueDataTypes.Flag",
-    "TADA.ResultMeasureValue", "TADA.ActivityMediaName",
-    "ActivityTypeCode"
-  ))
-
-  # keep track of starting and ending number of rows
-  start <- dim(.data)[1]
-
-  # run TADA_FindQCActivities if needed
-  if (("TADA.MeasureQualifierCode.Flag" %in% colnames(.data)) == TRUE) {
-    .data <- .data
-  }
-
-  if (("TADA.MeasureQualifierCode.Flag" %in% colnames(.data)) == FALSE) {
-    .data <- TADA_FindQCActivities(.data, clean = FALSE, flaggedonly = FALSE)
-  }
-
-  # remove text, NAs and QC results
-  .data <- dplyr::filter(.data, TADA.ResultMeasureValueDataTypes.Flag != "Text" &
-    TADA.ResultMeasureValueDataTypes.Flag != "NA - Not Available" &
-    TADA.ActivityType.Flag == "Non_QC" & # filter out QA/QC ActivityTypeCode's
-    !is.na(TADA.ResultMeasureValue))
-
-  # remove columns that are not required for TADA workflow
-  print("TADA_Autofilter: removing columns not required for TADA workflow if they contain only NAs.")
-
-  # create list of columns containing all NA values.
-  na.cols <- .data %>%
-    purrr::keep(~ all(is.na(.x))) %>%
-    names()
-
-  # create list of columns to be removed by comparing columns containing all NA values to required columns.
-  # any required columns with all NA values will be excluded from the list of columns to remove.
-  remove.cols <- setdiff(na.cols, require.cols)
-
-  # remove not required columns containing all NA values from data frame.
-  .data <- .data %>%
-    dplyr::select(-dplyr::contains(remove.cols))
-
-  # check to make sure required columns contain some data that is not NA
-  req.check <- intersect(require.cols, na.cols)
-
-  # create character string for list of required columns containing only NAs
-  req.paste <- stringi::stri_replace_last_fixed(paste(as.character(req.check), collapse = ", ", sep = ""), ", ", " and ")
-
-  # remove column name lists
-  rm(na.cols)
-
-  # create character string for list of removed columns
-  remove.paste <- stringi::stri_replace_last_fixed(paste(as.character(remove.cols), collapse = ", ", sep = ""), ", ", " and ")
-
-  # print list of columns removed from data frame
-  if (length(remove.cols) > 0) {
-    print(paste0("The following column(s) were removed as they contained only NAs: ", remove.paste, "."))
-  } else {
-    print("All columns contained some non-NA values and were retained in the data frame.")
-  }
-
-  # remove columns that are not required for TADA workflow
-  print("TADA_Autofilter: checking required columns for non-NA values.")
-
-  # if some required columns contain only NA values print a warning message.
-  if (length(req.check) > 0) {
-    print(paste0("TADA Required column(s) ", req.paste, " contain only NA values. This may impact other TADA functions."))
-  } else {
-    print("All TADA Required columns contain some non-NA values.")
-  }
-
-  # remove intermediate objects
-  rm(req.paste, remove.cols, remove.paste, req.check)
-
-  end <- dim(.data)[1]
-
-  # print number of results removed
-  if (!start == end) {
-    net <- start - end
-    print(paste0("Function removed ", net, " results. These results are either text or NA and cannot be plotted or represent quality control activities (not routine samples or measurements)."))
-  }
-
-  return(.data)
-}
-
-
 #' TADA_RetainRequired
 #'
 #' This function removes all duplicate columns where TADA has created a new column with a TADA prefix.