From 2a5f73063b5b978c039cf6229b1e2f915f3e8fc1 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Thu, 30 May 2024 15:34:15 -0400 Subject: [PATCH 01/16] include schema name in sql query --- R/get_conversion_factors.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/get_conversion_factors.R b/R/get_conversion_factors.R index 6a0ddec..064c2ed 100644 --- a/R/get_conversion_factors.R +++ b/R/get_conversion_factors.R @@ -33,7 +33,7 @@ get_conversion_factors <- function(channel){ # creates the sql based on user input - sqlStatement <- convert.qry <- "select * from SURVAN_CONVERSION_FACTORS" + sqlStatement <- convert.qry <- "select * from SVDBS.SURVAN_CONVERSION_FACTORS" query <- DBI::dbGetQuery(channel,sqlStatement) From 75929848806cc6f911d4373095bfe7bc50485438 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Thu, 30 May 2024 15:39:46 -0400 Subject: [PATCH 02/16] expand description for get_length_weight --- R/get_length_weight.R | 5 ++- man/get_length_weight.Rd | 5 ++- man/get_survdat_data.Rd | 94 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/R/get_length_weight.R b/R/get_length_weight.R index 2128a9b..7e23e83 100644 --- a/R/get_length_weight.R +++ b/R/get_length_weight.R @@ -1,6 +1,7 @@ -#'Generate a table of length weight coefficients +#' Extract species specific LENGTH-WEIGHT COEFFICIENTS from SVDBS #' -#'Pulls the length-weight coefficients calculated from NOAA Tech Memo NMFS-NE-171. +#'Pulls the length-weight coefficients from SVDBS LENGTH_WEIGHT_COEFFICIENTS table +#' These coefficients are described in NOAA Tech Memo NMFS-NE-171. #' #' #' @inheritParams get_survdat_data diff --git a/man/get_length_weight.Rd b/man/get_length_weight.Rd index 1d17e02..49a1f98 100644 --- a/man/get_length_weight.Rd +++ b/man/get_length_weight.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/get_length_weight.R \name{get_length_weight} \alias{get_length_weight} -\title{Generate a table of length weight coefficients} +\title{Extract species specific LENGTH-WEIGHT COEFFICIENTS from SVDBS} \usage{ get_length_weight(channel) } @@ -17,7 +17,8 @@ Returns a data.table (nx12). \item{SVLWCOEFF}{The natural log of the coefficient of the length-weight equation, ln a.} } \description{ -Pulls the length-weight coefficients calculated from NOAA Tech Memo NMFS-NE-171. +Pulls the length-weight coefficients from SVDBS LENGTH_WEIGHT_COEFFICIENTS table +These coefficients are described in NOAA Tech Memo NMFS-NE-171. } \seealso{ Other survdat: diff --git a/man/get_survdat_data.Rd b/man/get_survdat_data.Rd index e28e0e3..636b19e 100644 --- a/man/get_survdat_data.Rd +++ b/man/get_survdat_data.Rd @@ -1,9 +1,21 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/get_survdat_data.R +% Please edit documentation in R/get_cruise_data.R, R/get_survdat_data.R \name{get_survdat_data} \alias{get_survdat_data} \title{Extracts Survey data from Database} \usage{ +get_survdat_data( + channel, + filterByYear = NA, + all.season = F, + shg.check = T, + conversion.factor = T, + use.SAD = F, + getBio = F, + getLengths = T, + getWeightLength = F +) + get_survdat_data( channel, filterByYear = NA, @@ -93,9 +105,67 @@ The date: The expression: +\item{functionCall}{The call used to create the data pul} + +A list containing a Data frame (data.table) (n x 21), a list of SQL queries used to pull the data, +the date of the pull, and the call expression +Each row of the data.table represents the number at length of a species on a specific tow along with physical attributes of the tow. + +The data frame (Descriptions taken from NEFSC Data dictionary) + +\item{CRUISE6}{Code uniquely identifying cruise. The first four digits indicate the year and the last two digit uniquely identify the cruise within the year. The 5th byte signifies cruises other than groundfish: Shrimp survey = 7 (i.e. 201470), State of Massachusetts survey = 9 (i.e. 201491), Food habits = 5 (i.e.199554)} +\item{STATION}{Unique sequential order in which stations have been completed. Hangups and short tows each receive a non-repeated consecutive number.} +\item{STRATUM}{ A predefined area where a net dredge, or other piece of gear was deployed. Code consists of 2 parts: Stratum group code number (2 bytes) and stratum number (3 bytes). Stratum group refers to if area fished is inshore or offshore North or South of Cape Hatteras or the type of cruise (shellfish, State of MA, offshore deepwater). The stratum number (third and fourth digits of code) refers to area defined by depth zone. See SVDBS.SVMSTRATA. The fifth digit of the code increases the length of the stratum number for revised strata after the Hague Line was established. Stratum group code: 01 = Trawl, offshore north of Hatteras; 02 = BIOM; 03 = Trawl, inshore north of Hatteras; 04 = Shrimp; 05 = Scotian shelf; 06 = Shellfish; 07 = Trawl, inshore south of Hatteras; 08 = Trawl, Offshore south of Hatteras; 09 = MA DMF; 99 = Offshore deepwater (outside the stratified area). A change in Bottom Trawl Stratum for the Gulf of Maine-Bay of Fundy has been in effect since Spring 1987, and may be summarized as follows: Previous strata: 01350; Present strata: 01351, 01352.} +\item{TOW}{Sequential number representing order in which station was selected within a stratum.} +\item{SVSPP}{A standard code which represents a species caught in a trawl or dredge. Refer to the SVDBS.SVSPECIES_LIST} +\item{CATCHSEX}{Code used to identify species that are sexed at the catch level. See SVDBS.SEX_CODES} +\item{SVVESSEL}{Standard two character code for a survey vessel. Refer to SVDBS.SV_VESSEL} +\item{YEAR}{ Year in which cruise was conducted.} +\item{SEASON}{ Season of the year in which cruise was conducted.} +\item{LAT}{Beginning latitude of tow in decimal degrees.(DECDEG_BEGLAT)} +\item{LON}{Beginning longitude of tow in decimal degrees.(DECDEG_BEGLON)} +\item{EST_TOWDATE}{Date and time represented by Eastern Standard Time (EST) for the start of a tow or deployment.(BEGIN_EST_TOWDATE)} +\item{DEPTH}{ A four digit number recording the average depth, to the nearest meter, during a survey gear deployment.(AVGDEPTH)} +\item{SURFTEMP}{Surface temperature of water (degrees Celcius).} +\item{SURFSALIN}{Salinity at water surface in practical salinity units (PSU).} +\item{BOTTEMP}{Bottom temperature (degrees Celsius).} +\item{BOTSALIN}{Bottom salinity in Practical Salinity Units (PSU).} +\item{ABUNDANCE}{Expanded number of individuals of a species caught at a given station.(EXPCATCHNUM)} +\item{BIOMASS}{Expanded catch weight of a species caught at a given station. (EXPCATCHWT)} +\item{LENGTH}{Measured length of species in centimeters (cm). Measure method differs by species.} +\item{NUMLEN}{Expanded number of specimens at a given length.(EXPNUMLEN)} + +Additional columns if bio = T (UNION_FSCS_SVBIO) + +\item{INDID}{A unique identifier for each fish sampled.} +\item{INDWT}{Individual weight (KG) of species being sampled.} +\item{SEX}{Code indicating sex of fish or invertebrate species. See SVDBS.FSCS_SEX_CODES if using fscs data and SVDBS.SEX_CODES if using non FSCS data. Codes 0, 1, 2 and 4 are the only valid codes in fscs tables.} +\item{MATURITY}{Stage of maturation of the fish being sampled. See SVDBS.FSCS_MATURITY_CODES} +\item{AGE}{Age of specimen in years.} +\item{STOM_VOLUME}{Volume of the stomach contents of the fish sampled, measured to the nearest tenth of a cubic centimeter (cc).} +\item{STOM_WGT}{ Stomach weight of an individual fish in grams.} + +The list of sql statements: + +\item{cruise}{Select unique list of cruises. Table = MSTR_CRUISE} +\item{station}{Select unique set of stations from result of \code{cruise}. Table = UNION_FSCS_SVSTA} +\item{catch}{Select species abundance and biomass data from result of \code{station}. Table = UNION_FSCS_SVCAT} +\item{length}{Select Lengths of species found in \code{catch}. Table = UNION_FSCS_SVLEN} +\item{sad}{Select ????? from Survey Analysis Database. Table = STOCKEFF.I_SV_MERGED_CATCH_CALIB_O} +\item{conversions}{Select conversion factors. Table = SURVAN_CONVERSION_FACTORS} +\item{bio}{Select bio stats. Table = UNION_FSCS_SVBIO} + +The date: + + \item{pullDate}{The date the data was pulled from the database} + +The expression: + \item{functionCall}{The call used to create the data pul} } \description{ +Connects to svdbs and pulls data from MSTR_CRUISE, UNION_FSCS_SVCAT, UNION_FSCS_SVLEN, UNION_FSCS_SVSTA, UNION_FSCS_SVBIO + Connects to svdbs and pulls data from MSTR_CRUISE, UNION_FSCS_SVCAT, UNION_FSCS_SVLEN, UNION_FSCS_SVSTA, UNION_FSCS_SVBIO } \examples{ @@ -110,8 +180,30 @@ get_survdat_data(channel,conversion.factor = T, bio=T) get_survdat_data(channel,conversion.factor = T, bio=F) } +\dontrun{ +# First create a connection object to the database +channel <- dbutils::connect_to_database("serverName","userName") +# pull survey data, applies conversion factors (Door, net, vessel) and join with +# biological data to return individual sex, age, maturity, stomach data +get_survdat_data(channel,conversion.factor = T, bio=T) + +# Same data pull without individual biological data +get_survdat_data(channel,conversion.factor = T, bio=F) +} + } \seealso{ +Other survdat: +\code{\link{calc_stratified_mean}()}, +\code{\link{calc_swept_area}()}, +\code{\link{get_area}()}, +\code{\link{get_length_weight}()}, +\code{\link{get_survdat_clam_data}()}, +\code{\link{get_survdat_scallop_data}()}, +\code{\link{post_strat}()}, +\code{\link{strat_mean}()}, +\code{\link{swept_area}()} + Other survdat: \code{\link{calc_stratified_mean}()}, \code{\link{calc_swept_area}()}, From b614325cbd9813b3adea2c40aff346ea9cc9e2a6 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Thu, 30 May 2024 16:53:36 -0400 Subject: [PATCH 03/16] removed internal functions from export and removed help pages --- NAMESPACE | 4 -- R/apply_conversion_factors.r | 6 +-- R/post_strat.R | 2 +- R/strat_mean.R | 10 ++--- R/strat_prep.R | 4 +- R/swept_area.R | 2 +- man/apply_conversion_factors.Rd | 31 ---------------- man/post_strat.Rd | 43 ---------------------- man/strat_mean.Rd | 65 --------------------------------- man/strat_prep.Rd | 43 ---------------------- man/swept_area.Rd | 61 ------------------------------- 11 files changed, 11 insertions(+), 260 deletions(-) delete mode 100644 man/apply_conversion_factors.Rd delete mode 100644 man/post_strat.Rd delete mode 100644 man/strat_mean.Rd delete mode 100644 man/strat_prep.Rd delete mode 100644 man/swept_area.Rd diff --git a/NAMESPACE b/NAMESPACE index 6c395b6..05ffbbd 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -18,10 +18,6 @@ export(get_survdat_scallop_data) export(get_vessel) export(plot_data_area) export(plot_shapefile) -export(post_strat) -export(strat_mean) -export(strat_prep) -export(swept_area) importFrom(data.table,"%like%") importFrom(data.table,"key") importFrom(magrittr,"%>%") diff --git a/R/apply_conversion_factors.r b/R/apply_conversion_factors.r index cf522fb..6c27538 100644 --- a/R/apply_conversion_factors.r +++ b/R/apply_conversion_factors.r @@ -17,6 +17,7 @@ #' This is an internal function. #' To pull the values of the conversion factors use \code{\link{get_conversion_factors}} #' +#'@noRd @@ -30,10 +31,7 @@ apply_conversion_factors <- function(channel,survdat.raw,use.SAD = F) { survdat[, ABUNDANCE := as.double(ABUNDANCE)] #Grab all conversion factors off the network - convert.qry <- "select * - from svdbs.survan_conversion_factors" - - convert <- data.table::as.data.table(DBI::dbGetQuery(channel, convert.qry)) + convert <- data.table::as.data.table(get_conversion_factors(channel)$data) #DCF < 1985 Door Conversion dcf.spp <- convert[DCF_WT > 0, SVSPP] diff --git a/R/post_strat.R b/R/post_strat.R index 44aca3e..8f37aaf 100644 --- a/R/post_strat.R +++ b/R/post_strat.R @@ -17,7 +17,7 @@ #' #'@family survdat #' -#' @export +#' @noRd post_strat <- function (surveyData, areaPolygon, areaDescription, na.keep = F) { diff --git a/R/strat_mean.R b/R/strat_mean.R index 116c45d..19f9e30 100644 --- a/R/strat_mean.R +++ b/R/strat_mean.R @@ -23,14 +23,14 @@ #' #' @importFrom data.table "key" #' -#'@family survdat +#'@family surdat #' #'@examples #'\dontrun{ #' # Called internally #'} #' -#' @export +#' @noRd strat_mean <- function (prepData, groupDescription = "SVSPP", filterByGroup = "all", @@ -44,7 +44,7 @@ strat_mean <- function (prepData, groupDescription = "SVSPP", filterByGroup = "a stratmeanData <- unique(stratmeanData, by = key(stratmeanData)) stratmeanData[, c('LENGTH', 'NUMLEN') := NULL] } - + data.table::setnames(stratmeanData, c(groupDescription, areaDescription), c('group', 'strat')) @@ -145,9 +145,9 @@ strat_mean <- function (prepData, groupDescription = "SVSPP", filterByGroup = "a stratmeanData[, glen := NULL] data.table::setkey(stratmeanData, YEAR, SVSPP, CATCHSEX) } - + if(seasonFlag == F) stratmeanData[, SEASON := 'ALL'] - + data.table::setnames(stratmeanData, 'group', groupDescription) return(stratmeanData[]) diff --git a/R/strat_prep.R b/R/strat_prep.R index 01c9dec..e1a6ff5 100644 --- a/R/strat_prep.R +++ b/R/strat_prep.R @@ -26,7 +26,7 @@ #' # Called internally #' } #' -#' @export +#' @noRd strat_prep <- function (surveyData, areaPolygon = "NEFSC strata", @@ -53,7 +53,7 @@ strat_prep <- function (surveyData, areaPolygon = "NEFSC strata", # post stratify if necessary if(poststratFlag){ message("Post stratifying ...") - surveyData <- survdat::post_strat(surveyData, areaPolygon, areaDescription) + surveyData <- survdat:::post_strat(surveyData, areaPolygon, areaDescription) } else { #Add extra column to original data to mimic what happens when post-stratifying surveyData <- surveyData[, areaDescription := STRATUM] diff --git a/R/swept_area.R b/R/swept_area.R index bab589d..30b5461 100644 --- a/R/swept_area.R +++ b/R/swept_area.R @@ -29,7 +29,7 @@ #'# Called internally #'} #' -#'@export +#'@noRd swept_area <- function (prepData, stratmeanData, q = NULL, a = 0.0384, diff --git a/man/apply_conversion_factors.Rd b/man/apply_conversion_factors.Rd deleted file mode 100644 index e8338d5..0000000 --- a/man/apply_conversion_factors.Rd +++ /dev/null @@ -1,31 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/apply_conversion_factors.r -\name{apply_conversion_factors} -\alias{apply_conversion_factors} -\title{Applies conversion factors to Raw Survey data pull} -\usage{ -apply_conversion_factors(channel, survdat.raw, use.SAD = F) -} -\arguments{ -\item{channel}{an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to connect -to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}})} - -\item{survdat.raw}{Data.table. Result of \code{\link{get_survdat_data}}} - -\item{use.SAD}{Boolean. Use Survey Analysis Database (SAD) for assessed species. Default = F} -} -\value{ -A list containing a Data frame (data.table) (n x 21) and a list of SQL queries used to pull the data -Each row of the data.table represents the number at length of a species on a specific tow along with physical attributes of the tow. -See \code{\link{get_survdat_data}} for more details -} -\description{ -Connects to svdbs and pulls conversion factors from SURVAN_CONVERSION_FACTORS -Explain what conversions do ... -} -\section{Internal}{ - -This is an internal function. -To pull the values of the conversion factors use \code{\link{get_conversion_factors}} -} - diff --git a/man/post_strat.Rd b/man/post_strat.Rd deleted file mode 100644 index f92755c..0000000 --- a/man/post_strat.Rd +++ /dev/null @@ -1,43 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/post_strat.R -\name{post_strat} -\alias{post_strat} -\title{Assigns points to polygon} -\usage{ -post_strat(surveyData, areaPolygon, areaDescription, na.keep = F) -} -\arguments{ -\item{surveyData}{Data table. NEFSC survey data generated by \code{get_survdat_data.R}} - -\item{areaPolygon}{sf object or character string. Default = "NEFSC strata". The default option uses the survey strata shapefile bundled with the package. -To use any other shapefile for stratification, the shapefile must be read in as an \link[sf]{sf} object and the \code{areaDescription} argument must be specified.} - -\item{areaDescription}{Character String. Column name from \code{areaPolygon} -that contains the strata designations.} - -\item{na.keep}{Boolean. Logical value to indicate whether original strata names -should be retained.} -} -\value{ -Returns a \code{surveyData} data.table with one additional column labeled - with the value of \code{areaDescription} - -\item{areaDescription}{The name of the region (found in \code{areaPolygon}) - that a record in \code{surveyData} is assigned to} -} -\description{ -Assign survey data (points, lat and lon) to designated regions (polygons) from a shape file. -} -\seealso{ -Other survdat: -\code{\link{calc_stratified_mean}()}, -\code{\link{calc_swept_area}()}, -\code{\link{get_area}()}, -\code{\link{get_length_weight}()}, -\code{\link{get_survdat_clam_data}()}, -\code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} -} -\concept{survdat} diff --git a/man/strat_mean.Rd b/man/strat_mean.Rd deleted file mode 100644 index 51be57d..0000000 --- a/man/strat_mean.Rd +++ /dev/null @@ -1,65 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/strat_mean.R -\name{strat_mean} -\alias{strat_mean} -\title{Calculate stratified mean biomass and abundance} -\usage{ -strat_mean( - prepData, - groupDescription = "SVSPP", - filterByGroup = "all", - mergesexFlag = T, - seasonFlag, - areaDescription, - poststratFlag -) -} -\arguments{ -\item{prepData}{Data table. NEFSC survey data generated by \code{get_survdat_data} -and modified by \code{strat_prep}.} - -\item{groupDescription}{Character string. Column of \code{prepData} which -contains the groups (e.g. "SVSPP") on which the means are based.} - -\item{filterByGroup}{Character or numeric vector. Set of groups to subset from -\code{groupDescription}. The default "all" will calculate means for all groups.} - -\item{mergesexFlag}{Boolean. Logical value to merge sexed species such as dogfish.} - -\item{seasonFlag}{Boolean. Logical value to merge seasons (F) or keep them separate (T).} - -\item{areaDescription}{Character String. Column name from \code{areaPolygon} -that contains the strata designations.} - -\item{poststratFlag}{Boolean. Logical value indicating whether the original -strata design was used or not. Changes the calculation for the variance for -post-stratified uses of survey data.} -} -\value{ -Returns a table with stratified mean biomass and abundance for each group - indicated by the \code{groupDescription}. In addition, the variance and standard - error for both means are provided. -} -\description{ -Calculates the stratified mean biomass and abundance. Also calculates the -associated variance and standard error. -} -\examples{ -\dontrun{ -# Called internally -} - -} -\seealso{ -Other survdat: -\code{\link{calc_stratified_mean}()}, -\code{\link{calc_swept_area}()}, -\code{\link{get_area}()}, -\code{\link{get_length_weight}()}, -\code{\link{get_survdat_clam_data}()}, -\code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{swept_area}()} -} -\concept{survdat} diff --git a/man/strat_prep.Rd b/man/strat_prep.Rd deleted file mode 100644 index e103059..0000000 --- a/man/strat_prep.Rd +++ /dev/null @@ -1,43 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/strat_prep.R -\name{strat_prep} -\alias{strat_prep} -\title{Prepare survey data for calculating stratified mean} -\usage{ -strat_prep( - surveyData, - areaPolygon = "NEFSC strata", - areaDescription = "STRATA", - filterByArea = "all", - filterBySeason = "all" -) -} -\arguments{ -\item{surveyData}{Data table. NEFSC survey data generated by \code{get_survdat_data.R}} - -\item{areaPolygon}{sf object or character string. Default = "NEFSC strata". The default option uses the survey strata shapefile bundled with the package. -To use any other shapefile for stratification, the shapefile must be read in as an \link[sf]{sf} object and the \code{areaDescription} argument must be specified.} - -\item{areaDescription}{Character String. Column name from \code{areaPolygon} -that contains the strata designations.} - -\item{filterByArea}{Numeric vector. Set of areas to subset from the -\code{areaDescription} of the \code{areaPolygon}.} - -\item{filterBySeason}{Character string. Which seasons of the \code{surveyData} -should be included. Choices include "SPRING", "FALL", or "all".} -} -\value{ -Returns a prepSurvey data object with added columns for the number of - tows (ntows) and stratum weights (W.h). -} -\description{ -Calculates the number of tows per strata and strata weight. Run before -sub setting the data by species of interest to ensure all stations are accounted. -} -\concept{survdat - -\@examples -\dontrun{ -# Called internally -}} diff --git a/man/swept_area.Rd b/man/swept_area.Rd deleted file mode 100644 index 2b5f28b..0000000 --- a/man/swept_area.Rd +++ /dev/null @@ -1,61 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/swept_area.R -\name{swept_area} -\alias{swept_area} -\title{Calculate the estimate of total biomass and abundance} -\usage{ -swept_area( - prepData, - stratmeanData, - q = NULL, - a = 0.0384, - areaDescription, - groupDescription = "SVSPP" -) -} -\arguments{ -\item{q}{Table of survey catchability with a column of group names and a column of -catchabilities. If not provided, assumes a q of 1 for each group (Minimum swept area -estimates).} - -\item{a}{The average swept area of the trawl. Default value is the swept area of a -standard NOAA Ship Albatross IV tow.} - -\item{areaDescription}{Column of survdat containing strata designations.} - -\item{groupDescription}{Column of survdat upon which the totals are based (i.e. SVSPP).} - -\item{survdat}{NEFSC survey data generated by Survdat.R and modified by stratprep().} - -\item{stratmean}{Output from strat_mean().} -} -\value{ -Returns a table with the estimates of total biomass and abundance as well as -the stratified mean biomass and abundance for each group indicated by the groupDescription -parameter. In addition, the variance of the totals and variance and standard error -for both means are provided. -} -\description{ -Calculates the estimate of total biomass and abundance using the swept area. -Variance of the estimate is also calculated. Requires the outputs from both -the stratprep and stratmean functions. -} -\examples{ -\dontrun{ -# Called internally -} - -} -\seealso{ -Other survdat: -\code{\link{calc_stratified_mean}()}, -\code{\link{calc_swept_area}()}, -\code{\link{get_area}()}, -\code{\link{get_length_weight}()}, -\code{\link{get_survdat_clam_data}()}, -\code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()} -} -\concept{survdat} From c06d83b279ee5fab580cf030d0c2518148475908 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Thu, 30 May 2024 16:55:00 -0400 Subject: [PATCH 04/16] added missing arguments to functions, sweithced family groupings of functions --- R/calc_stratified_mean.R | 20 ++++-- R/calc_swept_area.R | 2 +- R/get_length_weight.R | 2 +- R/get_survdat_data.R | 27 ++++--- R/get_survdat_scallop_data.R | 10 +-- man/calc_stratified_mean.Rd | 6 +- man/calc_swept_area.Rd | 6 +- man/get_area.Rd | 6 +- man/get_conversion_factors.Rd | 1 + man/get_cruise_purpose.Rd | 1 + man/get_length_weight.Rd | 25 +++---- man/get_maturity.Rd | 1 + man/get_sex.Rd | 1 + man/get_sex_fscs.Rd | 1 + man/get_species.Rd | 1 + man/get_strata.Rd | 1 + man/get_survdat_clam_data.Rd | 6 +- man/get_survdat_data.Rd | 123 +++++--------------------------- man/get_survdat_scallop_data.Rd | 10 +-- man/get_vessel.Rd | 1 + pkgdown/_pkgdown.yml | 9 --- 21 files changed, 82 insertions(+), 178 deletions(-) diff --git a/R/calc_stratified_mean.R b/R/calc_stratified_mean.R index 65d7364..0af8f55 100644 --- a/R/calc_stratified_mean.R +++ b/R/calc_stratified_mean.R @@ -2,8 +2,20 @@ #' #' Calculates the stratified mean. Details of method found here ... #' -#' @inheritParams strat_prep -#' @inheritParams strat_mean +#' @param surveyData Data table. NEFSC survey data generated by \code{get_survdat_data.R} +#' @param areaPolygon sf object or character string. Default = "NEFSC strata". The default option uses the survey strata shapefile bundled with the package. +#' To use any other shapefile for stratification, the shapefile must be read in as an \link[sf]{sf} object and the \code{areaDescription} argument must be specified. +#' @param areaDescription Character String. Column name from \code{areaPolygon} +#' that contains the strata designations. +#' @param filterByArea Numeric vector. Set of areas to subset from the +#' \code{areaDescription} of the \code{areaPolygon}. +#' @param filterBySeason Character string. Which seasons of the \code{surveyData} +#' should be included. Choices include "SPRING", "FALL", or "all". +#' @param groupDescription Character string. Column of \code{prepData} which +#' contains the groups (e.g. "SVSPP") on which the means are based. +#' @param filterByGroup Character or numeric vector. Set of groups to subset from +#' \code{groupDescription}. The default "all" will calculate means for all groups. +#' @param mergesexFlag Boolean. Logical value to merge sexed species such as dogfish. #' @param tidy Boolean. Return output in long format (Default = F). #' @param returnPrepData Boolean. Return both \code{stratmeanData} and \code{prepData} #' as a list object. The default (F) returns only the \code{stratmeanData} as a @@ -58,7 +70,7 @@ calc_stratified_mean <- function(surveyData, areaPolygon = 'NEFSC strata', #Run stratification prep message("Prepping data ...") - prepData <- survdat::strat_prep(surveyData, areaPolygon, areaDescription, + prepData <- survdat:::strat_prep(surveyData, areaPolygon, areaDescription, filterByArea, filterBySeason) #Calculate stratified mean @@ -66,7 +78,7 @@ calc_stratified_mean <- function(surveyData, areaPolygon = 'NEFSC strata', #Check if calculating mean base on all station or by season if(filterBySeason[1] == "all"){seasonFlag <- F}else{seasonFlag <- T} - stratmeanData <- survdat::strat_mean(prepData, groupDescription, filterByGroup, + stratmeanData <- survdat:::strat_mean(prepData, groupDescription, filterByGroup, mergesexFlag, seasonFlag = seasonFlag, areaDescription, poststratFlag) diff --git a/R/calc_swept_area.R b/R/calc_swept_area.R index f7f5b40..b449b54 100644 --- a/R/calc_swept_area.R +++ b/R/calc_swept_area.R @@ -44,7 +44,7 @@ calc_swept_area <- function(surveyData, areaPolygon = 'NEFSC strata', #Calculate total biomass/abundance estimates message("Calculating Swept Area Estimate ...") - sweptareaData <- survdat::swept_area(prepData = stratmeanData$prepData, + sweptareaData <- survdat:::swept_area(prepData = stratmeanData$prepData, stratmeanData = stratmeanData$stratmeanData, q = q, areaDescription = areaDescription, groupDescription = groupDescription) diff --git a/R/get_length_weight.R b/R/get_length_weight.R index 7e23e83..a78c51e 100644 --- a/R/get_length_weight.R +++ b/R/get_length_weight.R @@ -12,7 +12,7 @@ #'\item{SVLWEXP}{The exponent of the length-weight equation, b.} #'\item{SVLWCOEFF}{The natural log of the coefficient of the length-weight equation, ln a.} #' -#'@family survdat +#'@family helper #' #'@export diff --git a/R/get_survdat_data.R b/R/get_survdat_data.R index 3e13ef7..9615175 100644 --- a/R/get_survdat_data.R +++ b/R/get_survdat_data.R @@ -1,23 +1,28 @@ -#' Extracts Survey data from Database +#' Extracts Survey data from Database #' -#'Connects to svdbs and pulls data from MSTR_CRUISE, UNION_FSCS_SVCAT, UNION_FSCS_SVLEN, UNION_FSCS_SVSTA, UNION_FSCS_SVBIO +#' Connects to svdbs and pulls data from MSTR_CRUISE, UNION_FSCS_SVCAT, +#' UNION_FSCS_SVLEN, UNION_FSCS_SVSTA, UNION_FSCS_SVBIO #' -#' @param channel an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}}) +#' @param channel an Object inherited from \link[DBI]{DBIConnection-class}. +#' This object is used to communicate with the database engine. +#' (see \code{\link[dbutils]{connect_to_database}}) #' @param filterByYear Numeric vector. Subset of years from which to pull data. #' If not specified then all years are pulled. (Default = NA) #' @param all.season Boolean. Spring and Fall only (F) otherwise T. (Default = F) #' @param shg.check Boolean. use only SHG <=136 or TOGA <= 1324 (>2008). (Default = T) #' @param conversion.factor Boolean. Whether to apply conversion factors to the data pull, (Default = T) #' @param use.SAD Boolean. Use Survey Analysis Database (SAD) for assessed species. (Default = F) -#' @param getBio Boolean. Include biology data for each fish weight, sex,, stomach weight, stomach volume, age, maturity +#' @param getBio Boolean. Include biology data for each fish weight, sex, +#' stomach weight, stomach volume, age, maturity #' @param getLengths Boolean. Include length data which includes the length in #' cm and the number at length. (Default = T) #' @param getWeightLength Boolean. Include the weight at length by applying length #' weight coefficients from SVDBS. (Default = F) #' -#' @return A list containing a Data frame (data.table) (n x 21), a list of SQL queries used to pull the data, -#' the date of the pull, and the call expression -#' Each row of the data.table represents the number at length of a species on a specific tow along with physical attributes of the tow. +#' @return A list containing a Data frame (data.table) (n x 21), +#' a list of SQL queries used to pull the data, the date of the pull, +#' and the call expression. Each row of the data.table represents the number +#' at length of a species on a specific tow along with physical attributes of the tow. #' #' The data frame (Descriptions taken from NEFSC Data dictionary) #' @@ -65,15 +70,14 @@ #' #' The date: #' -#' \item{pullDate}{The date the data was pulled from the database} +#' \item{pullDate}{The date the data was pulled from the database} #' #' The expression: #' -#' \item{functionCall}{The call used to create the data pul} +#' \item{functionCall}{The call used to create the data pull} #' #' @importFrom data.table "%like%" -#' -#'@family survdat +#' @family survdat #' #'@examples #'\dontrun{ @@ -89,6 +93,7 @@ #' #'@export + get_survdat_data <- function(channel, filterByYear = NA, all.season = F, shg.check = T, conversion.factor = T, use.SAD = F, getBio = F, getLengths = T, getWeightLength = F) { diff --git a/R/get_survdat_scallop_data.R b/R/get_survdat_scallop_data.R index 88f068d..5c4f18f 100644 --- a/R/get_survdat_scallop_data.R +++ b/R/get_survdat_scallop_data.R @@ -6,8 +6,6 @@ #' @param filterByYear Numeric vector. Subset of years from which to pull data. #' If not specified then all years are pulled. (Default = NA) #' @param shg.check Boolean. use only SHG <=136 or TOGA <= 1324 (>2008). (Default = T) -#' @param conversion.factor Boolean. Whether to apply conversion factors to the data pull, (Default = T) -#' @param getBio Boolean. Include biology data for each fish weight, sex,, stomach weight, stomach volume, age, maturity #' @param getLengths Boolean. Include length data which includes the length in #' cm and the number at length. (Default = T) #' @param getWeightLength Boolean. Include the weight at length by applying length @@ -88,8 +86,12 @@ #' #'@export -get_survdat_scallop_data <- function(channel, filterByYear = NA, shg.check = T, - getLengths = T, getWeightLength = F, scallopOnly = T){ +get_survdat_scallop_data <- function(channel, + filterByYear = NA, + shg.check = T, + getLengths = T, + getWeightLength = F, + scallopOnly = T){ call <- capture_function_call() diff --git a/man/calc_stratified_mean.Rd b/man/calc_stratified_mean.Rd index e6cdb1a..6fdd3ff 100644 --- a/man/calc_stratified_mean.Rd +++ b/man/calc_stratified_mean.Rd @@ -72,12 +72,8 @@ calc_stratified_mean(surveyData=data$survdat, areaPolygon=area, areaDescription= Other survdat: \code{\link{calc_swept_area}()}, \code{\link{get_area}()}, -\code{\link{get_length_weight}()}, \code{\link{get_survdat_clam_data}()}, \code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} +\code{\link{get_survdat_scallop_data}()} } \concept{survdat} diff --git a/man/calc_swept_area.Rd b/man/calc_swept_area.Rd index 8ec83c1..8767575 100644 --- a/man/calc_swept_area.Rd +++ b/man/calc_swept_area.Rd @@ -76,12 +76,8 @@ calc_swept_area(surveyData=data$survdat, areaPolygon=area, areaDescription="EPU" Other survdat: \code{\link{calc_stratified_mean}()}, \code{\link{get_area}()}, -\code{\link{get_length_weight}()}, \code{\link{get_survdat_clam_data}()}, \code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} +\code{\link{get_survdat_scallop_data}()} } \concept{survdat} diff --git a/man/get_area.Rd b/man/get_area.Rd index 717eb5a..f4db4ec 100644 --- a/man/get_area.Rd +++ b/man/get_area.Rd @@ -40,12 +40,8 @@ get_area(areaPolygon = area, areaDescription="STRATA") Other survdat: \code{\link{calc_stratified_mean}()}, \code{\link{calc_swept_area}()}, -\code{\link{get_length_weight}()}, \code{\link{get_survdat_clam_data}()}, \code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} +\code{\link{get_survdat_scallop_data}()} } \concept{survdat} diff --git a/man/get_conversion_factors.Rd b/man/get_conversion_factors.Rd index 8fc8f95..70d0bd3 100644 --- a/man/get_conversion_factors.Rd +++ b/man/get_conversion_factors.Rd @@ -40,6 +40,7 @@ get_conversion_factors(channel) Other helper: \code{\link{get_cruise_purpose}()}, +\code{\link{get_length_weight}()}, \code{\link{get_maturity}()}, \code{\link{get_sex}()}, \code{\link{get_sex_fscs}()}, diff --git a/man/get_cruise_purpose.Rd b/man/get_cruise_purpose.Rd index 9aaf732..1b82c62 100644 --- a/man/get_cruise_purpose.Rd +++ b/man/get_cruise_purpose.Rd @@ -40,6 +40,7 @@ get_cruise_purpose(channel) Other helper: \code{\link{get_conversion_factors}()}, +\code{\link{get_length_weight}()}, \code{\link{get_maturity}()}, \code{\link{get_sex}()}, \code{\link{get_sex_fscs}()}, diff --git a/man/get_length_weight.Rd b/man/get_length_weight.Rd index 49a1f98..4b14e20 100644 --- a/man/get_length_weight.Rd +++ b/man/get_length_weight.Rd @@ -7,7 +7,9 @@ get_length_weight(channel) } \arguments{ -\item{channel}{an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}})} +\item{channel}{an Object inherited from \link[DBI]{DBIConnection-class}. +This object is used to communicate with the database engine. + (see \code{\link[dbutils]{connect_to_database}})} } \value{ Returns a data.table (nx12). @@ -21,15 +23,14 @@ Pulls the length-weight coefficients from SVDBS LENGTH_WEIGHT_COEFFICIENTS table These coefficients are described in NOAA Tech Memo NMFS-NE-171. } \seealso{ -Other survdat: -\code{\link{calc_stratified_mean}()}, -\code{\link{calc_swept_area}()}, -\code{\link{get_area}()}, -\code{\link{get_survdat_clam_data}()}, -\code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} +Other helper: +\code{\link{get_conversion_factors}()}, +\code{\link{get_cruise_purpose}()}, +\code{\link{get_maturity}()}, +\code{\link{get_sex}()}, +\code{\link{get_sex_fscs}()}, +\code{\link{get_species}()}, +\code{\link{get_strata}()}, +\code{\link{get_vessel}()} } -\concept{survdat} +\concept{helper} diff --git a/man/get_maturity.Rd b/man/get_maturity.Rd index 1828049..efe3f00 100644 --- a/man/get_maturity.Rd +++ b/man/get_maturity.Rd @@ -41,6 +41,7 @@ get_maturity(channel) Other helper: \code{\link{get_conversion_factors}()}, \code{\link{get_cruise_purpose}()}, +\code{\link{get_length_weight}()}, \code{\link{get_sex}()}, \code{\link{get_sex_fscs}()}, \code{\link{get_species}()}, diff --git a/man/get_sex.Rd b/man/get_sex.Rd index f1de4a6..6643a8e 100644 --- a/man/get_sex.Rd +++ b/man/get_sex.Rd @@ -41,6 +41,7 @@ get_sex(channel) Other helper: \code{\link{get_conversion_factors}()}, \code{\link{get_cruise_purpose}()}, +\code{\link{get_length_weight}()}, \code{\link{get_maturity}()}, \code{\link{get_sex_fscs}()}, \code{\link{get_species}()}, diff --git a/man/get_sex_fscs.Rd b/man/get_sex_fscs.Rd index ae34f13..6384adb 100644 --- a/man/get_sex_fscs.Rd +++ b/man/get_sex_fscs.Rd @@ -41,6 +41,7 @@ get_sex_fscs(channel) Other helper: \code{\link{get_conversion_factors}()}, \code{\link{get_cruise_purpose}()}, +\code{\link{get_length_weight}()}, \code{\link{get_maturity}()}, \code{\link{get_sex}()}, \code{\link{get_species}()}, diff --git a/man/get_species.Rd b/man/get_species.Rd index 9701dc4..e08691a 100644 --- a/man/get_species.Rd +++ b/man/get_species.Rd @@ -57,6 +57,7 @@ get_species(channel,species= c("73","135")) Other helper: \code{\link{get_conversion_factors}()}, \code{\link{get_cruise_purpose}()}, +\code{\link{get_length_weight}()}, \code{\link{get_maturity}()}, \code{\link{get_sex}()}, \code{\link{get_sex_fscs}()}, diff --git a/man/get_strata.Rd b/man/get_strata.Rd index 22c0bf8..fdcfe99 100644 --- a/man/get_strata.Rd +++ b/man/get_strata.Rd @@ -56,6 +56,7 @@ get_strata(channel,strata=c("6940","6400","6380")) Other helper: \code{\link{get_conversion_factors}()}, \code{\link{get_cruise_purpose}()}, +\code{\link{get_length_weight}()}, \code{\link{get_maturity}()}, \code{\link{get_sex}()}, \code{\link{get_sex_fscs}()}, diff --git a/man/get_survdat_clam_data.Rd b/man/get_survdat_clam_data.Rd index 7716d26..65dce17 100644 --- a/man/get_survdat_clam_data.Rd +++ b/man/get_survdat_clam_data.Rd @@ -85,11 +85,7 @@ Other survdat: \code{\link{calc_stratified_mean}()}, \code{\link{calc_swept_area}()}, \code{\link{get_area}()}, -\code{\link{get_length_weight}()}, \code{\link{get_survdat_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} +\code{\link{get_survdat_scallop_data}()} } \concept{survdat} diff --git a/man/get_survdat_data.Rd b/man/get_survdat_data.Rd index 636b19e..cf0198c 100644 --- a/man/get_survdat_data.Rd +++ b/man/get_survdat_data.Rd @@ -1,21 +1,9 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/get_cruise_data.R, R/get_survdat_data.R +% Please edit documentation in R/get_survdat_data.R \name{get_survdat_data} \alias{get_survdat_data} -\title{Extracts Survey data from Database} +\title{Extracts Survey data from Database} \usage{ -get_survdat_data( - channel, - filterByYear = NA, - all.season = F, - shg.check = T, - conversion.factor = T, - use.SAD = F, - getBio = F, - getLengths = T, - getWeightLength = F -) - get_survdat_data( channel, filterByYear = NA, @@ -29,7 +17,9 @@ get_survdat_data( ) } \arguments{ -\item{channel}{an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}})} +\item{channel}{an Object inherited from \link[DBI]{DBIConnection-class}. +This object is used to communicate with the database engine. + (see \code{\link[dbutils]{connect_to_database}})} \item{filterByYear}{Numeric vector. Subset of years from which to pull data. If not specified then all years are pulled. (Default = NA)} @@ -42,7 +32,8 @@ If not specified then all years are pulled. (Default = NA)} \item{use.SAD}{Boolean. Use Survey Analysis Database (SAD) for assessed species. (Default = F)} -\item{getBio}{Boolean. Include biology data for each fish weight, sex,, stomach weight, stomach volume, age, maturity} +\item{getBio}{Boolean. Include biology data for each fish weight, sex, +stomach weight, stomach volume, age, maturity} \item{getLengths}{Boolean. Include length data which includes the length in cm and the number at length. (Default = T)} @@ -51,9 +42,10 @@ cm and the number at length. (Default = T)} weight coefficients from SVDBS. (Default = F)} } \value{ -A list containing a Data frame (data.table) (n x 21), a list of SQL queries used to pull the data, -the date of the pull, and the call expression -Each row of the data.table represents the number at length of a species on a specific tow along with physical attributes of the tow. +A list containing a Data frame (data.table) (n x 21), +a list of SQL queries used to pull the data, the date of the pull, +and the call expression. Each row of the data.table represents the number +at length of a species on a specific tow along with physical attributes of the tow. The data frame (Descriptions taken from NEFSC Data dictionary) @@ -101,72 +93,15 @@ The list of sql statements: The date: - \item{pullDate}{The date the data was pulled from the database} +\item{pullDate}{The date the data was pulled from the database} The expression: -\item{functionCall}{The call used to create the data pul} - -A list containing a Data frame (data.table) (n x 21), a list of SQL queries used to pull the data, -the date of the pull, and the call expression -Each row of the data.table represents the number at length of a species on a specific tow along with physical attributes of the tow. - -The data frame (Descriptions taken from NEFSC Data dictionary) - -\item{CRUISE6}{Code uniquely identifying cruise. The first four digits indicate the year and the last two digit uniquely identify the cruise within the year. The 5th byte signifies cruises other than groundfish: Shrimp survey = 7 (i.e. 201470), State of Massachusetts survey = 9 (i.e. 201491), Food habits = 5 (i.e.199554)} -\item{STATION}{Unique sequential order in which stations have been completed. Hangups and short tows each receive a non-repeated consecutive number.} -\item{STRATUM}{ A predefined area where a net dredge, or other piece of gear was deployed. Code consists of 2 parts: Stratum group code number (2 bytes) and stratum number (3 bytes). Stratum group refers to if area fished is inshore or offshore North or South of Cape Hatteras or the type of cruise (shellfish, State of MA, offshore deepwater). The stratum number (third and fourth digits of code) refers to area defined by depth zone. See SVDBS.SVMSTRATA. The fifth digit of the code increases the length of the stratum number for revised strata after the Hague Line was established. Stratum group code: 01 = Trawl, offshore north of Hatteras; 02 = BIOM; 03 = Trawl, inshore north of Hatteras; 04 = Shrimp; 05 = Scotian shelf; 06 = Shellfish; 07 = Trawl, inshore south of Hatteras; 08 = Trawl, Offshore south of Hatteras; 09 = MA DMF; 99 = Offshore deepwater (outside the stratified area). A change in Bottom Trawl Stratum for the Gulf of Maine-Bay of Fundy has been in effect since Spring 1987, and may be summarized as follows: Previous strata: 01350; Present strata: 01351, 01352.} -\item{TOW}{Sequential number representing order in which station was selected within a stratum.} -\item{SVSPP}{A standard code which represents a species caught in a trawl or dredge. Refer to the SVDBS.SVSPECIES_LIST} -\item{CATCHSEX}{Code used to identify species that are sexed at the catch level. See SVDBS.SEX_CODES} -\item{SVVESSEL}{Standard two character code for a survey vessel. Refer to SVDBS.SV_VESSEL} -\item{YEAR}{ Year in which cruise was conducted.} -\item{SEASON}{ Season of the year in which cruise was conducted.} -\item{LAT}{Beginning latitude of tow in decimal degrees.(DECDEG_BEGLAT)} -\item{LON}{Beginning longitude of tow in decimal degrees.(DECDEG_BEGLON)} -\item{EST_TOWDATE}{Date and time represented by Eastern Standard Time (EST) for the start of a tow or deployment.(BEGIN_EST_TOWDATE)} -\item{DEPTH}{ A four digit number recording the average depth, to the nearest meter, during a survey gear deployment.(AVGDEPTH)} -\item{SURFTEMP}{Surface temperature of water (degrees Celcius).} -\item{SURFSALIN}{Salinity at water surface in practical salinity units (PSU).} -\item{BOTTEMP}{Bottom temperature (degrees Celsius).} -\item{BOTSALIN}{Bottom salinity in Practical Salinity Units (PSU).} -\item{ABUNDANCE}{Expanded number of individuals of a species caught at a given station.(EXPCATCHNUM)} -\item{BIOMASS}{Expanded catch weight of a species caught at a given station. (EXPCATCHWT)} -\item{LENGTH}{Measured length of species in centimeters (cm). Measure method differs by species.} -\item{NUMLEN}{Expanded number of specimens at a given length.(EXPNUMLEN)} - -Additional columns if bio = T (UNION_FSCS_SVBIO) - -\item{INDID}{A unique identifier for each fish sampled.} -\item{INDWT}{Individual weight (KG) of species being sampled.} -\item{SEX}{Code indicating sex of fish or invertebrate species. See SVDBS.FSCS_SEX_CODES if using fscs data and SVDBS.SEX_CODES if using non FSCS data. Codes 0, 1, 2 and 4 are the only valid codes in fscs tables.} -\item{MATURITY}{Stage of maturation of the fish being sampled. See SVDBS.FSCS_MATURITY_CODES} -\item{AGE}{Age of specimen in years.} -\item{STOM_VOLUME}{Volume of the stomach contents of the fish sampled, measured to the nearest tenth of a cubic centimeter (cc).} -\item{STOM_WGT}{ Stomach weight of an individual fish in grams.} - -The list of sql statements: - -\item{cruise}{Select unique list of cruises. Table = MSTR_CRUISE} -\item{station}{Select unique set of stations from result of \code{cruise}. Table = UNION_FSCS_SVSTA} -\item{catch}{Select species abundance and biomass data from result of \code{station}. Table = UNION_FSCS_SVCAT} -\item{length}{Select Lengths of species found in \code{catch}. Table = UNION_FSCS_SVLEN} -\item{sad}{Select ????? from Survey Analysis Database. Table = STOCKEFF.I_SV_MERGED_CATCH_CALIB_O} -\item{conversions}{Select conversion factors. Table = SURVAN_CONVERSION_FACTORS} -\item{bio}{Select bio stats. Table = UNION_FSCS_SVBIO} - -The date: - - \item{pullDate}{The date the data was pulled from the database} - -The expression: - -\item{functionCall}{The call used to create the data pul} +\item{functionCall}{The call used to create the data pull} } \description{ -Connects to svdbs and pulls data from MSTR_CRUISE, UNION_FSCS_SVCAT, UNION_FSCS_SVLEN, UNION_FSCS_SVSTA, UNION_FSCS_SVBIO - -Connects to svdbs and pulls data from MSTR_CRUISE, UNION_FSCS_SVCAT, UNION_FSCS_SVLEN, UNION_FSCS_SVSTA, UNION_FSCS_SVBIO +Connects to svdbs and pulls data from MSTR_CRUISE, UNION_FSCS_SVCAT, +UNION_FSCS_SVLEN, UNION_FSCS_SVSTA, UNION_FSCS_SVBIO } \examples{ \dontrun{ @@ -180,39 +115,13 @@ get_survdat_data(channel,conversion.factor = T, bio=T) get_survdat_data(channel,conversion.factor = T, bio=F) } -\dontrun{ -# First create a connection object to the database -channel <- dbutils::connect_to_database("serverName","userName") -# pull survey data, applies conversion factors (Door, net, vessel) and join with -# biological data to return individual sex, age, maturity, stomach data -get_survdat_data(channel,conversion.factor = T, bio=T) - -# Same data pull without individual biological data -get_survdat_data(channel,conversion.factor = T, bio=F) -} - } \seealso{ Other survdat: \code{\link{calc_stratified_mean}()}, \code{\link{calc_swept_area}()}, \code{\link{get_area}()}, -\code{\link{get_length_weight}()}, -\code{\link{get_survdat_clam_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} - -Other survdat: -\code{\link{calc_stratified_mean}()}, -\code{\link{calc_swept_area}()}, -\code{\link{get_area}()}, -\code{\link{get_length_weight}()}, \code{\link{get_survdat_clam_data}()}, -\code{\link{get_survdat_scallop_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} +\code{\link{get_survdat_scallop_data}()} } \concept{survdat} diff --git a/man/get_survdat_scallop_data.Rd b/man/get_survdat_scallop_data.Rd index 5aef084..14247f0 100644 --- a/man/get_survdat_scallop_data.Rd +++ b/man/get_survdat_scallop_data.Rd @@ -28,10 +28,6 @@ cm and the number at length. (Default = T)} weight coefficients from SVDBS. (Default = F)} \item{scallopOnly}{Boolean. Include only sea scallop catch. (Default = T)} - -\item{conversion.factor}{Boolean. Whether to apply conversion factors to the data pull, (Default = T)} - -\item{getBio}{Boolean. Include biology data for each fish weight, sex,, stomach weight, stomach volume, age, maturity} } \value{ A list containing a Data frame (data.table) (n x 21), a list of SQL queries used to pull the data, @@ -111,11 +107,7 @@ Other survdat: \code{\link{calc_stratified_mean}()}, \code{\link{calc_swept_area}()}, \code{\link{get_area}()}, -\code{\link{get_length_weight}()}, \code{\link{get_survdat_clam_data}()}, -\code{\link{get_survdat_data}()}, -\code{\link{post_strat}()}, -\code{\link{strat_mean}()}, -\code{\link{swept_area}()} +\code{\link{get_survdat_data}()} } \concept{survdat} diff --git a/man/get_vessel.Rd b/man/get_vessel.Rd index 8ad9634..7234997 100644 --- a/man/get_vessel.Rd +++ b/man/get_vessel.Rd @@ -41,6 +41,7 @@ get_vessel(channel) Other helper: \code{\link{get_conversion_factors}()}, \code{\link{get_cruise_purpose}()}, +\code{\link{get_length_weight}()}, \code{\link{get_maturity}()}, \code{\link{get_sex}()}, \code{\link{get_sex_fscs}()}, diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml index 4391482..5a5db19 100644 --- a/pkgdown/_pkgdown.yml +++ b/pkgdown/_pkgdown.yml @@ -6,15 +6,7 @@ reference: - contents: - calc_swept_area - calc_stratified_mean -- title: "Intermediate Functions" - desc: "Functions used in estimation" -- contents: - get_area - - post_strat - - strat_prep - - strat_mean - - swept_area - - apply_conversion_factors - title: "Get Data" desc: "Functions that pull data from the SVDBS database" - contents: @@ -34,7 +26,6 @@ reference: - get_vessel - get_conversion_factors - get_length_weight - - get_length_weight_data - title: "Plotting" desc: "Basic plotting functions to visualize data" - contents: From cac942b399c1615269f93dfcc1f9b8e59242110c Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Thu, 30 May 2024 16:55:57 -0400 Subject: [PATCH 05/16] ignore other folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a33b88f..625ea64 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ inst/doc #folder testing docs +other From 73d2315e4db4dfec4cf82cb870f2e8832314429e Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Fri, 31 May 2024 14:52:24 -0400 Subject: [PATCH 06/16] added the sql statement to the return value --- R/apply_conversion_factors.r | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/apply_conversion_factors.r b/R/apply_conversion_factors.r index 6c27538..985b21d 100644 --- a/R/apply_conversion_factors.r +++ b/R/apply_conversion_factors.r @@ -31,7 +31,9 @@ apply_conversion_factors <- function(channel,survdat.raw,use.SAD = F) { survdat[, ABUNDANCE := as.double(ABUNDANCE)] #Grab all conversion factors off the network - convert <- data.table::as.data.table(get_conversion_factors(channel)$data) + conversionFactors <- get_conversion_factors(channel) + convert.qry <- conversionFactors$sql + convert <- data.table::as.data.table(conversionFactors$data) #DCF < 1985 Door Conversion dcf.spp <- convert[DCF_WT > 0, SVSPP] From 071d1f91d6b3dfd8d26d4f2ae859d43ca0c95afa Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:28:18 -0400 Subject: [PATCH 07/16] adde package version as part of data pull metadata --- R/get_survdat_data.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/get_survdat_data.R b/R/get_survdat_data.R index 9615175..7ccb964 100644 --- a/R/get_survdat_data.R +++ b/R/get_survdat_data.R @@ -99,6 +99,8 @@ get_survdat_data <- function(channel, filterByYear = NA, all.season = F, getBio = F, getLengths = T, getWeightLength = F) { call <- capture_function_call() + version <- packageVersion("survdat") + # Cruise List -------------------------------------------------------------- #Generate cruise list @@ -293,6 +295,6 @@ get_survdat_data <- function(channel, filterByYear = NA, all.season = F, - return(list(survdat=survdat,sql=sql,pullDate=date(),functionCall = call)) + return(list(survdat=survdat,sql=sql,pullDate=date(),functionCall = call,version=version)) } From 738a77e717c0c9ff995de487e4b2efe6f9e7ee82 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 4 Jun 2024 15:41:00 -0400 Subject: [PATCH 08/16] updated new to reflect first survdat release --- news.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/news.md b/news.md index 07fd877..94706d2 100644 --- a/news.md +++ b/news.md @@ -1,4 +1,6 @@ -# survdat version history +# survdat 1.0 + +Benchmark release From 7acfe2778796ef3e7168814d72ac802d27a32408 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:48:07 -0400 Subject: [PATCH 09/16] removed news.md --- news.md | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 news.md diff --git a/news.md b/news.md deleted file mode 100644 index 94706d2..0000000 --- a/news.md +++ /dev/null @@ -1,6 +0,0 @@ -# survdat 1.0 - -Benchmark release - - - From b7963b70ba52ca74559a914c3758b5e370cfba1e Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:48:47 -0400 Subject: [PATCH 10/16] added NEWS.md --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 NEWS.md diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..c4b36af --- /dev/null +++ b/NEWS.md @@ -0,0 +1,6 @@ +# survdat 1.0 + +Benchmark release + + + From 1de527eac618741ae4dfc0ccd493597a7015ffce Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:23:09 -0400 Subject: [PATCH 11/16] Added kableExtra as a Suggests dependency --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7ab9f34..78bd92e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -35,5 +35,6 @@ RoxygenNote: 7.3.1 Suggests: knitr, rmarkdown, - markdown + markdown, + kableExtra VignetteBuilder: knitr From 07f111cd279d40503ee3e36f6ee1400179232367 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:24:20 -0400 Subject: [PATCH 12/16] updated survdat.rmd to fix getBio issue and to update data pull output --- vignettes/survdat.Rmd | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/vignettes/survdat.Rmd b/vignettes/survdat.Rmd index 2d23ead..afe533b 100644 --- a/vignettes/survdat.Rmd +++ b/vignettes/survdat.Rmd @@ -28,7 +28,7 @@ Once you are granted permission to access the database and have Oracle's instant ## Pull Data -Pulling the data takes up to 15 minutes to complete. see `get_survdat_data` for details. To pull the survey data with conversion corrections applied use the following: +Pulling the data can take up to 15 minutes to complete. see `get_survdat_data` for details. To pull the survey data with conversion corrections applied use the following: ``` data <- get_survdat_data(channel) @@ -37,11 +37,18 @@ data <- get_survdat_data(channel) To pull biological traits of individual fish sampled on the survey cruise ``` -data <- get_survdat_data(channel, bio=TRUE) +data <- get_survdat_data(channel, getBio=TRUE) ``` -`data` is a list of 2 elements. The data pull and the set of sql statements made to pull the data +`data` is a list of 5 elements. +* The data +* The set of sql statements made to pull the data +* The function call statement +* The date of the pull +* The version of the package used to create the data pull + +For detailed information ## Plot the shapefiles @@ -58,7 +65,7 @@ plot_shapefile(areaPolygon) ## Stratified Mean - +Under development. ## Swept area biomass From 66eba2c78e76946caa3eafe8c1abc92517403a34 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:26:35 -0400 Subject: [PATCH 13/16] added vignettes as placeholders - lookupTables (describe helper functions) - pullingData (different flags and corresponding output) - surveyDesign (brief history of changes) --- vignettes/lookupTables.Rmd | 26 +++++++++++++++++++ vignettes/pullingData.Rmd | 47 +++++++++++++++++++++++++++++++++ vignettes/surveyDesign.Rmd | 53 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 vignettes/lookupTables.Rmd create mode 100644 vignettes/pullingData.Rmd create mode 100644 vignettes/surveyDesign.Rmd diff --git a/vignettes/lookupTables.Rmd b/vignettes/lookupTables.Rmd new file mode 100644 index 0000000..921f4fd --- /dev/null +++ b/vignettes/lookupTables.Rmd @@ -0,0 +1,26 @@ +--- +title: "Lookup tables" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Lookup tables} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(survdat) +``` + +Bundled with the package are several helper functions to compliment the main data pulls + +## Species tables + +## Strata tables + diff --git a/vignettes/pullingData.Rmd b/vignettes/pullingData.Rmd new file mode 100644 index 0000000..dfedc2e --- /dev/null +++ b/vignettes/pullingData.Rmd @@ -0,0 +1,47 @@ +--- +title: "Pulling Data" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Pulling Data} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(survdat) +``` + +Explain the differences between resulting data objects based on argument flags. + +In Development + + +## Biological data + +When using the flag `getBio = T` additional fields are pulled, namely Maturity, Age, Individual weight, stomach volume, stomach weight, and sex. Species that have lengths but no additional "biological" data are dropped from the data pull. + +## Weight-Length data + +Length-weight conversion coefficients are applied using values from Wigley et al. These reside in the table LENGTH_WEIGHT_COEFFICIENTS + +## Length data + +## Calibration factors + +These reside in the table SURVAN_CONVERSION_FACTORS + +Describe how they are applied + +### Vessel effects + +### Gear Effects + +### Door Effects + diff --git a/vignettes/surveyDesign.Rmd b/vignettes/surveyDesign.Rmd new file mode 100644 index 0000000..7bec674 --- /dev/null +++ b/vignettes/surveyDesign.Rmd @@ -0,0 +1,53 @@ +--- +title: "Survey Design" +output: rmarkdown::html_vignette +bibliography: refs.bib +link-citations: true +vignette: > + %\VignetteIndexEntry{Survey Design} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(survdat) +``` + +The Northeast Fisheries Science Center (NEFSC) has been conducting standardized bottom trawl surveys +in the fall since 1963 and spring since 1968. The surveys follow a stratified random design. Fish +species and several invertebrate species are enumerated on a tow by tow basis [@Azarovitz_1981]. The data are housed in the NEFSC's survey database (SVDBS) maintained by the Ecosystem Survey Branch. + +Direct pulls from the database are not advisable as there have been several gear modifications and +vessel changes over the course of the time series [@Miller_2010]. T R package `survdat` was developed as a database query that applies the appropriate calibration factors for a seamless time series since the 1960s. + + +The R package `survdat` is used to pull and process the data. `survdat` identifies those research cruises associated with the seasonal bottom trawl surveys and pulls the station and biological data. Station data includes tow identification (cruise, station, and stratum), tow location and date, as well as several environmental variables (depth, surface/bottom salinity, and surface/bottom temperature). Stations are filtered using a station, haul, gear (SHG) code for tows prior to 2009 and a tow, operations, gear, and aquisition (TOGA) code from 2009 onward. The codes that correspond to a representative tow (SHG <= 136 or TOGA <= 1324) are the same used by assessment biologists at the NEFSC. Biological data includes the total biomass and abundance by species, as well as lengths and number at length. + +`survdat` applies the calibration factors. There are four calibrartion factors applied (Table 64.1). Calibration factors are pulled directly from SVDBS. Vessel conversions were made from either the NOAA Ship **Delaware II** or NOAA Ship **Henry Bigelow** to the NOAA **Ship Albatross IV** which was the primary vessel for most of the time series. The Albatross was decommissioned in 2009 and the Bigelow is now the primary vessel for the bottom trawl survey. + + +```{r calibration, eval = T, echo = F} +cal.factors <- data.frame(Name = c('Door Conversion', 'Net Conversion', 'Vessel Conversion I', 'Vessel Conversion II'), + Code = c('DCF', 'GCF', 'VCF', 'BCF'), + Applied = c('<1985', '1973 - 1981 (Spring)', 'Delaware II records', 'Henry Bigelow records')) +kableExtra::kable(cal.factors, booktabs = TRUE, + caption = "Calibration factors for NEFSC trawl survey data") +``` + +Describe a little about the survey process that pertains to the data. + +eg. + +Since 19xx all species landed on the survey vessel are measured for length. A subset of these are then individually weighed, sexed, assessed for stage of maturity, and undergo a stomach content analysis + +At the tow level ... expanded number, expanded weight etc.. + + +## References From ae239c0b5010679a47ee36f38bd43de098debe59 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:26:57 -0400 Subject: [PATCH 14/16] added references .bib --- vignettes/refs.bib | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 vignettes/refs.bib diff --git a/vignettes/refs.bib b/vignettes/refs.bib new file mode 100644 index 0000000..2268258 --- /dev/null +++ b/vignettes/refs.bib @@ -0,0 +1,23 @@ + +@article{Azarovitz_1981, + title = {A brief historical review of the {Woods} {Hole} {Laboratory} trawl survey time series}, + volume = {58}, + shorttitle = {A brief historical review of the {Woods} {Hole} {Laboratory} trawl survey time series}, + journal = {Canadian Special Publication of Fisheries and Aquatic Sciences}, + author = {Azarovitz, T.R.}, + year = {1981}, + keywords = {survdat}, + pages = {62--67} +} + +@techreport{Miller_2010, + address = {Woods Hole, MA}, + title = {Estimation of {Albatross} {IV} to {Henry} {B}. {Bigelow} calibration factors}, + number = {10-05}, + institution = {National Marine Fisheries Service}, + author = {Miller, T. J. and Das, C. and Politis, P. J. and Miller, A. S. and Lucey, S. M. and Legault, C. M. and Brown, R. W. and Rago, P. J.}, + year = {2010}, + keywords = {survdat}, + pages = {233}, + url = {https://www.nefsc.noaa.gov/publications/crd/crd1005/crd1005.pdf} +} From 99f759679bf48fe036418f4d4cbf7f5d0e088462 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:27:23 -0400 Subject: [PATCH 15/16] added a articles section and organized vignettes --- pkgdown/_pkgdown.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml index 5a5db19..17d7f9d 100644 --- a/pkgdown/_pkgdown.yml +++ b/pkgdown/_pkgdown.yml @@ -36,4 +36,22 @@ reference: - contents: - survdat +navbar: + components: + articles: + text: Articles + menu: + - text: Introduction + - text: Survey design + href: articles/surveyDesign.html + - text: Pulling the data + href: articles/pullingData.html + - text: Lookup tables + href: articles/lookupTables.html + - text: -------- + - text: Methodology + - text: Estimating Biomass + href: articles/methodology.html + - text: Estimating stratified mean + href: articles/calc_Strat_mean.html From 36939adce52186cca55db7ed9573fca882d76caf Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:28:04 -0400 Subject: [PATCH 16/16] blank line changes --- other/Prestrat.r | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/other/Prestrat.r b/other/Prestrat.r index 5689434..6798683 100644 --- a/other/Prestrat.r +++ b/other/Prestrat.r @@ -8,11 +8,11 @@ prestrat <- function (survdat, areas, strat.col, area.col = 'area') { #strat.col <- column of survdat and areas with the strata names #area.col <- column of areas with the area of the stratum in square kilometers #sp.col <- column of survdat with the species codes of interest - + #------------------------------------------------------------------------------- #Required packages library(data.table) - + #------------------------------------------------------------------------------- #User created functions #count occurances @@ -21,15 +21,15 @@ prestrat <- function (survdat, areas, strat.col, area.col = 'area') { out <- sum(num) return(out) } - + #------------------------------------------------------------------------------- x <- copy(survdat) y <- copy(areas) - + setnames(x, strat.col, 'STRAT') - setnames(y, c(strat.col, area.col), + setnames(y, c(strat.col, area.col), c('STRAT', 'S.AREA')) - + #Station data - remove catch/length setkey(x, CRUISE6, STRAT, STATION) stations <- unique(x) @@ -55,9 +55,9 @@ prestrat <- function (survdat, areas, strat.col, area.col = 'area') { #Merge catch with station data strat.survdat <- merge(x, stations, by = c('YEAR', 'CRUISE6', 'STRAT', 'STATION')) - + setnames(strat.survdat, c('STRAT', 'S.AREA'), c(strat.col, area.col)) - + return(strat.survdat) - } \ No newline at end of file + }