diff --git a/NAMESPACE b/NAMESPACE index a41500c..6292bcf 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,6 +1,15 @@ # Generated by roxygen2: do not edit by hand +export(aggregate_area) +export(aggregate_gear) +export(assign_area) +export(calc_DK) export(comland) +export(disaggregate_skates_hakes) +export(get_comdisc_raw_data) +export(get_comland_data) +export(get_comland_raw_data) +export(get_herring_data) export(get_areas) export(get_comland_data) export(get_foreign_data) diff --git a/R/adjust_inflation.R b/R/adjust_inflation.R index 9cd6f97..f15b0db 100644 --- a/R/adjust_inflation.R +++ b/R/adjust_inflation.R @@ -6,8 +6,8 @@ #' #' #'@param comland Data frame. master data frame containing species landings -#'@param refmonth Integer. Reference month -#'@param refyear Integer. Reference year +#'@param refMonth Integer. Reference month +#'@param refYear Integer. Reference year #' #'@return comland data frame adjusted for inflation #' @@ -16,7 +16,17 @@ -adjust_inflation <- function(comland,refyear,refmonth){ +adjust_inflation <- function(comland, refYear, refMonth){ + + call <- c(comland$call, capture_function_call()) + + #Pulling data + message("Adjusting for inflation ...") + + #pull out comland data + sql <- comland$sql + comland <- comland$comland + temp <- tempfile() download.file("http://download.bls.gov/pub/time.series/wp/wp.data.3.ProcessedFoods", temp) inflate <- data.table::as.data.table(read.delim(temp)) @@ -37,5 +47,9 @@ adjust_inflation <- function(comland,refyear,refmonth){ #Remove extra column comland[, PPI := NULL] - return(comland) + + return(list(comland = comland[], + sql = sql, + pullDate = date(), + functionCall = call)) } diff --git a/R/aggregate_area.R b/R/aggregate_area.R new file mode 100644 index 0000000..bac11b8 --- /dev/null +++ b/R/aggregate_area.R @@ -0,0 +1,60 @@ +#' Assign landing records to an aggregated area +#' +#' Takes the output from \code{get_comland_data} and further aggregates from NAFO +#' statistical areas to a user defined area. Allows for species to be assigned by +#' proportions to more than two user defined areas from one stat area +#' +#'@param comland Data set generated by \code{get_comland_data} +#'@param userAreas Data frame. Definitions to aggregate statistical areas to user defined +#' areas +#'@param areaDescription Character. Name of column in userAreas that defines the new +#' area. +#'@param propDescription Character. Name of column in userAreas that defines the +#' proportions of landings assigned to new area. +#' +#'@export + +aggregate_area <- function(comland, userAreas, areaDescription, propDescription, + applyPropValue = T){ + + #Pulling data + message("Aggregating Areas ...") + + #Grab just the data + comData <- comland$comland + + call <- dbutils::capture_function_call() + + #Convert userAreas to data.table + areas <- data.table::as.data.table(userAreas) + data.table::setnames(areas, c(areaDescription, propDescription), c('newarea', 'prop')) + + #Merge new area descriptions to landings + new.area <- merge(comData, areas, by = c('NESPP3', 'AREA'), all.x = T, allow.cartesian=TRUE) + + #If no proportion assume 100% in + new.area[is.na(prop), prop := 1] + + #Proportion landings to new areas + new.area[, newspplivmt := SPPLIVMT * prop] + if(applyPropValue) new.area[, newsppvalue := SPPVALUE * prop] + + #Drop extra columns and rename + if(applyPropValue){ + new.area[, c('SPPLIVMT', 'SPPVALUE', 'prop') := NULL] + data.table::setnames(new.area, c('newarea', 'newspplivmt', 'newsppvalue'), + c(areaDescription, 'SPPLIVMT', 'SPPVALUE')) + } else { + new.area[, c('SPPLIVMT', 'prop') := NULL] + data.table::setnames(new.area, c('newarea', 'newspplivmt'), + c(areaDescription, 'SPPLIVMT')) + } + + + #Add changes back into comland + comland$comland <- new.area[] + comland$call <- c(comland$call, call) + comland$userAreas <- userAreas + + return(comland[]) +} diff --git a/R/aggregate_gear.R b/R/aggregate_gear.R new file mode 100644 index 0000000..a33d1e0 --- /dev/null +++ b/R/aggregate_gear.R @@ -0,0 +1,54 @@ +#' Assign landing records to an aggregated area +#' +#' Takes the output from \code{get_comland_data} and further aggregates from NAFO +#' statistical areas to a user defined area. Allows for species to be assigned by +#' proportions to more than two user defined areas from one stat area +#' +#'@param comland Data set generated by \code{get_comland_data} +#'@param userAreas Data frame. Definitions to aggregate statistical areas to user defined +#' areas +#'@param areaDescription Character. Name of column in userAreas that defines the new +#' area. +#'@param propDescription Character. Name of column in userAreas that defines the +#' proportions of landings assigned to new area. +#' +#'@export + +aggregate_gear <- function(comData, userGears, fleetDescription){ + + call <- dbutils::capture_function_call() + + #Convert userGears to data.table + gears <- data.table::as.data.table(userGears) + gears <- data.table::setnames(gears, fleetDescription, 'fleet') + + #Assign gears to fleets + #Generate NEGEAR2 codes from NEGEAR + if(is.numeric(comData$NEGEAR)){ + comData[NEGEAR < 100, NEGEAR3 := paste0(0, NEGEAR)] + comData[NEGEAR >= 100, NEGEAR3 := NEGEAR] + comData[, NEGEAR2 := as.numeric(substr(NEGEAR3, 1, 2))] + } else { + comData[, NEGEAR2 := as.numeric(substr(NEGEAR, 1, 2))] + } + + fleets <- unique(gears$fleet) + + for(ifleet in 1:length(fleets)){ + fleet.gear <- gears[fleet == fleets[ifleet], NEGEAR2] + fleet.mesh <- unique(gears[fleet == fleets[ifleet], MESHCAT]) + #Check if there is a mesh characteristic associated with this gear + if(is.na(fleet.mesh)){ + comData[NEGEAR2 %in% fleet.gear, fleet := fleets[ifleet]] + } else { + comData[NEGEAR2 %in% fleet.gear & MESHCAT == fleet.mesh, fleet := fleets[ifleet]] + } + } + + comData[, fleet := as.factor(fleet)] + + #Rename columns + data.table::setnames(comData, 'fleet', fleetDescription) + + return(comData[]) +} diff --git a/R/assign_area.R b/R/assign_area.R new file mode 100644 index 0000000..a08e902 --- /dev/null +++ b/R/assign_area.R @@ -0,0 +1,70 @@ +#' Assigns points to polygon +#' +#' Assign observer data (points, lat and lon) to designated regions (polygons) from a shape file. +#' +#' +# @inheritParams strat_prep +#' @param na.keep Boolean. Logical value to indicate whether original strata names +#' should be retained. +#' +#' @return Returns a \code{comdiscData} data.table with one additional column labeled +#' with the value of \code{areaDescription} +#' +#' \item{areaDescription}{The name of the region (found in \code{areaPolygon}) +#' that a record in \code{surveyData} is assigned to} +#' +#' @importFrom magrittr "%>%" +#' +#'@family comdisc +#' +#' @export + + +assign_area <- function (comdiscData, areaPolygon, areaDescription, na.keep = F) { + + # transform Regional Shape file using lambert conformal conic coordinate ref system + crs <- "+proj=lcc +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-72 +x_0=0 +y_0=0 +datum=NAD83 +units=m +no_defs +ellps=GRS80 +towgs84=0,0,0" + + areas <- areaPolygon %>% + dplyr::rename(areaDescription = areaDescription) %>% + sf::st_transform(., crs) + + #Need unique link3, lat lon column to make this work + comdiscData[, linkLL := paste0(LINK3, LAT, LON)] + #Should probably do this in the raw data pull + #remove stations missing lat or lon + comdiscData <- comdiscData[!is.na(LAT), ] + comdiscData <- comdiscData[!is.na(LON), ] + + # find unique stations and transform to required crs + locations <- comdiscData %>% + dplyr::select(linkLL, LAT, LON) %>% + dplyr::distinct() %>% + sf::st_as_sf(., coords = c("LON","LAT"), crs=4326) %>% + sf::st_transform(., crs) + + + # Intersect the locations with the polygon + # Assigns locations with polygons + location_area <- sf::st_join(locations, areas, join = sf::st_intersects) %>% + dplyr::select(names(locations), areaDescription) %>% + sf::st_drop_geometry() %>% + dplyr::arrange(linkLL) + + # Join observer data with locations (which now are assigned to an area based on the shape file) + master <- base::merge(comdiscData, location_area, by = c("linkLL")) %>% + dplyr::rename(!!areaDescription := areaDescription) + + # check to see if we want to keep points that fall outside of all of the polygons found in the shape file + if (!(na.keep)) { # removes all points that fall outside of the areas defined by the polygons in stratum + master <- master %>% + dplyr::filter(!is.na(get(areaDescription))) %>% + data.table::as.data.table() + } + #Drop linkLL column + master[, linkLL := NULL] + + return(master[]) + +} + diff --git a/R/calc_DK.R b/R/calc_DK.R new file mode 100644 index 0000000..8b79047 --- /dev/null +++ b/R/calc_DK.R @@ -0,0 +1,62 @@ +#' Calculate discard to kept ratio +#' +#' Use observer data to calculate the ratio of discards to kept by species. +#' +#' +# @inheritParams strat_prep +#' @param na.keep Boolean. Logical value to indicate whether original strata names +#' should be retained. +#' +#' @return Returns a \code{comdiscData} data.table with one additional column labeled +#' with the value of \code{areaDescription} +#' +#' \item{areaDescription}{The name of the region (found in \code{areaPolygon}) +#' that a record in \code{surveyData} is assigned to} +#' +#' @importFrom magrittr "%>%" +#' +#'@family comdisc +#' +#' @export + + +calc_DK <- function(comdiscData, areaDescription, fleetDescription){ + + #Standardize column names + comdiscData <- data.table::setnames(comdiscData, c(areaDescription, + fleetDescription), + c('area', 'fleet')) + + #sum catch by species/disposition/area/fleet + ob.sums <- comdiscData[, sum(SPPLIVMT), by = c('YEAR', 'area', 'fleet', 'NESPP3', + 'CATDISP')] + #identify discards + ob.discard <- ob.sums[CATDISP == 0, ] + + setnames(ob.discard, "V1", "DISCARD") + ob.discard[, CATDISP := NULL] + + #Sum kept by area/fleet + ob.kept <- ob.sums[CATDISP == 1, sum(V1), by = c('YEAR', 'area', 'fleet')] + + setnames(ob.kept, "V1", "KEPT.ALL") + + #Merge discards and kept + dk <- merge(ob.kept, ob.discard, by = c('YEAR', 'area', 'fleet')) + + #Calculate ratio + dk[, DK := DISCARD / KEPT.ALL] + #NAs result if divide by 0 so set DK to 1 (all discards) + dk[is.na(DK), DK := 1.0] + + #Remove extra columns + dk[, c('KEPT.ALL', 'DISCARD') := NULL] + + #Replace standard column names + #Standardize column names + data.table::setnames(dk, c('area', 'fleet'), + c(areaDescription, fleetDescription)) + + return(dk[]) +} + \ No newline at end of file diff --git a/R/disaggregate_skates_hakes.R b/R/disaggregate_skates_hakes.R new file mode 100644 index 0000000..c54dc86 --- /dev/null +++ b/R/disaggregate_skates_hakes.R @@ -0,0 +1,202 @@ +#'#Disaggregate skates and hakes +#' +#'Determine proportion of little/winter skates and silver hake in landings data 7/13 +#'SML +#' +#'@param comland Data frame. Master data frame containing species landings +#'@param skate.hake.us Data frame. Landings of skates and hakes in USA +#' +#'@return updated comland +#' +#'@importFrom data.table ":=" "key" +#' +#' @noRd +#' @export + +disaggregate_skates_hakes <- function(comland, channel, filterByYear, filterByArea) { + + message("Grabbing survey data to disaggregate skates and hakes ... ") + + #Grab survey data from NEFSC bottom trawl survey + survey <- survdat::get_survdat_data(channel, filterByYear, getLengths = F) + + #Skates---- + message("Disaggregating little and winter skates from skates(ns) ... ") + skates <- 22:28 + skate.survey <- survey$survdat[SVSPP %in% skates, ] + + #Identify Stat areas catch occured in + Stat.areas <- sf::st_read(dsn=system.file("extdata","Statistical_Areas_2010.shp", + package="comlandr"), quiet = T) + skate.survey <- survdat::post_strat(skate.survey, Stat.areas, 'Id') + data.table::setnames(skate.survey, 'Id', 'AREA') + + #Filter By Area + if(!is.na(filterByArea[1])) skate.survey <- skate.survey[AREA %in% filterByArea, ] + + #Figure out proportion of skates + data.table::setkey(skate.survey, YEAR, SEASON, AREA) + + skates.prop <- skate.survey[, .(skates.all = sum(BIOMASS)), + by = key(skate.survey)] + + little <- skate.survey[SVSPP == 26, .(little = sum(BIOMASS)), + by = key(skate.survey)] + + skates.prop <- merge(skates.prop, little, by = key(skate.survey), all = T) + + winter <- skate.survey[SVSPP == 23, .(winter = sum(BIOMASS)), + by = key(skate.survey)] + + skates.prop <- merge(skates.prop, winter, by = key(skate.survey), all = T) + + skates.prop[, little.per := little/skates.all] + skates.prop[, winter.per := winter/skates.all] + + #Drop extra columns and fix NAs + skates.prop[, c('skates.all', 'little', 'winter') := NULL] + skates.prop[is.na(little.per), little.per := 0] + skates.prop[is.na(winter.per), winter.per := 0] + + #disaggregate little and winter skates from skates(ns) - use survey in half years + #Generate season variable in comland + comland.skates <- comland$comland[NESPP3 == 365, ] + comland.skates[MONTH %in% 1:6, SEASON := 'SPRING'] + comland.skates[MONTH %in% 7:12, SEASON := 'FALL'] + + comland.skates <- merge(comland.skates, skates.prop, + by = c('YEAR', 'SEASON', 'AREA'), all.x = T) + + #Fix NAs + comland.skates[is.na(little.per), little.per := 0] + comland.skates[is.na(winter.per), winter.per := 0] + + #Disaggregate + comland.skates[, little := little.per * SPPLIVMT] + comland.skates[, little.value := round(little.per * SPPVALUE)] + + comland.skates[, winter := winter.per * SPPLIVMT] + comland.skates[, winter.value := round(winter.per * SPPVALUE)] + + comland.skates[, other.skate := SPPLIVMT - (little + winter)] + comland.skates[, other.skate.value := SPPVALUE - (little.value + winter.value)] + + #Little (366), winter (367), skates(ns) (365) + #put skates in comland format to merge back + little <- comland.skates[, list(YEAR, AREA, MONTH, NEGEAR, + TONCL1, NESPP3, UTILCD, MESHCAT, MKTCAT, + little, little.value)] + little[, NESPP3 := 366] + data.table::setnames(little, c('little', 'little.value'), c('SPPLIVMT', 'SPPVALUE')) + little <- little[SPPLIVMT > 0, ] + + winter <- comland.skates[, list(YEAR, AREA, MONTH, NEGEAR, + TONCL1, NESPP3, UTILCD, MESHCAT, MKTCAT, + winter, winter.value)] + winter[, NESPP3 := 367] + data.table::setnames(winter, c('winter', 'winter.value'), c('SPPLIVMT', 'SPPVALUE')) + winter <- winter[SPPLIVMT > 0, ] + + other <- comland.skates[, list(YEAR, AREA, MONTH, NEGEAR, + TONCL1, NESPP3, UTILCD, MESHCAT, MKTCAT, + other.skate, other.skate.value)] + other[, NESPP3 := 365] + data.table::setnames(other, c('other.skate', 'other.skate.value'), c('SPPLIVMT', 'SPPVALUE')) + other <- other[SPPLIVMT > 0, ] + + #merge all three and reformat for comland + skates.add.back <- data.table::rbindlist(list(little, winter, other)) + + data.table::setcolorder(skates.add.back, names(comland$comland)) + + comland$comland <- data.table::rbindlist(list(comland$comland[NESPP3 != 365, ], + skates.add.back)) + + #Hakes ---- + message("Disaggregating silver and offshore hake from whiting ... ") + + #Grab hake data from NEFSC bottom trawl survey + hake <- c(69, 72) + hake.survey <- survey$survdat[SVSPP %in% hake, ] + + #Identify Stat areas catch occured in + hake.survey <- survdat::post_strat(hake.survey, Stat.areas, 'Id') + data.table::setnames(hake.survey, 'Id', 'AREA') + + #Filter By Area + if(!is.na(filterByArea[1])) hake.survey <- hake.survey[AREA %in% filterByArea, ] + + #Figure out proportion of skates + data.table::setkey(hake.survey, YEAR, SEASON, AREA) + + hake.prop <- hake.survey[, .(hake.all = sum(BIOMASS, na.rm = T)), + by = key(hake.survey)] + + silvers <- hake.survey[SVSPP == 72, .(silver = sum(BIOMASS, na.rm = T)), + by = key(hake.survey)] + + hake.prop <- merge(hake.prop, silvers, all = T) + hake.prop[is.na(silver), silver := 0] + + hake.prop[, silver.per := silver / hake.all] + + hake.prop[, offshore.per := 1 - silver.per] + hake.prop[, c('hake.all', 'silver') := NULL] + + #disaggregate silver and offshore hake from whiting - use survey in half years + #Generate season variable in comland + comland.hakes <- comland$comland[NESPP3 == 507, ] + comland.hakes[MONTH %in% 1:6, SEASON := 'SPRING'] + comland.hakes[MONTH %in% 7:12, SEASON := 'FALL'] + + comland.hakes <- merge(comland.hakes, hake.prop, + by = c('YEAR', 'SEASON', 'AREA'), all.x = T) + + #Fix NAs + comland.hakes[is.na(silver.per), silver.per := 0] + comland.hakes[is.na(offshore.per), offshore.per := 0] + + #Disaggregate + comland.hakes[, silver := silver.per * SPPLIVMT] + comland.hakes[, silver.value := round(silver.per * SPPVALUE)] + + comland.hakes[, offshore := offshore.per * SPPLIVMT] + comland.hakes[, offshore.value := round(offshore.per * SPPVALUE)] + + comland.hakes[, other.hakes := SPPLIVMT - (silver + offshore)] + comland.hakes[, other.hakes.value := SPPVALUE - (silver.value + offshore.value)] + + #Silver (509), offshore (508), whiting (507) + #put hakes in comland format to merge back + silver <- comland.hakes[, list(YEAR, AREA, MONTH, NEGEAR, + TONCL1, NESPP3, UTILCD, MESHCAT, MKTCAT, + silver, silver.value)] + silver[, NESPP3 := 509] + data.table::setnames(silver, c('silver', 'silver.value'), c('SPPLIVMT', 'SPPVALUE')) + silver <- silver[SPPLIVMT > 0, ] + + offshore <- comland.hakes[, list(YEAR, AREA, MONTH, NEGEAR, + TONCL1, NESPP3, UTILCD, MESHCAT, MKTCAT, + offshore, offshore.value)] + offshore[, NESPP3 := 508] + data.table::setnames(offshore, c('offshore', 'offshore.value'), c('SPPLIVMT', 'SPPVALUE')) + offshore <- offshore[SPPLIVMT > 0, ] + + other <- comland.hakes[, list(YEAR, AREA, MONTH, NEGEAR, + TONCL1, NESPP3, UTILCD, MESHCAT, MKTCAT, + other.hakes, other.hakes.value)] + other[, NESPP3 := 507] + data.table::setnames(other, c('other.hakes', 'other.hakes.value'), c('SPPLIVMT', 'SPPVALUE')) + other <- other[SPPLIVMT > 0, ] + + #merge all three and reformat for comland + hakes.add.back <- data.table::rbindlist(list(silver, offshore, other)) + + data.table::setcolorder(hakes.add.back, names(comland$comland)) + + comland$comland <- data.table::rbindlist(list(comland$comland[NESPP3 != 507, ], + hakes.add.back)) + + return(comland) + +} diff --git a/R/get_comdisc_raw_data.R b/R/get_comdisc_raw_data.R new file mode 100644 index 0000000..df81af0 --- /dev/null +++ b/R/get_comdisc_raw_data.R @@ -0,0 +1,168 @@ +#' Extracts observer data from Database +#' +#' Connects to obdbs and pulls fields from OBSPP, OBINC, ASMSPP, and ASMINC +#' +#'@inheritParams get_comlands_raw_data +#' +#'@return Data frame (data.table) (n x 10) +#'Each row of the data.table represents a species record for a given tow/trip +#' +#'\item{YEAR}{Year of trip/tow} +#'\item{MONTH}{Month of trip/tow} +#'\item{NEGEAR}{Fishing gear used on trip/tow} +#'\item{TONCL1}{Tonnage class of the fishing vessel} +#'\item{NESPP3}{Species code (3 charachters)} +#'\item{NESPP4}{Species code and market code (4 characters)} +#'\item{AREA}{Statistical area in which species was reportly caught} +#'\item{UTILCD}{Utilization code} +#'\item{SPPLIVLB}{live weight (landed = "n") or landed weight (landed="y") in lbs} +#'\item{SPPVALUE}{The value of landed catch to the nearest dollar (U.S.), paid to fisherman by dealer, for a given species.} +#' +#'@section File Creation: +#' +#'A file containing the data.table above will also be saved to the users machine in the directory provided +#' +#'@export + +get_comdisc_raw_data <- function(channel, filterByYear){ + + message("Pulling observer data from database. This could take a while (> 1 hour) ... ") + + #output objects + comdisc <- c() + sql <- c() + + #Create year vector + if(is.na(filterByYear[1])){ + years <- ">= 1989" + }else{ + years <- paste0("in (", comlandr:::sqltext(filterByYear), ")") + } + + ob.qry <- paste0("select year, month, area, negear, nespp4, hailwt, catdisp, drflag, + tripid, haulnum, lathbeg, lonhbeg, link3 + from OBSPP + where obsrflag = 1 + and program not in ('127', '900', '250', '160') + and year ", years, + "\n union + select year, month, area, negear, nespp4, hailwt, catdisp, drflag, + tripid, haulnum, lathbeg, lonhbeg, link3 + from ASMSPP + where obsrflag = 1 + and program not in ('127', '900', '250', '160') + and year ", years) + + ob <- data.table::as.data.table(DBI::dbGetQuery(channel, ob.qry)) + sql <- c(sql, ob.qry) + + #Add protected species here + mammal.qry <- paste0("select distinct a.year, a.month, b.area, b.negear, a.nespp4, + 1 as hailwt, 0 as catdisp, 1 as drflag, a.tripid, a.haulnum, + b.lathbeg, b.lonhbeg, a.link3 + from obinc a, obspp b + where a.tripid = b.tripid + and a.year ", years, + "\n union + select distinct a.year, a.month, b.area, b.negear, a.nespp4, + 1 as hailwt, 0 as catdisp, 1 as drflag, a.tripid, a.haulnum, + b.lathbeg, b.lonhbeg, a.link3 + from asminc a, asmspp b + where a.tripid = b.tripid + and a.year ", years) + + mammal <- data.table::as.data.table(DBI::dbGetQuery(channel, mammal.qry)) + sql <- c(sql, mammal.qry) + + ob <- data.table::rbindlist(list(ob, mammal)) + + #Grab otter trawl gear tables to get mesh size for small verses large mesh + mesh.qry <- paste0("select link3, codmsize + from OBOTGH + where year ", years) + mesh <- data.table::as.data.table(DBI::dbGetQuery(channel, mesh.qry)) + sql <- c(sql, mesh.qry) + + #Convert mesh size from mm to inches + mesh[, CODMSIZE := CODMSIZE * 0.0393701] + mesh[CODMSIZE <= 3, MESHCAT := 'SM'] + mesh[CODMSIZE > 3, MESHCAT := 'LG'] + mesh[, CODMSIZE := NULL] + + ob <- merge(ob, mesh, by = 'LINK3', all.x = T) + + #Clean up data set + #Remove those with unknown disposition + ob <- ob[CATDISP != 9, ] + + #remove record if weight is missing + ob <- ob[!is.na(HAILWT), ] + + #remove non-living items (clappers and stomach contents) and unknown living matter + ob <- ob[!(NESPP4 %in% c(0, 6800:6802, 6805, 6810, 6820, 6830, 6850:6857, 6882, + 6883, 6894:6897))] + + #Convert lat/lon to decimal degrees + ob[, LAT := as.numeric(substr(LATHBEG, 1, 2)) + ((as.numeric(substr(LATHBEG, 3, 4)) + + as.numeric(substr(LATHBEG, 5, 6))) + /60)] + ob[, LON := (as.numeric(substr(LONHBEG, 1, 2)) + ((as.numeric(substr(LONHBEG, 3, 4)) + + as.numeric(substr(LONHBEG, 5, 6))) + /60)) * -1] + ob[, c('LATHBEG', 'LONHBEG') := NULL] + + #Convert weights + convert.qry <- "select nespp4_obs, catdisp_code, drflag_code, cf_lndlb_livlb, cf_rptqty_lndlb + from obspecconv" + convert <- data.table::as.data.table(DBI::dbGetQuery(channel, convert.qry)) + sql <- c(sql, convert.qry) + + setnames(convert, + c('NESPP4_OBS', 'CATDISP_CODE', 'DRFLAG_CODE'), + c('NESPP4', 'CATDISP', 'DRFLAG')) + + setkey(convert, + NESPP4, + CATDISP, + DRFLAG) + + ob.code <- merge(ob, convert, by = key(convert), all.x = T) + + #missing cf's will be set to 1 Assume living + ob.code[is.na(CF_LNDLB_LIVLB), CF_LNDLB_LIVLB := 1] + ob.code[is.na(CF_RPTQTY_LNDLB), CF_RPTQTY_LNDLB := 1] + + ob.code[, C.HAILWT := HAILWT * CF_RPTQTY_LNDLB * CF_LNDLB_LIVLB] + + #Grab PR flags + prflag.qry <- "select NESPP4, cetacean, turtle, pinniped + from obspec" + + prflag <- data.table::as.data.table(DBI::dbGetQuery(channel, prflag.qry)) + sql <- c(sql, prflag.qry) + + prflag[CETACEAN == 1 | TURTLE == 1 | PINNIPED == 1, PR := 1] + prflag[is.na(PR), PR := 0] + prflag[, c('CETACEAN', 'TURTLE', 'PINNIPED') := NULL] + + comdisc <- merge(ob.code, prflag, by = 'NESPP4', all.x = T) + + #Convert to metric tons to align with commercial landings data + comdisc[PR == 0, SPPLIVMT := C.HAILWT * 0.00045359237] + + #Change to NESPP3 to combine market categories + comdisc[PR == 0, NESPP3 := substring(NESPP4, 1, 3)] + #Birds, mammals, etc don't have unique NESPP3 codes + comdisc[is.na(NESPP3), NESPP3 := NESPP4] + + comdisc[PR == 0, MKTCAT := as.numeric(substring(NESPP4, 4, 4))] + comdisc[is.na(MKTCAT), MKTCAT := 0] + + #drop extra columns NESPP4 + comdisc[, c('DRFLAG', 'CF_LNDLB_LIVLB', 'CF_RPTQTY_LNDLB', 'HAILWT', 'C.HAILWT', + 'NESPP4') := NULL] + + return(list(comdisc = comdisc[], + sql = sql)) +} + diff --git a/R/get_comland_data.R b/R/get_comland_data.R index a59dae7..c1cfa58 100644 --- a/R/get_comland_data.R +++ b/R/get_comland_data.R @@ -27,75 +27,48 @@ #' #'A file containing the data.table above will also be saved to the users machine in the directory provided #' +#' #'@export -get_comland_data <- function(channel,landed,endyear,out.dir) { - - - message("Pulling landings data from database. This could take a while (> 1 hour) ... ") - #Landings - tables <- c(paste0('WOLANDS', 64:81), - paste0('WODETS', 82:93), - paste0('CFDETS', 1994:endyear, 'AA')) - - #Generate one table - comland <- c() - for(i in 1:length(tables)){ - landings.qry <- paste("select year, month, negear, toncl1, nespp3, nespp4, area, - spplivlb, spplndlb, sppvalue, utilcd - from", tables[i]) - - comland.yr <- as.data.table(DBI::dbGetQuery(channel, landings.qry)) - - data.table::setkey(comland.yr, - YEAR, - MONTH, - NEGEAR, - TONCL1, - NESPP3, - NESPP4, - AREA, - UTILCD) - - message("Pulled data from ",tables[i]," ...") - - # Use landed weight instead of live weight for shellfish - if(landed == 'y') {comland.yr[NESPP3 %in% 743:800, SPPLIVLB := SPPLNDLB]} - - #Sum landings and value - #landings - comland.yr[, V1 := sum(SPPLIVLB), by = key(comland.yr)] - #value - #Fix null values - comland.yr[is.na(SPPVALUE), SPPVALUE := 0] - comland.yr[, V2 := sum(SPPVALUE), by = key(comland.yr)] - - #Remove extra rows/columns - comland.yr <- unique(comland.yr, by = key(comland.yr)) - comland.yr[, c('SPPLIVLB', 'SPPLNDLB', 'SPPVALUE') := NULL] - - #Rename summed columns - data.table::setnames(comland.yr, c('V1', 'V2'), c('SPPLIVLB', 'SPPVALUE')) - - comland <- data.table::rbindlist(list(comland, comland.yr)) - } - - # save in RODBC format - comland$YEAR <- as.integer(comland$YEAR) - comland$MONTH <- as.integer(comland$MONTH) - comland$NEGEAR <- as.integer(comland$NEGEAR) - comland$TONCL1 <- as.integer(comland$TONCL1) - comland$NESPP3 <- as.integer(comland$NESPP3) - comland$NESPP4 <- as.integer(comland$NESPP4) - comland$UTILCD <- as.integer(comland$UTILCD) - comland$AREA <- as.factor(comland$AREA) - - - - # Save file. - if(landed == 'n') saveRDS(comland, file = file.path(out.dir, paste0("comland_raw_US_livewt.RDS"))) - if(landed == 'y') saveRDS(comland, file = file.path(out.dir, paste0("comland_raw_US_meatwt.RDS"))) - +get_comland_data <- function(channel, filterByYear = NA, filterByArea = NA, useLanded = T, + removeParts = T, useHerringMaine = T, useForeign = T, + refYear = NA, refMonth = NA, disagSkatesHakes = T, + aggArea = F, userAreas = comlandr::mskeyAreas, + areaDescription = 'EPU', propDescription = 'MeanProp', + aggGear = F, userGears = comlandr::mykeyGears, + fleetDescription = 'Fleet') { + + call <- dbutils::capture_function_call() + + #Pull raw data + comland <- comlandr::get_comland_raw_data(channel, filterByYear, filterByArea, + useLanded, removeParts) + + #Pull herring data from the state of Maine + if(useHerringMaine) comland <- comlandr::get_herring_data(channel, comland, + filterByYear, filterByArea) + + #Pull foreign landings + if(useForeign) comland <- comlandr::get_foreign_data(channel, comland, filterByYear, + filterByArea) + + #Apply correction for inflation + if(!is.na(refYear)) comland <- comlandr::adjust_inflation(comland, refYear, refMonth) + + #Disaggregate skates and hakes + if(disagSkatesHakes) comland <- comlandr::disaggregate_skates_hakes(comland, channel, + filterByYear, filterByArea) + + #Aggregate areas + if(aggArea) comland <- aggregate_area(comland, userAreas, areaDescription, + propDescription) + + #Aggregate gears + if(aggGear) comland <- aggregate_gear(comland, userGears, fleetDescription) + + comland$call <- call + + message("Some data may be CONFIDENTIAL ... DO NOT disseminate without proper Non-disclosure agreement.") return(comland) } diff --git a/R/get_comland_raw_data.R b/R/get_comland_raw_data.R new file mode 100644 index 0000000..4b3cdd2 --- /dev/null +++ b/R/get_comland_raw_data.R @@ -0,0 +1,154 @@ +#' Extracts commercial data from Database +#' +#' Connects to cfdbs and pulls fields from WOLANDS, WODETS, CFDETS +#' +#'@param channel an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to connect +#' to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}}) +#'@param endyear Numeric Scalar. Final year of query. +#'@param landed Character String. Use landed weight ("y" - meatwt) for scallops and clams instead of live weight ("n" - livewt). +#'@param out.dir path to directory where final output will be saved +#' +#' +#'@return Data frame (data.table) (n x 10) +#'Each row of the data.table represents a species record for a given tow/trip +#' +#'\item{YEAR}{Year of trip/tow} +#'\item{MONTH}{Month of trip/tow} +#'\item{NEGEAR}{Fishing gear used on trip/tow} +#'\item{TONCL1}{Tonnage class of the fishing vessel} +#'\item{NESPP3}{Species code (3 charachters)} +#'\item{NESPP4}{Species code and market code (4 characters)} +#'\item{AREA}{Statistical area in which species was reportly caught} +#'\item{UTILCD}{Utilization code} +#'\item{SPPLIVLB}{live weight (landed = "n") or landed weight (landed="y") in lbs} +#'\item{SPPVALUE}{The value of landed catch to the nearest dollar (U.S.), paid to fisherman by dealer, for a given species.} +#' +#'@section File Creation: +#' +#'A file containing the data.table above will also be saved to the users machine in the directory provided +#' +#'@export + +get_comland_raw_data <- function(channel, filterByYear = NA, filterByArea = NA, + useLanded = T, removeParts = T){ + + #If not specifying a year default to 1964 - 2019 + if(is.na(filterByYear)) filterByYear <- 1964:2019 + + message(paste0("Pulling landings data from ", + filterByYear[1], " to ", filterByYear[length(filterByYear)], + ". This could take a while (> 1 hour) ... ")) + + #Generate vector of tables to loop through + if(any(filterByYear < 1964)) stop("Landings data start in 1964") + + tables <- as.numeric(c(substr(filterByYear[which(filterByYear <= 1993)], 3, 4), + filterByYear[which(filterByYear > 1993)])) + tables[which(tables > 1993)] <- paste0('CFDETS', tables[which(tables > 1993)], 'AA') + tables[which(tables > 63 & tables <= 81)] <- paste0('WOLANDS', tables[which(tables > 63 & tables <= 81)]) + tables[which(tables > 81 & tables <= 93)] <- paste0('WODETS', tables[which(tables > 81 & tables <= 93)]) + + #output objects + comland <- c() + sql <- c() + + for(itab in 1:length(tables)){ + #Data query + #Need to add mesh data post 1981 + if(substr(tables[itab], 1, 3) == 'WOL'){ + landings.qry <- paste("select year, month, negear, toncl1, nespp3, nespp4, area, + spplivlb, spplndlb, sppvalue, utilcd + from", tables[itab]) + if(!is.na(filterByArea[1])){ + landings.qry <- paste0(landings.qry, " where area in (", survdat:::sqltext(filterByArea), ") + order by area") + } + comland.yr <- data.table::as.data.table(DBI::dbGetQuery(channel, landings.qry)) + comland.yr[, MESH := 5] #Identify all as large mesh + } else { + if(filterByYear[itab] > 1993){ + trip.table <- paste0('CFDETT', filterByYear[itab], 'AA') + } + if(filterByYear[itab] > 1981 & filterByYear[itab] <= 1993){ + trip.table <- paste0('WODETT', substr(filterByYear[itab], 3, 4)) + } + landings.qry <- paste("select a.year, a.month, a.negear, a.toncl1, a.nespp3, + a.nespp4, a.area, a.spplivlb, a.spplndlb, a.sppvalue, + a.utilcd, b.mesh + from", tables[itab], "a,", trip.table, "b + where a.link = b.link") + if(!is.na(filterByArea[1])){ + landings.qry <- paste0(landings.qry, " and a.area in (", survdat:::sqltext(filterByArea), ") + order by area") + } + comland.yr <- data.table::as.data.table(DBI::dbGetQuery(channel, landings.qry)) + } + sql <- c(sql, landings.qry) + + #Identify small/large mesh fisheries + comland.yr[MESH <= 3, MESHCAT := 'SM'] + comland.yr[MESH > 3, MESHCAT := 'LG'] + comland.yr[, MESH := NULL] + + # Use landed weight instead of live weight for shellfish + if(useLanded) {comland.yr[NESPP3 %in% 743:800, SPPLIVLB := SPPLNDLB]} + + # Remove fish parts so live weight is not double counted + if(removeParts){ + comland.yr <- comland.yr[!NESPP4 %in% c('0119', '0123', '0125', '0127', + '0812', '0819', '0828', '0829', + '1731', '2351', '2690', '2699', + '3472', paste0(348:359, 8), + '3868', paste0(469:471, 4), + paste0(480:499, 8), '5018', + '5039', '5261', '5265'), ] + } + + #Sum landings and value + data.table::setkey(comland.yr, + YEAR, + MONTH, + NEGEAR, + MESHCAT, + TONCL1, + NESPP3, + AREA, + UTILCD) + #landings + comland.yr[, V1 := sum(SPPLIVLB, na.rm = T), by = key(comland.yr)] + #value + comland.yr[, V2 := sum(SPPVALUE, na.rm = T), by = key(comland.yr)] + + #Create market category + comland.yr[, MKTCAT := substr(NESPP4, 4, 4)] + + #Remove extra rows/columns + comland.yr <- unique(comland.yr, by = key(comland.yr)) + comland.yr[, c('SPPLIVLB', 'SPPLNDLB', 'SPPVALUE', 'NESPP4') := NULL] + + #Rename summed columns + data.table::setnames(comland.yr, c('V1', 'V2'), c('SPPLIVLB', 'SPPVALUE')) + + comland <- data.table::rbindlist(list(comland, comland.yr)) + + message("Pulled data from ",tables[itab]," ...") + + } + + #Convert number fields from chr to num + numberCols <- c('YEAR', 'MONTH', 'NEGEAR', 'TONCL1', 'NESPP3', 'UTILCD', 'AREA', + 'MKTCAT') + comland[, (numberCols):= lapply(.SD, as.numeric), .SDcols = numberCols][] + + #Adjust pounds to metric tons + comland[, SPPLIVMT := SPPLIVLB * 0.00045359237] + comland[, SPPLIVLB := NULL] + + #standardize YEAR field + comland[YEAR < 100, YEAR := YEAR + 1900L] + + + return(list(comland = comland[], + sql = sql)) +} + diff --git a/R/get_discard_data.R b/R/get_discard_data.R new file mode 100644 index 0000000..db52015 --- /dev/null +++ b/R/get_discard_data.R @@ -0,0 +1,447 @@ +# #Comdisc.r +# library(here); library(data.table); library(comlandr) +# +# #channel <- dbutils::connect_to_database(server="nova",uid="slucey") +# +# #get_observer_data <- function(channel, filterByYear, ) +# endyear <- 2017 +# +# strat.var <- c('YEAR', 'QY', 'EPU', 'GEAR') +# haullevel <- F #Toggle whether to save haul by haul data or not +# landings.file <- 'comland_meatwt_deflated.RData' +# +# +# +# #------------------------------------------------------------------------------- +# #User created functions +# #Sums the number of occurances +# count<-function(x){ +# num<-rep(1,length(x)) +# out<-sum(num) +# return(out) +# } +# +# #------------------------------------------------------------------------------- +# +# filterByYear <- 1989 +# +# #Create year vector +# if(is.na(filterByYear[1])){ +# years <- ">= 1989" +# }else{ +# years <- paste0("in (", survdat:::sqltext(filterByYear), ")") +# } +# +# ob.qry <- paste0("select year, month, area, negear, nespp4, hailwt, catdisp, drflag, +# tripid, haulnum, lathbeg, lonhbeg, link3 +# from OBSPP +# where obsrflag = 1 +# and program not in ('127', '900', '250', '160') +# and year ", years, +# "\n union +# select year, month, area, negear, nespp4, hailwt, catdisp, drflag, +# tripid, haulnum, lathbeg, lonhbeg, link3 +# from ASMSPP +# where obsrflag = 1 +# and program not in ('127', '900', '250', '160') +# and year ", years) +# +# ob <- data.table::as.data.table(DBI::dbGetQuery(channel, ob.qry)) +# +# #Add protected species here +# mammal.qry <- paste0("select distinct a.year, a.month, b.area, b.negear, a.nespp4, +# 1 as hailwt, 0 as catdisp, 1 as drflag, a.tripid, a.haulnum, +# b.lathbeg, b.lonhbeg, a.link3 +# from obinc a, obspp b +# where a.tripid = b.tripid +# and a.year ", years, +# "\n union +# select distinct a.year, a.month, b.area, b.negear, a.nespp4, +# 1 as hailwt, 0 as catdisp, 1 as drflag, a.tripid, a.haulnum, +# b.lathbeg, b.lonhbeg, a.link3 +# from asminc a, asmspp b +# where a.tripid = b.tripid +# and a.year ", years) +# +# mammal <- data.table::as.data.table(DBI::dbGetQuery(channel, mammal.qry)) +# +# ob <- rbindlist(list(ob, mammal)) +# +# #Grab otter trawl gear tables to get mesh size for small verses large mesh +# mesh.qry <- paste0("select link3, codmsize +# from OBOTGH +# where year ", years) +# mesh <- data.table::as.data.table(DBI::dbGetQuery(channel, mesh.qry)) +# +# #Convert mesh size from mm to inches +# mesh[, CODMSIZE := CODMSIZE * 0.0393701] +# mesh[CODMSIZE <= 3, MESHCAT := 'SM'] +# mesh[CODMSIZE > 3, MESHCAT := 'LG'] +# mesh[, CODMSIZE := NULL] +# +# ob <- merge(ob, mesh, by = 'LINK3', all.x = T) +# +# #Clean up data set +# #Remove those with unknown disposition +# ob <- ob[CATDISP != 9, ] +# +# #remove record if weight is missing +# ob <- ob[!is.na(HAILWT), ] +# +# #remove non-living items (clappers and stomach contents) and unknown living matter +# ob <- ob[!(NESPP4 %in% c(0, 6800:6802, 6805, 6810, 6820, 6830, 6850:6857, 6882, 6883, 6894:6897))] +# +# #Convert lat/lon to decimal degrees +# ob[, LATDD := substr(LATHBEG, 1, 2) + ((substr(LATHBEG, 3, 4) + substr(LATHBEG, 5, 6)) +# / 60)] +# +# #Convert weights +# convert.qry <- "select nespp4_obs, catdisp_code, drflag_code, cf_lndlb_livlb, cf_rptqty_lndlb +# from obspecconv" +# convert <- data.table::as.data.table(DBI::dbGetQuery(channel, convert.qry)) +# +# setnames(convert, +# c('NESPP4_OBS', 'CATDISP_CODE', 'DRFLAG_CODE'), +# c('NESPP4', 'CATDISP', 'DRFLAG')) +# +# setkey(convert, +# NESPP4, +# CATDISP, +# DRFLAG) +# +# ob.code <- merge(ob, convert, by = key(convert), all.x = T) +# +# #missing cf's will be set to 1 Assume living +# ob.code[is.na(CF_LNDLB_LIVLB), CF_LNDLB_LIVLB := 1] +# ob.code[is.na(CF_RPTQTY_LNDLB), CF_RPTQTY_LNDLB := 1] +# +# ob.code[, C.HAILWT := HAILWT * CF_RPTQTY_LNDLB * CF_LNDLB_LIVLB] +# +# #Grab common name and PR flags +# comname.qry <- "select NESPP4, comname, sciname, cetacean, turtle, pinniped +# from obspec" +# +# comname <- data.table::as.data.table(DBI::dbGetQuery(channel, comname.qry)) +# comname[CETACEAN == 1 | TURTLE == 1 | PINNIPED == 1, PR := 1] +# comname[is.na(PR), PR := 0] +# comname[, c('CETACEAN', 'TURTLE', 'PINNIPED') := NULL] +# +# ob.code <- merge(comname, ob.code, by = 'NESPP4') +# +# #Convert to metric tons to align with commercial landings data +# ob.code[PR == 0, C.HAILWT := C.HAILWT * 0.00045359237] +# +# #Change to NESPP3 to combine market categories +# ob.code[, NESPP3 := substring(NESPP4, 1, 3)] +# #Birds, mammals, etc don't have unique NESPP3 codes +# ob.code[is.na(NESPP3), NESPP3 := NESPP4] +# +# ob.code[, MKTCAT := as.numeric(substring(NESPP4, 4, 4))] +# +# #drop NESPP4 +# ob.code[, NESPP4 := NULL] +# +# #Deal with skate(ns) for little and winter skates +# source(file.path(data.dir.2, 'Comland_skates_hakes.R')) +# +# #get little skates and winter skates from skates(ns) - use survey in half years +# #Generate Half year variable in comland +# ob.skates <- ob.code[NESPP3 == 365, ] +# ob.skates[MONTH %in% 1:6, Half := 1] +# ob.skates[MONTH %in% 7:12, Half := 2] +# +# setkey(skate.hake.us, +# YEAR, +# Half, +# AREA) +# +# ob.skates <- merge(ob.skates, skate.hake.us, by = key(skate.hake.us), all.x = T) +# +# ob.skates[NESPP3 == 365, little := little.per * C.HAILWT] +# ob.skates[is.na(little), little := 0] +# +# ob.skates[NESPP3 == 365, winter := winter.per * C.HAILWT] +# ob.skates[is.na(winter), winter := 0] +# +# ob.skates[NESPP3 == 365, other.skate := C.HAILWT - (little + winter)] +# +# #Little (366), winter (367), skates(ns) (365) +# #put skates in ob.code format to merge back +# little <- ob.skates[, list(COMNAME, SCINAME, PR, CATDISP, DRFLAG, +# YEAR, MONTH, AREA, NEGEAR, HAILWT, +# TRIPID, HAULNUM, LINK1, LINK3, CF_LNDLB_LIVLB, +# CF_RPTQTY_LNDLB, little, NESPP3, MKTCAT)] +# little[, NESPP3 := 366] +# setnames(little, "little", "C.HAILWT") +# little <- little[C.HAILWT > 0, ] +# +# winter <- ob.skates[, list(COMNAME, SCINAME, PR, CATDISP, DRFLAG, +# YEAR, MONTH, AREA, NEGEAR, HAILWT, +# TRIPID, HAULNUM, LINK1, LINK3, CF_LNDLB_LIVLB, +# CF_RPTQTY_LNDLB, winter, NESPP3, MKTCAT)] +# winter[, NESPP3 := 367] +# setnames(winter, "winter", "C.HAILWT") +# winter <- winter[C.HAILWT > 0, ] +# +# other <- ob.skates[, list(COMNAME, SCINAME, PR, CATDISP, DRFLAG, +# YEAR, MONTH, AREA, NEGEAR, HAILWT, +# TRIPID, HAULNUM, LINK1, LINK3, CF_LNDLB_LIVLB, +# CF_RPTQTY_LNDLB, other.skate, NESPP3, MKTCAT)] +# other[, NESPP3 := 365] +# setnames(other, "other.skate", "C.HAILWT") +# other <- other[C.HAILWT > 0, ] +# +# #merge all three and reformat for ob +# skates.add.back <- rbindlist(list(little, winter, other)) +# +# setcolorder(skates.add.back, names(ob.code)) +# +# ob.code <- rbindlist(list(ob.code[NESPP3 != 365, ], skates.add.back)) +# +# #Assign stat areas to EPUs +# gom <- c(500,510,512:515) +# gb <- c(521:526,551,552,561,562) +# mab <- c(537,539,600,612:616,621,622,625,626,631,632) +# ss <- c(463:467,511) +# +# ob.code[AREA %in% gom, EPU := 'GOM'] +# ob.code[AREA %in% gb, EPU := 'GB'] +# ob.code[AREA %in% mab, EPU := 'MAB'] +# ob.code[AREA %in% ss, EPU := 'SS'] +# ob.code[is.na(EPU), EPU := 'OTHER'] +# ob.code[, EPU := factor(EPU, levels = c('GOM', 'GB', 'MAB', 'SS', 'OTHER'))] +# +# #Create quarter year variable +# ob.code[MONTH %in% 1:3, QY := 1] +# ob.code[MONTH %in% 4:6, QY := 2] +# ob.code[MONTH %in% 7:9, QY := 3] +# ob.code[MONTH %in% 10:12, QY := 4] +# +# #Aggregate Gear +# otter <- 50:59 +# dredge.sc <- 131:132 +# pot <- c(189:190, 200:219, 300, 301) +# longline <- c(10, 40) +# seine <- c(70:79, 120:129, 360) +# gillnet <- c(100:119, 500, 510, 520) +# midwater <- c(170, 370) +# dredge.o <- c(281, 282, 380:400) +# +# ob.code[NEGEAR %in% otter, GEAR := 'otter'] +# ob.code[NEGEAR %in% dredge.sc, GEAR := 'dredge.sc'] +# ob.code[NEGEAR %in% pot, GEAR := 'pot'] +# ob.code[NEGEAR %in% longline, GEAR := 'longline'] +# ob.code[NEGEAR %in% seine, GEAR := 'seine'] +# ob.code[NEGEAR %in% gillnet, GEAR := 'gillnet'] +# ob.code[NEGEAR %in% midwater, GEAR := 'midwater'] +# ob.code[NEGEAR %in% dredge.o, GEAR := 'dredge.o'] +# ob.code[is.na(GEAR), GEAR := 'other'] +# ob.code[, GEAR := as.factor(GEAR)] +# +# ob.code[, c('DRFLAG', 'MONTH', 'AREA', 'NEGEAR', +# 'HAILWT', 'CF_LNDLB_LIVLB', 'CF_RPTQTY_LNDLB') := NULL] +# +# setkeyv(ob.code, c(strat.var, 'NESPP3', 'CATDISP')) +# +# if(haullevel == T){#This is broken +# ob.haul <- ob.code +# save(comdisc, file = file.path(out.dir, "Observer_Discards_by_Haul.RData")) +# } +# +# ob.sums <- ob.code[, sum(C.HAILWT), by = key(ob.code)] +# +# #Make a new function +# #Calculate kept and discards +# ob.discard <- ob.sums[CATDISP == 0, ] +# +# setnames(ob.discard, +# "V1", +# "DISCARD") +# +# setkeyv(ob.sums, strat.var) +# +# ob.kept <- ob.sums[CATDISP == 1, sum(V1), by = key(ob.sums)] +# +# setnames(ob.kept, +# "V1", +# "KEPT.ALL") +# +# ob.all <- merge(ob.kept, ob.discard, by = key(ob.sums)) +# +# ob.all[, CATDISP := NULL] +# +# ob.all[, DK := DISCARD / KEPT.ALL] +# ob.all[is.na(DK), DK := 1.0] +# ob.all[, c('KEPT.ALL', 'DISCARD') := NULL] +# +# #Get landings +# load(file.path(data.dir, landings.file)) +# +# setkeyv(comland, strat.var) +# +# tot.land <- comland[, sum(SPPLIVMT), by = key(comland)] +# +# setnames(tot.land, +# "V1", +# "TOT.LAND") +# +# comdisc <- merge(ob.all, tot.land, by = key(comland)) +# +# comdisc[, DISC := DK * TOT.LAND] +# +# #Variance +# #Need to add back individual trip data +# rm(ob) #Free up memory +# setkeyv(comdisc, c(strat.var, 'NESPP3')) +# +# disc.var <- unique(comdisc, by = key(comdisc)) +# +# #Trip kept all +# setkeyv(ob.code, c(strat.var, 'TRIPID')) +# +# trip.kept <- ob.code[CATDISP == 1, sum(C.HAILWT), by = key(ob.code)] +# setnames(trip.kept, "V1", "trip.k") +# +# #Trip discard by species +# setkeyv(ob.code, c(strat.var, 'TRIPID', 'NESPP3')) +# +# trip.disc <- ob.code[CATDISP == 0, sum(C.HAILWT), by = key(ob.code)] +# setnames(trip.disc, "V1", "trip.d") +# +# trip.all <- merge(trip.disc, trip.kept, by = c(strat.var, 'TRIPID'), all = T) +# trip.all[is.na(trip.k), trip.k := 0] +# +# disc.var <- merge(disc.var, trip.all, by = c(strat.var, 'NESPP3')) +# +# #Calculate the number of observed trips +# setkeyv(ob.code, c(strat.var, 'TRIPID')) +# +# trips <- unique(ob.code, by = key(ob.code)) +# +# trip.count <- trips[, count(TRIPID), by = strat.var] +# +# setnames(trip.count, "V1", "n") +# +# disc.var <- merge(disc.var, trip.count, by = strat.var) +# +# #Calculate the total number of trips +# #CFDBS is on sole - need to switch connection +# odbcClose(channel) +# if(Sys.info()['sysname']=="Windows"){ +# channel <- odbcDriverConnect() +# } else { +# channel <- odbcConnect('sole', uid, pwd) +# } +# +# tables <- c(paste('WODETS', 89:93, sep = ''), +# paste('CFDETS', 1994:endyear, 'AA', sep = '')) +# +# comtrip.qry <- "select year, month, area, negear, count(link) as N +# from WODETS89 +# group by year, month, area, negear" +# comtrip <- as.data.table(sqlQuery(channel, comtrip.qry)) +# +# for(i in 2:length(tables)){ +# tripyr.qry <- paste("select year, month, area, negear, count(link) as N +# from", tables[i], +# "group by year, month, area, negear") +# tripyr <- as.data.table(sqlQuery(channel, tripyr.qry)) +# +# comtrip <- rbindlist(list(comtrip, tripyr)) +# } +# +# comtrip[AREA %in% gom, EPU := 'GOM'] +# comtrip[AREA %in% gb, EPU := 'GB'] +# comtrip[AREA %in% mab, EPU := 'MAB'] +# comtrip[AREA %in% ss, EPU := 'SS'] +# comtrip[is.na(EPU), EPU := 'OTHER'] +# comtrip[, EPU := factor(EPU, levels = c('GOM', 'GB', 'MAB', 'SS', 'OTHER'))] +# +# comtrip[YEAR < 100, YEAR := YEAR + 1900] +# +# comtrip[MONTH %in% 1:3, QY := 1] +# comtrip[MONTH %in% 4:6, QY := 2] +# comtrip[MONTH %in% 7:9, QY := 3] +# comtrip[MONTH %in% 10:12, QY := 4] +# +# comtrip[NEGEAR %in% otter, GEAR := 'otter'] +# comtrip[NEGEAR %in% dredge.sc, GEAR := 'dredge.sc'] +# comtrip[NEGEAR %in% pot, GEAR := 'pot'] +# comtrip[NEGEAR %in% longline, GEAR := 'longline'] +# comtrip[NEGEAR %in% seine, GEAR := 'seine'] +# comtrip[NEGEAR %in% gillnet, GEAR := 'gillnet'] +# comtrip[NEGEAR %in% midwater, GEAR := 'midwater'] +# comtrip[NEGEAR %in% dredge.o, GEAR := 'dredge.o'] +# comtrip[is.na(GEAR), GEAR := 'other'] +# comtrip[, GEAR := as.factor(GEAR)] +# +# setkeyv(comtrip, strat.var) +# +# comtrip.count <- comtrip[, sum(N), by = key(comtrip)] +# +# setnames(comtrip.count, "V1", "N") +# +# disc.var <- merge(disc.var, comtrip.count, by = key(comtrip), all.x = T) +# +# #Fix groups that don't line up properly - actual value of N not that important only relative size +# N.avg <- disc.var[, mean(N, na.rm = T)] +# disc.var[is.na(N), N := N.avg] +# +# #Calculate variance +# #Need to expand so zero discards by species are represented +# setkeyv(disc.var, c(strat.var, 'TRIPID')) +# var.trips <- unique(disc.var, by = key(disc.var)) +# #drop species specific data +# var.trips[, c('NESPP3', 'DK', 'DISC', 'trip.d') := NULL] +# +# #Get list of species +# spp <- unique(disc.var[, NESPP3]) +# all.spp.var <- c() +# for(i in 1:length(spp)){ +# spp.trip <- disc.var[NESPP3 == spp[i], ] +# #Get rid of extra data +# spp.trip[, c('TOT.LAND', 'DISC', 'trip.k', 'n', 'N') := NULL] +# +# spp.var <- merge(var.trips, spp.trip, by = c(strat.var, 'TRIPID'), all.x = T) +# +# #Fix NAs +# spp.var[is.na(NESPP3), NESPP3 := spp[i]] +# spp.var[is.na(trip.d), trip.d := 0] +# +# #Merge in DK ratios +# setkeyv(spp.trip, strat.var) +# spp.dk <- unique(spp.trip, by = key(spp.trip)) +# spp.var[, DK := NULL] +# spp.dk[, c('NESPP3', 'TRIPID', 'trip.d') := NULL] +# spp.var <- merge(spp.var, spp.dk, by = strat.var, all.x = T) +# spp.var[is.na(DK), DK := 0] +# +# spp.var[, step.1 := (sum(trip.d^2 + DK^2 * trip.k^2 - 2 * DK * trip.d * trip.k)/(n - 1)), by = strat.var] +# +# setkeyv(spp.var, strat.var) +# spp.var <- unique(spp.var, by = key(spp.var)) +# spp.var[, c('TRIPID', 'trip.d', 'trip.k', 'DK') := NULL] +# +# spp.var[, DISC.VAR := TOT.LAND^2 * ((N - n)/n*N) * (1/(TOT.LAND/n)^2) * step.1] +# spp.var[, c('TOT.LAND', 'n', 'N', 'step.1') := NULL] +# +# all.spp.var <- rbindlist(list(all.spp.var, spp.var)) +# } +# comdisc <- merge(comdisc, all.spp.var, by = c(strat.var, 'NESPP3'), all.x = T) +# +# #Add species names +# #Change to NESPP3 to combine market categories +# comname[NESPP4 < 100, NESPP3 := as.numeric(substring(NESPP4, 1, 1))] +# comname[NESPP4 > 99 & NESPP4 < 1000, NESPP3 := as.numeric(substring(NESPP4, 1, 2))] +# comname[(NESPP4 > 999 & NESPP4 < 6100) | +# NESPP4 %in% c(7100:7109, 8020:8029), NESPP3 := as.numeric(substring(NESPP4, 1, 3))] +# #Birds, mammals, etc don't have unique NESPP3 codes +# comname[NESPP4 > 6099 & !NESPP4 %in% c(7100:7109, 8020:8029), NESPP3 := NESPP4] +# +# setkey(comname, NESPP3) +# comname <- unique(comname, by = key(comname)) +# comname[, c('NESPP4', 'SCINAME') := NULL] +# +# comdisc <- merge(comname, comdisc, by = 'NESPP3') +# +# save(comdisc, file = file.path(out.dir, "Comdisc.RData")) diff --git a/R/get_herring_data.R b/R/get_herring_data.R new file mode 100644 index 0000000..5f655a5 --- /dev/null +++ b/R/get_herring_data.R @@ -0,0 +1,126 @@ +#' Processes herring data +#' +#'Herring Data comes from the state of Maine. +#' +#'@param channel DBI object. connection object for database access +#'@param comland Data frame. master data frame containing species landings +#' +#'@return Processed Herring data added to comland +#' +#'@importFrom data.table ":=" "key" +#' +#' @noRd +#' @export + +get_herring_data <- function(channel, comland, filterByYear, filterByArea) { + + #Pulling data + message("Pulling Atlantic herring data from maine_herring_catch ...") + + if(is.na(filterByYear[1])){ + years <- ">= 1963" + }else{ + years <- paste0("in (", survdat:::sqltext(filterByYear), ")") + } + + herr.qry <- paste0("select year, month, stock_area, negear, gearname, keptmt, discmt + from maine_herring_catch + where year ", years) + if(!is.na(filterByArea[1])){ + herr.qry <- paste0(herr.qry, " and stock_area in (", survdat:::sqltext(filterByArea), ") + order by stock_area") + } + + sql <- c(comland$sql, herr.qry) + + #pull out comland data + comland <- comland$comland + + herr.catch <- data.table::as.data.table(DBI::dbGetQuery(channel, herr.qry)) + + #Convert number fields from chr to num + numberCols <- c('YEAR', 'MONTH', 'STOCK_AREA', 'NEGEAR', 'GEARNAME') + herr.catch[, (numberCols):= lapply(.SD, as.numeric), .SDcols = numberCols] + + #Aggregate data + data.table::setkey(herr.catch, YEAR, MONTH, STOCK_AREA, NEGEAR) + + herring <- herr.catch[, list(sum(KEPTMT, na.rm = T), sum(DISCMT, na.rm = T)), + by = key(herr.catch)] + + data.table::setnames(herring, c('STOCK_AREA', 'V1', 'V2'), + c('AREA', 'SPPLIVMT', 'DISCMT')) + + #Using averages from comland to fill in categories + herring[, MKTCAT := 5] + + herring[, TONCL1 := 3] + + herring[, UTILCD := 0] + + herring[, MESHCAT := 'LG'] + + #compute price/utilization from CF tables + herring.comland <- comland[NESPP3 == 168, ] + + #Price from comland + herring.price <- herring.comland[, (sum(SPPVALUE, na.rm = T) / sum(SPPLIVMT, na.rm = T)), + by = c('YEAR', 'MONTH')] + + data.table::setnames(herring.price, 'V1', 'price') + + herring <- merge(herring, herring.price, by = c('YEAR', 'MONTH'), all.x = T) + + #Use 1964 prices for < 1964 + herring[YEAR < 1964, price := mean(herring[YEAR == 1964, price])] + #Calculate SPPVALUE from price + herring[, SPPVALUE := round(price * SPPLIVMT)] + + #Utilization from comland + herring.util <- herring.comland[, sum(SPPLIVMT), by = c('YEAR', 'MONTH', 'UTILCD')] + data.table::setnames(herring.util, 'V1', 'SPPLIVMT') + + herring.util[, SPPLIVMT.ALL := sum(SPPLIVMT), by = c('YEAR', 'MONTH')] + + herring.util[, Prop := SPPLIVMT/SPPLIVMT.ALL] + + data.table::setorder(herring.util, YEAR, MONTH, Prop) + + herring.util[, cum.prop := cumsum(Prop), by = c('YEAR', 'MONTH')] + + #Apply proportions to Maine data set + #Not pulled all the time - current through 2017 + herring[, Total := sum(SPPLIVMT), by = c('YEAR', 'MONTH')] + + herring[, Prop := SPPLIVMT / Total] + + data.table::setorder(herring, YEAR, MONTH, Prop) + herring[, cum.prop := cumsum(Prop), by = c('YEAR', 'MONTH')] + + for(iyear in unique(herring.util[, YEAR])){ + for(imonth in unique(herring.util[YEAR == iyear, MONTH])){ + cum.prop.low <- 0 + for(iutil in herring.util[YEAR == iyear & MONTH == imonth, UTILCD]){ + cum.prop.high <- herring.util[YEAR == iyear & MONTH == imonth & + UTILCD == iutil, cum.prop] + herring[YEAR == iyear & MONTH == imonth & cum.prop <= cum.prop.high & + cum.prop > cum.prop.low, UTILCD := iutil] + cum.prop.low <- cum.prop.high + } + } + } + + #fix column headings + herring[, c('Total', 'Prop', 'cum.prop', 'price', 'DISCMT') := NULL] + herring[, NESPP3 := 168] + + data.table::setcolorder(herring, names(comland)) + + #remove herring from data pull and add in Maine numbers + comland <- data.table::rbindlist(list(comland[NESPP3 != 168, ], herring)) + + + return(list(comland = comland[], + sql = sql)) + +} diff --git a/R/sqltext.R b/R/sqltext.R new file mode 100644 index 0000000..4f143bf --- /dev/null +++ b/R/sqltext.R @@ -0,0 +1,18 @@ +#' Convert output to text for DB query +#' +#' @param x +#' +#' Not exported +#' @noRd + +sqltext <- function(x){ + out <- x[1] + if(length(x) > 1){ + for(i in 2:length(x)){ + out <- paste(out, x[i], sep = "','") + } + } + out <- paste("'", out, "'", sep = '') + return(out) +} + diff --git a/data-raw/All_Species_Proportions.rds b/data-raw/All_Species_Proportions.rds new file mode 100644 index 0000000..94e7719 Binary files /dev/null and b/data-raw/All_Species_Proportions.rds differ diff --git a/data-raw/aggregate_area_table.R b/data-raw/aggregate_area_table.R new file mode 100644 index 0000000..4819127 --- /dev/null +++ b/data-raw/aggregate_area_table.R @@ -0,0 +1,17 @@ +#Aggregate areas table + +#The proportion of catch inside and outside was calculated from the MS Keyrun +#project for Georges Bank. +library(here); library(data.table) + +mskeyAreas <- readRDS(here::here('data-raw/All_Species_Proportions.rds')) +mskeyAreas[InOut == 'in', EPU := 'GB'] +mskeyAreas[AREA %in% c(521, 522, 551, 561) & InOut == 'out', EPU := 'GOM'] +mskeyAreas[AREA %in% c(526, 537, 538) & InOut == 'out', EPU := 'MAB'] +mskeyAreas[is.na(EPU), EPU := 'Other'] + +#Drop InOut column +mskeyAreas[, InOut := NULL] + +#Output to package +usethis::use_data(mskeyAreas, overwrite = TRUE) \ No newline at end of file diff --git a/data-raw/aggregate_gears_table.R b/data-raw/aggregate_gears_table.R new file mode 100644 index 0000000..cb9ea21 --- /dev/null +++ b/data-raw/aggregate_gears_table.R @@ -0,0 +1,29 @@ +#Aggregate gears table + +#Gear designations from the MS Keyrun project for Georges Bank. + +library(data.table) + +#Create Gear table +mskeyGears <- data.table(NEGEAR2 = c(5, + 5, 16, 32, 35, 36, + 1, 2, 8, 10, 50, 52, 14, 26, + 12, 17, 37, + 18, 15, 19, 20, 21, 23, 30, 33, 53, + 13, + 40, + 22, 25, 38, 41, + 3, 4, 6, 11), + MESHCAT = c('SM', rep('LG', 5), rep(NA, 30)), + Fleet = c('SM Mesh', + rep('LG Mesh', 5), + rep('Fixed Gear', 8), + rep('Pelagic', 3), + rep('Trap', 9), + 'Scallop Dredge', + 'Clam Dredge', + rep('Other Dredge', 4), + rep('HMS', 4))) + +#Output to package +usethis::use_data(mskeyGears, overwrite = TRUE) \ No newline at end of file diff --git a/data/mskeyAreas.rda b/data/mskeyAreas.rda new file mode 100644 index 0000000..eb318df Binary files /dev/null and b/data/mskeyAreas.rda differ diff --git a/data/mskeyGears.rda b/data/mskeyGears.rda new file mode 100644 index 0000000..a5fcb2e Binary files /dev/null and b/data/mskeyGears.rda differ diff --git a/man/aggregate_area.Rd b/man/aggregate_area.Rd new file mode 100644 index 0000000..22ff63a --- /dev/null +++ b/man/aggregate_area.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/aggregate_area.R +\name{aggregate_area} +\alias{aggregate_area} +\title{Assign landing records to an aggregated area} +\usage{ +aggregate_area( + comland, + userAreas, + areaDescription, + propDescription, + applyPropValue = T +) +} +\arguments{ +\item{comland}{Data set generated by \code{get_comland_data}} + +\item{userAreas}{Data frame. Definitions to aggregate statistical areas to user defined +areas} + +\item{areaDescription}{Character. Name of column in userAreas that defines the new +area.} + +\item{propDescription}{Character. Name of column in userAreas that defines the +proportions of landings assigned to new area.} +} +\description{ +Takes the output from \code{get_comland_data} and further aggregates from NAFO +statistical areas to a user defined area. Allows for species to be assigned by +proportions to more than two user defined areas from one stat area +} diff --git a/man/aggregate_gear.Rd b/man/aggregate_gear.Rd new file mode 100644 index 0000000..a36efee --- /dev/null +++ b/man/aggregate_gear.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/aggregate_gear.R +\name{aggregate_gear} +\alias{aggregate_gear} +\title{Assign landing records to an aggregated area} +\usage{ +aggregate_gear(comData, userGears, fleetDescription) +} +\arguments{ +\item{comland}{Data set generated by \code{get_comland_data}} + +\item{userAreas}{Data frame. Definitions to aggregate statistical areas to user defined +areas} + +\item{areaDescription}{Character. Name of column in userAreas that defines the new +area.} + +\item{propDescription}{Character. Name of column in userAreas that defines the +proportions of landings assigned to new area.} +} +\description{ +Takes the output from \code{get_comland_data} and further aggregates from NAFO +statistical areas to a user defined area. Allows for species to be assigned by +proportions to more than two user defined areas from one stat area +} diff --git a/man/assign_area.Rd b/man/assign_area.Rd new file mode 100644 index 0000000..bbb4302 --- /dev/null +++ b/man/assign_area.Rd @@ -0,0 +1,27 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/assign_area.R +\name{assign_area} +\alias{assign_area} +\title{Assigns points to polygon} +\usage{ +assign_area(comdiscData, areaPolygon, areaDescription, na.keep = F) +} +\arguments{ +\item{na.keep}{Boolean. Logical value to indicate whether original strata names +should be retained.} +} +\value{ +Returns a \code{comdiscData} data.table with one additional column labeled + with the value of \code{areaDescription} + +\item{areaDescription}{The name of the region (found in \code{areaPolygon}) + that a record in \code{surveyData} is assigned to} +} +\description{ +Assign observer data (points, lat and lon) to designated regions (polygons) from a shape file. +} +\seealso{ +Other comdisc: +\code{\link{calc_DK}()} +} +\concept{comdisc} diff --git a/man/calc_DK.Rd b/man/calc_DK.Rd new file mode 100644 index 0000000..57b78d6 --- /dev/null +++ b/man/calc_DK.Rd @@ -0,0 +1,27 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/calc_DK.R +\name{calc_DK} +\alias{calc_DK} +\title{Calculate discard to kept ratio} +\usage{ +calc_DK(comdiscData, areaDescription, fleetDescription) +} +\arguments{ +\item{na.keep}{Boolean. Logical value to indicate whether original strata names +should be retained.} +} +\value{ +Returns a \code{comdiscData} data.table with one additional column labeled + with the value of \code{areaDescription} + +\item{areaDescription}{The name of the region (found in \code{areaPolygon}) + that a record in \code{surveyData} is assigned to} +} +\description{ +Use observer data to calculate the ratio of discards to kept by species. +} +\seealso{ +Other comdisc: +\code{\link{assign_area}()} +} +\concept{comdisc} diff --git a/man/get_comdisc_raw_data.Rd b/man/get_comdisc_raw_data.Rd new file mode 100644 index 0000000..7c088fa --- /dev/null +++ b/man/get_comdisc_raw_data.Rd @@ -0,0 +1,32 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_comdisc_raw_data.R +\name{get_comdisc_raw_data} +\alias{get_comdisc_raw_data} +\title{Extracts observer data from Database} +\usage{ +get_comdisc_raw_data(channel, filterByYear) +} +\value{ +Data frame (data.table) (n x 10) +Each row of the data.table represents a species record for a given tow/trip + +\item{YEAR}{Year of trip/tow} +\item{MONTH}{Month of trip/tow} +\item{NEGEAR}{Fishing gear used on trip/tow} +\item{TONCL1}{Tonnage class of the fishing vessel} +\item{NESPP3}{Species code (3 charachters)} +\item{NESPP4}{Species code and market code (4 characters)} +\item{AREA}{Statistical area in which species was reportly caught} +\item{UTILCD}{Utilization code} +\item{SPPLIVLB}{live weight (landed = "n") or landed weight (landed="y") in lbs} +\item{SPPVALUE}{The value of landed catch to the nearest dollar (U.S.), paid to fisherman by dealer, for a given species.} +} +\description{ +Connects to obdbs and pulls fields from OBSPP, OBINC, ASMSPP, and ASMINC +} +\section{File Creation}{ + + +A file containing the data.table above will also be saved to the users machine in the directory provided +} + diff --git a/man/get_comland_data.Rd b/man/get_comland_data.Rd index ceed4f2..bec04e6 100644 --- a/man/get_comland_data.Rd +++ b/man/get_comland_data.Rd @@ -4,16 +4,34 @@ \alias{get_comland_data} \title{Extracts commercial data from Database} \usage{ -get_comland_data(channel, landed, endyear, out.dir) +get_comland_data( + channel, + filterByYear = NA, + filterByArea = NA, + useLanded = T, + removeParts = T, + useHerringMaine = T, + useForeign = T, + refYear = NA, + refMonth = NA, + disagSkatesHakes = T, + aggArea = F, + userAreas = comlandr::mskeyAreas, + areaDescription = "EPU", + propDescription = "MeanProp", + aggGear = F, + userGears = comlandr::mykeyGears, + fleetDescription = "Fleet" +) } \arguments{ \item{channel}{an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to connect to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}})} -\item{landed}{Character String. Use landed weight ("y" - meatwt) for scallops and clams instead of live weight ("n" - livewt).} - \item{endyear}{Numeric Scalar. Final year of query.} +\item{landed}{Character String. Use landed weight ("y" - meatwt) for scallops and clams instead of live weight ("n" - livewt).} + \item{out.dir}{path to directory where final output will be saved} } \value{ diff --git a/man/get_comland_raw_data.Rd b/man/get_comland_raw_data.Rd new file mode 100644 index 0000000..a75ceda --- /dev/null +++ b/man/get_comland_raw_data.Rd @@ -0,0 +1,48 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_comland_raw_data.R +\name{get_comland_raw_data} +\alias{get_comland_raw_data} +\title{Extracts commercial data from Database} +\usage{ +get_comland_raw_data( + channel, + filterByYear = NA, + filterByArea = NA, + useLanded = T, + removeParts = T +) +} +\arguments{ +\item{channel}{an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to connect +to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}})} + +\item{endyear}{Numeric Scalar. Final year of query.} + +\item{landed}{Character String. Use landed weight ("y" - meatwt) for scallops and clams instead of live weight ("n" - livewt).} + +\item{out.dir}{path to directory where final output will be saved} +} +\value{ +Data frame (data.table) (n x 10) +Each row of the data.table represents a species record for a given tow/trip + +\item{YEAR}{Year of trip/tow} +\item{MONTH}{Month of trip/tow} +\item{NEGEAR}{Fishing gear used on trip/tow} +\item{TONCL1}{Tonnage class of the fishing vessel} +\item{NESPP3}{Species code (3 charachters)} +\item{NESPP4}{Species code and market code (4 characters)} +\item{AREA}{Statistical area in which species was reportly caught} +\item{UTILCD}{Utilization code} +\item{SPPLIVLB}{live weight (landed = "n") or landed weight (landed="y") in lbs} +\item{SPPVALUE}{The value of landed catch to the nearest dollar (U.S.), paid to fisherman by dealer, for a given species.} +} +\description{ +Connects to cfdbs and pulls fields from WOLANDS, WODETS, CFDETS +} +\section{File Creation}{ + + +A file containing the data.table above will also be saved to the users machine in the directory provided +} +