Merge pull request #52 from NOAA-EDAB/fix_foreign_stockeff

fixes to foreign data
NOAA-EDAB · Nov 27, 2023 · 5c0bb16 · 5c0bb16
2 parents ba74623 + 965e3ea
commit 5c0bb16
Show file tree

Hide file tree

Showing 3 changed files with 110 additions and 52 deletions.
diff --git a/R/get_comland_data.R b/R/get_comland_data.R
@@ -4,22 +4,40 @@
 #'
 #'@param channel an Object inherited from \link[DBI]{DBIConnection-class}. This object is used to connect
 #' to communicate with the database engine. (see \code{\link[dbutils]{connect_to_database}})
-#'@param endyear Numeric Scalar. Final year of query.
-#'@param landed Character String. Use landed weight ("y" - meatwt) for scallops and clams instead of live weight ("n" - livewt).
-#'@param out.dir path to directory where final output will be saved
+#'@param filterByYear numeric vector
+#'@param filterByArea numeric vector
+#'@param useLanded boolean
+#'@param removeParts boolean
+#'@param useHerringMaine boolean
+#'@param useForeign boolean
+#'@param refYear numeric
+#'@param refMonth numeric
+#'@param disaggSkatesHakes boolean
+#'@param aggArea boolean
+#'@param userAreas data frame
+#'@param areaDescription character string
+#'@param propDescription character string
+#'@param applyPropLand boolean
+#'@param applyPropValue boolean
+#'@param aggGear boolean
+#'@param userGears data frame
+#'@param fleetDescription character string
+#'@param unkVar character vector
+#'@param knStrata character vector
 #'
-#'
-#'@return Data frame (data.table) (n x 10)
+#'@return Data frame (data.table) (n x 12)
 #'Each row of the data.table represents a species record for a given tow/trip
 #'
 #'\item{YEAR}{Year of trip/tow}
 #'\item{MONTH}{Month of trip/tow}
 #'\item{NEGEAR}{Fishing gear used on trip/tow}
-#'\item{TONCL1}{Tonnage class of the fishing vessel}
+#'\item{TONCL2}{Tonnage class of the fishing vessel (2 digit value)}
 #'\item{NESPP3}{Species code (3 charachters)}
-#'\item{NESPP4}{Species code and market code (4 characters)}
-#'\item{AREA}{Statistical area in which species was reportly caught}
+#'\item{MARKET_CODE}{market code (2 characters)}
+#'\item{MESHCAT}{Code to describe the mesh size for a trawl vessel}
+#'\item{AREA/EPU}{Statistical area/ Ecological Production Unit in which species was reportly caught}
 #'\item{UTILCD}{Utilization code}
+#'\item{US}{Landing from the USA vessels or foreign vessels}
 #'\item{SPPLIVLB}{live weight (landed = "n") or landed weight (landed="y") in lbs}
 #'\item{SPPVALUE}{The value of landed catch to the nearest dollar (U.S.), paid to fisherman by dealer, for a given species.}
 #'
@@ -34,31 +52,31 @@
 #'@export
 
 
-get_comland_data <- function(channel, filterByYear = NA, 
-                             filterByArea = NA, useLanded = T, removeParts = T, 
-                             useHerringMaine = T, useForeign = T, refYear = NA, 
-                             refMonth = NA, disagSkatesHakes = T, aggArea = F, 
-                             userAreas = comlandr::mskeyAreas, 
+get_comland_data <- function(channel, filterByYear = NA,
+                             filterByArea = NA, useLanded = T, removeParts = T,
+                             useHerringMaine = T, useForeign = T, refYear = NA,
+                             refMonth = NA, disagSkatesHakes = T, aggArea = F,
+                             userAreas = comlandr::mskeyAreas,
                              areaDescription = 'EPU', propDescription = 'MeanProp',
                              applyPropLand = T, applyPropValue = T,
                              aggGear = F, userGears = comlandr::mskeyGears,
-                             fleetDescription = 'Fleet', unkVar = 'AREA',
-                             knStrata = c('NESPP3', 'YEAR', 'HY', 'QY', 'MONTH',
-                                          'NEGEAR', 'TONCL1', 'AREA')) {
+                             fleetDescription = 'Fleet',
+                             unkVar = c('MONTH','NEGEAR','AREA'),
+                             knStrata = c('HY', 'QY','MONTH','NEGEAR', 'TONCL2', 'AREA')) {
+
 
-
 
   call <- dbutils::capture_function_call()
 
   #Pull raw data
-  comland <- comlandr::get_comland_raw_data(channel, 
-                                            filterByYear, filterByArea, 
+  comland <- comlandr::get_comland_raw_data(channel,
+                                            filterByYear, filterByArea,
                                             useLanded, removeParts)
 
   #Pull herring data from the state of Maine
   if(useHerringMaine){
     comland <- comlandr::get_herring_data(channel, comland,
-                                          filterByYear, filterByArea, 
+                                          filterByYear, filterByArea,
                                           useForeign)
   }
 
@@ -74,35 +92,35 @@ get_comland_data <- function(channel, filterByYear = NA,
         as.integer()
       filterByArea <- c(filterByArea, NAFOAreas)
     }
-    
+
     #Pull data and process to look like comland data
     comland.foreign <- comlandr::get_foreign_data(filterByYear, filterByArea)
-    comland.foreign <- comlandr::process_foreign_data(channel, comland.foreign, 
+    comland.foreign <- comlandr::process_foreign_data(channel, comland.foreign,
                                                       useHerringMaine)
-   
+
     #Combine foreign landings
-    comland$comland <- data.table::rbindlist(list(comland$comland, comland.foreign), 
+    comland$comland <- data.table::rbindlist(list(comland$comland, comland.foreign),
                                              use.names = T)
-  } 
-  
-  
+  }
+
+
   #Apply correction for inflation
   if(!is.na(refYear)) comland <- comlandr::adjust_inflation(comland, refYear, refMonth)
 
   #Disaggregate skates and hakes
-  if(disagSkatesHakes) comland <- comlandr::disaggregate_skates_hakes(comland, 
-                                                                      channel, 
+  if(disagSkatesHakes) comland <- comlandr::disaggregate_skates_hakes(comland,
+                                                                      channel,
                                                             filterByYear, filterByArea)
 
   #Aggregate areas
-  if(aggArea) comland <- comlandr::aggregate_area(comland, userAreas, 
+  if(aggArea) comland <- comlandr::aggregate_area(comland, userAreas,
                                                   areaDescription, propDescription,
-                                                  useForeign, channel, 
+                                                  useForeign, channel,
                                                   applyPropLand, applyPropValue)
 
   #Aggregate gears
   if(aggGear) comland <- comlandr::aggregate_gear(comland, userGears, fleetDescription)
-  
+
   #Impute unknown catch variables
   if(!is.null(unkVar)) comland <- comlandr::assign_unknown(comland, unkVar, knStrata)
 

diff --git a/R/process_foreign_data.R b/R/process_foreign_data.R
@@ -26,10 +26,10 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){
 
   ## All code copied directly from nafo_comland
   nafoland <- nafoland[Divcode %in% c(47, 51:56, 61:63) & Code > 3, ]
-  
+
   # Need to revisit to make more flexible if get_foreign_data sets
   # aggregateCountry = F
-  
+
   # nafoland <- nafoland[, Country :=NULL]
 
   nafoland <- nafoland[SPPLIVMT != 0,]
@@ -43,7 +43,7 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){
   # nafoland[is.na(EPU),                EPU := 'OTHER']
 
   # nafoland[, Divcode := NULL]
-  
+
   ##Fix missing Scotian Shelf data from 21B
   SS.nafo <- data.table::as.data.table(read.csv(system.file("extdata","SS_NAFO_21A.csv",package="comlandr"), skip = 8))
 
@@ -84,7 +84,7 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){
   spp <- data.table::as.data.table(DBI::dbGetQuery(channel, "select NAFOSPP, NESPP3 from CFSPP"))
   spp$NAFOSPP <- as.integer(spp$NAFOSPP)
   spp$NESPP3 <- as.integer(spp$NESPP3)
-  
+
   #Fix missing NAFO codes
   missing.spp <- data.table::data.table(NAFOSPP = c(110, 141, 189, 480, 484, 487, 488, 489),
                                         NESPP3  = c(240, 509, 512, 366, 368, 367, 370, 369))
@@ -142,14 +142,14 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){
   nafoland[NAFOGEAR == 52, NEGEAR := 10L]
   nafoland[NAFOGEAR == 56, NEGEAR := 21L]
   nafoland[NAFOGEAR == 58, NEGEAR := 10L]
-  
+
   #Tonnage
   nafoland[TONCL1 == 7, TONCL1 := 6L]
 
   #Drop NAFO codes
   nafoland[, c('NAFOGEAR', 'NAFOSPP') := NULL]
 
-  
+
   #aggregate nafo landings
   #2 - aggregate by quarter year, half year, major gear, and small/large TC
   #This isn't necessary in this function any more
@@ -171,22 +171,26 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){
   # nafoland[NEGEAR == 99,          GEAR := 'unknown']
   # nafoland[is.na(GEAR),           GEAR := 'other']
   # nafoland[, GEAR := as.factor(GEAR)]
-  # 
+  #
   # nafoland[TONCL1 %in% 1:3, SIZE := 'small']
   # nafoland[TONCL1 > 3,      SIZE := 'large']
   # nafoland[TONCL1 == 0,     SIZE := 'unknown']
   # nafoland[, SIZE := as.factor(SIZE)]
 
-  nafoland.agg <- nafoland[, .(SPPLIVMT = sum(SPPLIVMT)), 
-                           by = c('NESPP3', 'YEAR', 'MONTH', 'NEGEAR', 'TONCL1',
+  # TONCL1 no longer used. TONCL2 used instead.
+  # Make 2 digits, add a zero
+  nafoland[,TONCL2 := as.numeric(paste0(TONCL1,"0"))]
+
+  nafoland.agg <- nafoland[, .(SPPLIVMT = sum(SPPLIVMT)),
+                           by = c('YEAR', 'MONTH', 'NEGEAR', 'TONCL2','NESPP3',
                                   'AREA')]
 
   #Create dummy variable for some columns in US landings
   nafoland.agg[, SPPVALUE := 0]
   nafoland.agg[, UTILCD := 0]
   nafoland.agg[, MESHCAT := NA]
-  nafoland.agg[, MKTCAT := 0]
-  
+  nafoland.agg[, MARKET_CODE := NA]
+
   #Add Nationality Flag
   nafoland.agg[, US := F]
 

diff --git a/man/get_comland_data.Rd b/man/get_comland_data.Rd