diff --git a/R/get_comland_data.R b/R/get_comland_data.R index 7152d2e..b649776 100644 --- a/R/get_comland_data.R +++ b/R/get_comland_data.R @@ -34,11 +34,11 @@ #'@export -get_comland_data <- function(channel, filterByYear = NA, - filterByArea = NA, useLanded = T, removeParts = T, - useHerringMaine = T, useForeign = T, refYear = NA, - refMonth = NA, disagSkatesHakes = T, aggArea = F, - userAreas = comlandr::mskeyAreas, +get_comland_data <- function(channel, filterByYear = NA, + filterByArea = NA, useLanded = T, removeParts = T, + useHerringMaine = T, useForeign = T, refYear = NA, + refMonth = NA, disagSkatesHakes = T, aggArea = F, + userAreas = comlandr::mskeyAreas, areaDescription = 'EPU', propDescription = 'MeanProp', applyPropLand = T, applyPropValue = T, aggGear = F, userGears = comlandr::mskeyGears, @@ -46,19 +46,19 @@ get_comland_data <- function(channel, filterByYear = NA, knStrata = c('NESPP3', 'YEAR', 'HY', 'QY', 'MONTH', 'NEGEAR', 'TONCL1', 'AREA')) { - + call <- dbutils::capture_function_call() #Pull raw data - comland <- comlandr::get_comland_raw_data(channel, - filterByYear, filterByArea, + comland <- comlandr::get_comland_raw_data(channel, + filterByYear, filterByArea, useLanded, removeParts) #Pull herring data from the state of Maine if(useHerringMaine){ comland <- comlandr::get_herring_data(channel, comland, - filterByYear, filterByArea, + filterByYear, filterByArea, useForeign) } @@ -74,35 +74,35 @@ get_comland_data <- function(channel, filterByYear = NA, as.integer() filterByArea <- c(filterByArea, NAFOAreas) } - + #Pull data and process to look like comland data comland.foreign <- comlandr::get_foreign_data(filterByYear, filterByArea) - comland.foreign <- comlandr::process_foreign_data(channel, comland.foreign, + comland.foreign <- comlandr::process_foreign_data(channel, comland.foreign, useHerringMaine) - + #Combine foreign landings - comland$comland <- data.table::rbindlist(list(comland$comland, comland.foreign), + comland$comland <- data.table::rbindlist(list(comland$comland, comland.foreign), use.names = T) - } - - + } + + #Apply correction for inflation if(!is.na(refYear)) comland <- comlandr::adjust_inflation(comland, refYear, refMonth) #Disaggregate skates and hakes - if(disagSkatesHakes) comland <- comlandr::disaggregate_skates_hakes(comland, - channel, + if(disagSkatesHakes) comland <- comlandr::disaggregate_skates_hakes(comland, + channel, filterByYear, filterByArea) #Aggregate areas - if(aggArea) comland <- comlandr::aggregate_area(comland, userAreas, + if(aggArea) comland <- comlandr::aggregate_area(comland, userAreas, areaDescription, propDescription, - useForeign, channel, + useForeign, channel, applyPropLand, applyPropValue) #Aggregate gears if(aggGear) comland <- comlandr::aggregate_gear(comland, userGears, fleetDescription) - + #Impute unknown catch variables if(!is.null(unkVar)) comland <- comlandr::assign_unknown(comland, unkVar, knStrata) diff --git a/R/process_foreign_data.R b/R/process_foreign_data.R index e8c1264..5f31317 100644 --- a/R/process_foreign_data.R +++ b/R/process_foreign_data.R @@ -26,10 +26,10 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){ ## All code copied directly from nafo_comland nafoland <- nafoland[Divcode %in% c(47, 51:56, 61:63) & Code > 3, ] - + # Need to revisit to make more flexible if get_foreign_data sets # aggregateCountry = F - + # nafoland <- nafoland[, Country :=NULL] nafoland <- nafoland[SPPLIVMT != 0,] @@ -43,7 +43,7 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){ # nafoland[is.na(EPU), EPU := 'OTHER'] # nafoland[, Divcode := NULL] - + ##Fix missing Scotian Shelf data from 21B SS.nafo <- data.table::as.data.table(read.csv(system.file("extdata","SS_NAFO_21A.csv",package="comlandr"), skip = 8)) @@ -84,7 +84,7 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){ spp <- data.table::as.data.table(DBI::dbGetQuery(channel, "select NAFOSPP, NESPP3 from CFSPP")) spp$NAFOSPP <- as.integer(spp$NAFOSPP) spp$NESPP3 <- as.integer(spp$NESPP3) - + #Fix missing NAFO codes missing.spp <- data.table::data.table(NAFOSPP = c(110, 141, 189, 480, 484, 487, 488, 489), NESPP3 = c(240, 509, 512, 366, 368, 367, 370, 369)) @@ -142,14 +142,14 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){ nafoland[NAFOGEAR == 52, NEGEAR := 10L] nafoland[NAFOGEAR == 56, NEGEAR := 21L] nafoland[NAFOGEAR == 58, NEGEAR := 10L] - + #Tonnage nafoland[TONCL1 == 7, TONCL1 := 6L] #Drop NAFO codes nafoland[, c('NAFOGEAR', 'NAFOSPP') := NULL] - + #aggregate nafo landings #2 - aggregate by quarter year, half year, major gear, and small/large TC #This isn't necessary in this function any more @@ -171,22 +171,26 @@ process_foreign_data <- function(channel, nafoland, useHerringMaine){ # nafoland[NEGEAR == 99, GEAR := 'unknown'] # nafoland[is.na(GEAR), GEAR := 'other'] # nafoland[, GEAR := as.factor(GEAR)] - # + # # nafoland[TONCL1 %in% 1:3, SIZE := 'small'] # nafoland[TONCL1 > 3, SIZE := 'large'] # nafoland[TONCL1 == 0, SIZE := 'unknown'] # nafoland[, SIZE := as.factor(SIZE)] - nafoland.agg <- nafoland[, .(SPPLIVMT = sum(SPPLIVMT)), - by = c('NESPP3', 'YEAR', 'MONTH', 'NEGEAR', 'TONCL1', + # TONCL1 no longer used. TONCL2 used instead. + # Make 2 digits, add a zero + nafoland[,TONCL2 := as.numeric(paste0(TONCL1,"0"))] + + nafoland.agg <- nafoland[, .(SPPLIVMT = sum(SPPLIVMT)), + by = c('YEAR', 'MONTH', 'NEGEAR', 'TONCL2','NESPP3', 'AREA')] #Create dummy variable for some columns in US landings nafoland.agg[, SPPVALUE := 0] nafoland.agg[, UTILCD := 0] nafoland.agg[, MESHCAT := NA] - nafoland.agg[, MKTCAT := 0] - + nafoland.agg[, MARKET_CODE := 0] + #Add Nationality Flag nafoland.agg[, US := F]