automated url download for GDP and unempl

indicator_processing/CalculateAllIndicators.R

@@ -22,6 +22,7 @@ plot(1)
 
 # fully automated - pull from internet ----------
 
+setwd(directory)
 setwd("indicator_processing/automated_download/")
 dir()
 
@@ -31,19 +32,27 @@ source("DHW.R")                   # degree heating weeks
 source("earthquakes.R")           # earthquakes
 source("kd490.R")                 # turbidity from Kd490   
 source("sst.R")                   # sea surface temperature
-
+source("Unemployment.R")          # unemployment rate
+source("GDP.R")                   # Gross Domestic Product
 
 # pull from data in folder ----------------------
 
-setwd("../non_automated")
+setwd(directory)
+setwd("indicator_processing/non_automated")
 dir()
 
-source("marine_debris.R")
-source("OA.R")
+source("marine_debris.R")         # marine debris
+source("OA.R")                    # ocean acidification
+source("CRMP_compile.R")          # fishery-indepenedent fish density, slope of size spectrum, coral cover indicators
+
+source("Sargassum_inundation.R")  # sargassum indicator
+
+
+
 
-source("marine_debris.R")
-source("marine_debris.R")
-source("marine_debris.R")
+source("SAU_recreational_catch.R")
+source("pollution.R")
+source("RVC.R")
 
 
 ###############################################################

indicator_processing/automated_download/GDP.R

@@ -0,0 +1,84 @@
+
+### Whole GDP data for PR and USVI from Amy Freitag on 3/5/24 (manually downloaded from FRED https://fred.stlouisfed.org/)
+### latest data run through 2021 for USVI and 2022 for PR.
+# edited by M. Karnauskas 05-16-24 for automated download
+
+# USVI data: https://fred.stlouisfed.org/series/MKTGDPVIA646NWDB
+# PR data: https://fred.stlouisfed.org/series/NYGDPMKTPCDPRI
+
+##########################################################
+
+rm(list = ls())
+dev.off()
+
+library(maps)
+library(plotTimeSeries)
+library(rvest)
+
+load("../spec_file.RData")
+
+# data download --------------------------------------------------------
+
+# Puerto Rico
+page <- read_html("https://fred.stlouisfed.org/series/NYGDPMKTPCDPRI")
+url <- page %>%
+  html_nodes("a") %>%       # find all links
+  html_attr("href")
+
+url[grep("data/", url)]
+
+url2 <- paste0("https://fred.stlouisfed.org", url[grep("data/", url)])
+pr_info <- read.csv(url(url2), header = F)
+
+pr <- pr_info[24:nrow(pr_info), ]
+
+prtab <- data.frame(cbind(substr(pr$V1, 1, 4), substr(pr$V1, 11, 50)))
+prtab$X1 <- as.numeric(as.vector(prtab$X1))
+prtab$X2 <- as.numeric(as.vector(prtab$X2))
+
+
+# USVI
+page <- read_html("https://fred.stlouisfed.org/series/MKTGDPVIA646NWDB")
+url <- page %>%
+  html_nodes("a") %>%       # find all links
+  html_attr("href")
+
+url[grep("data/", url)]
+
+url2 <- paste0("https://fred.stlouisfed.org", url[grep("data/", url)])
+vi_info <- read.csv(url(url2), header = F)
+
+vi <- vi_info[24:nrow(vi_info), ]
+
+vitab <- data.frame(cbind(substr(vi$V1, 1, 4), substr(vi$V1, 11, 50)))
+vitab$X1 <- as.numeric(as.vector(vitab$X1))
+vitab$X2 <- as.numeric(as.vector(vitab$X2))
+
+# merge data 
+
+yrs <- min(vitab$X1, prtab$X1) : max(vitab$X1, prtab$X1)
+mat <- data.frame(matrix(data = NA, nrow = length(yrs), ncol = 2))
+rownames(mat) <- yrs 
+
+mat[which(yrs %in% prtab$X1), 1] <- prtab[, 2]
+mat[which(yrs %in% vitab$X1), 2] <- vitab[, 2]
+mat <- mat / 10^9
+mat
+
+# save as indicator object ----------------------
+datdata <- yrs
+inddata <- mat
+labs <- c("Gross Domestic Product" , "Current U.S. dollars (billions)", "Puerto Rico",
+          "Gross Domestic Product" , "Current U.S. dollars (billions)", "USVI")
+indnames <- data.frame(matrix(labs, nrow = 3, byrow = F))
+inddata <- list(labels = indnames, indicators = inddata, datelist = datdata)
+class(inddata) <- "indicatordata"
+
+# plot and save ----------------------------------
+
+ind <- inddata
+plotIndicatorTimeSeries(ind, coltoplot = 1:2, plotrownum = 2, sublabel = TRUE)
+
+save(ind, file = "indicator_objects/GDP.RData")
+
+###############################  END  #############################

indicator_processing/automated_download/Unemployment.R

@@ -0,0 +1,71 @@
+
+### Unemployment data for PR and USVI from Amy Freitag on 3/5/24 (manually downloaded from FRED https://fred.stlouisfed.org/)
+### latest data run through 12/2023.
+# edited by M. Karnauskas 05-16-24 for automated download
+
+# USVI data: https://fred.stlouisfed.org/series/VIRINSUREDUR
+# PR data: https://fred.stlouisfed.org/series/PRUR
+
+
+rm(list = ls())
+dev.off()
+
+library(maps)
+library(plotTimeSeries)
+
+load("../spec_file.RData")
+
+
+# load data -------------------------------------
+
+urlvi <- "https://fred.stlouisfed.org/graph/fredgraph.csv?bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=off&txtcolor=%23444444&ts=12&tts=12&width=958&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=VIRINSUREDUR&scale=left&cosd=1986-02-08&coed=2024-04-27&line_color=%234572a7&link_values=false&line_style=solid&mark_type=none&mw=3&lw=2&ost=-99999&oet=99999&mma=0&fml=a&fq=Weekly%2C%20Ending%20Saturday&fam=avg&fgst=lin&fgsnd=2020-02-01&line_index=1&transformation=lin&vintage_date=2024-05-16&revision_date=2024-05-16&nd=1986-02-08"
+urlpr <- "https://fred.stlouisfed.org/graph/fredgraph.csv?bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=on&txtcolor=%23444444&ts=12&tts=12&width=958&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=PRUR&scale=left&cosd=1976-01-01&coed=2024-03-01&line_color=%234572a7&link_values=false&line_style=solid&mark_type=none&mw=3&lw=2&ost=-99999&oet=99999&mma=0&fml=a&fq=Monthly&fam=avg&fgst=lin&fgsnd=2020-02-01&line_index=1&transformation=lin&vintage_date=2024-05-16&revision_date=2024-05-16&nd=1976-01-01"
+
+vi <- read.csv(url(urlvi))
+pr <- read.csv(url(urlpr))
+
+class(vi$VIRINSUREDUR)
+class(pr$PRUR)
+
+pr$PRUR <- as.numeric(as.vector(pr$PRUR))
+
+vi$year <- substr(vi$DATE, 1, 4)
+pr$year <- substr(pr$DATE, 1, 4)
+vi$mon <- substr(vi$DATE, 6, 7)
+pr$mon <- substr(pr$DATE, 6, 7)
+vi$yrmon <- paste0(vi$year, vi$mon)
+pr$yrmon <- paste0(pr$year, pr$mon)
+
+vitab <- data.frame(tapply(vi$VIRINSUREDUR, vi$yrmon, mean, na.rm = T))
+vitab <- data.frame(vitab)
+vitab$date <- rownames(vitab)
+names(vitab) <- c("vi", "date")
+head(vitab)
+
+length(unique(pr$yrmon)) == nrow(pr)
+
+head(vitab)
+head(pr)
+
+d <- merge(pr, vitab, by.x = "yrmon", by.y = "date")
+d$dat2 <- paste0(d$year, month.abb[as.numeric(d$mon)])
+head(d)
+
+
+# save as indicator object ----------------------
+datdata <- d$dat2
+inddata <- data.frame(cbind(d$PRUR, d$vi))
+labs <- c("Unemployment rate" , "Percent", "Puerto Rico",
+          "Unemployment rate" , "Percent", "USVI")
+indnames <- data.frame(matrix(labs, nrow = 3, byrow = F))
+inddata <- list(labels = indnames, indicators = inddata, datelist = datdata)
+class(inddata) <- "indicatordata"
+
+# plot and save ----------------------------------
+ind <- inddata
+plotIndicatorTimeSeries(ind, plotrownum = 2, coltoplot = 1:2, sublabel = TRUE, dateformat = "%Y%b", trendAnalysis = T)
+
+save(ind, file = "../../indicator_objects/unemployment.RData")
+
+#############################  END  ####################################
+

indicator_processing/non_automated/CRMP_compile.R

@@ -2,24 +2,30 @@
 ## compile TCRMP and PRCRMP into data plots
 ## M. Karnauskas 2/8/2024
 #
-# run PRCRMP_benthic.R, TCRMP_benthic.R, PRCRMP_fish.R and TCRMP_fish.R first
+# run all codes in 'Caribbean-ESR\indicator_processing\non_automated\CRMP_processing' first
+# PRCRMP_benthic.R, TCRMP_benthic.R, PRCRMP_fish.R and TCRMP_fish.R 
+# these output data objects to Caribbean-ESR\indicator_data\PRCRMP and Caribbean-ESR\indicator_data\TCRMP
 # 
 
 rm(list = ls())
+dev.off()
+
+directory <- rprojroot::find_rstudio_root_file()
+
+setwd(directory)
+dir()
 
 library(plotTimeSeries)
 
 # load coral indicator data -------------------------
 
-setwd("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/")
-
-load("PRCRMP/coralspprich_PR.RData")
+load("indicator_data/PRCRMP/coralspprich_PR.RData")
 PRsr <- out1
-load("PRCRMP/percoralcov_PR.RData") 
+load("indicator_data/PRCRMP/percoralcov_PR.RData") 
 PRcc <- out1
-load("TCRMP/coralspprich_USVI.RData")
+load("indicator_data/TCRMP/coralspprich_USVI.RData")
 VIsr <- out1
-load("TCRMP/percoralcov_USVI.RData") 
+load("indicator_data/TCRMP/percoralcov_USVI.RData") 
 VIcc <- out1
 
 # extract years from linear mixed model ----------------------
@@ -63,13 +69,12 @@ indnames <- data.frame(matrix(labs, nrow = 3, byrow = F))
 s <- list(labels = indnames, indicators = inddata, datelist = datdata, ulim = ulidata, llim = llidata)
 class(s) <- "indicatordata"
 
-setwd("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_plots/") 
-
 plotIndicatorTimeSeries(s, coltoplot = 1:4, plotrownum = 2, sublabel = T, sameYscale = F, 
-                        widadj = 1, hgtadj = 1, trendAnalysis = T, type = "allLines", CItype = "band", outtype = "png")
+                        widadj = 1, hgtadj = 1, trendAnalysis = T, type = "allLines", CItype = "band")
+
+ind <- s
 
-inddata <- s
-save(inddata, file = "C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_objects/coral_spprichness_cover.RData")
+save(ind, file = "indicator_objects/coral_spprichness_cover.RData")
 
 
 
@@ -81,17 +86,13 @@ save(inddata, file = "C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_
 
 rm(list = ls())
 
-library(plotTimeSeries)
-
 # load fish indicator data -------------------------
 
-setwd("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/")
-
-load("PRCRMP/fish_density_PR.RData")
+load("indicator_data/PRCRMP/fish_density_PR.RData")
 PR <- fin
-load("TCRMP/fish_density_USVI.RData") 
+load("indicator_data/TCRMP/fish_density_USVI.RData") 
 VI <- findens
-load("TCRMP/slopeSizeSpec_USVI.RData") 
+load("indicator_data/TCRMP/slopeSizeSpec_USVI.RData") 
 sl <- fin
 
 # extract years ----------------------
@@ -127,13 +128,13 @@ indnames <- data.frame(matrix(labs, nrow = 3, byrow = F))
 s <- list(labels = indnames, indicators = inddata, datelist = datdata, ulim = ulidata, llim = llidata)
 class(s) <- "indicatordata"
 
-setwd("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_plots/") 
-
 plotIndicatorTimeSeries(s, coltoplot = 1:3, plotrownum = 3, sublabel = T, sameYscale = F, 
-                        widadj = 1, hgtadj = 1, trendAnalysis = T, type = "allLines", CItype = "band", outtype = "png")
+                        widadj = 1, hgtadj = 1, trendAnalysis = T, type = "allLines", CItype = "band")
+
+ind <- s
+
+save(ind, file = "indicator_objects/fish_density.RData")
 
-inddata <- s
-save(inddata, file = "C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_objects/fish_density.RData")
 
 #########################  END  ############################
 

indicator_processing/non_automated/PRCRMP_benthic.R → indicator_processing/non_automated/CRMP_processing/PRCRMP_benthic.R

@@ -27,6 +27,12 @@ rm(list = ls())
 #if(!require(lme4)){install.packages("lme4")}
 library(lme4)
 
+directory <- rprojroot::find_rstudio_root_file()
+
+setwd(directory)
+setwd("indicator_data/PRCRMP")
+dir()
+
 # download data -------------------------------
 
 url <- "https://www.nodc.noaa.gov/archive/arc0147/0204647/5.5/data/0-data/PRCRMP_Site_Classification_Database_(1-24-2022).csv"
@@ -35,10 +41,8 @@ met <- read.csv(url, stringsAsFactors = F)
 urlb <- "https://www.nodc.noaa.gov/archive/arc0147/0204647/4.4/data/0-data/PRCRMP_Benthic-sessile_data_1999-2021_(updated_12-6-2021).csv"
 ben <- read.csv(urlb, stringsAsFactors = F)
 
-met <- read.csv("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/PRCRMP_Site_Classification_Database_(11-25-2023).csv", 
-                stringsAsFactors = F)
-ben <- read.csv("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/PRCRMP_Benthic-sessile_data_1999-2023_(updated_11-30-2023).csv", 
-                stringsAsFactors = F)
+met <- read.csv("PRCRMP_Site_Classification_Database_(11-25-2023).csv", stringsAsFactors = F)
+ben <- read.csv("PRCRMP_Benthic-sessile_data_1999-2023_(updated_11-30-2023).csv", stringsAsFactors = F)
 
 head(met)
 head(ben)
@@ -221,8 +225,8 @@ lines(yrs, mod - modse, col = 2, lty = 2)
 
 cor(ind, mod)
 
-if (varint == "sprich")  {  save(out1, file = "C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/coralspprich_PR.RData")  }
-if (varint == "percov")  {  save(out1, file = "C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/percoralcov_PR.RData")   }
+if (varint == "sprich")  {  save(out1, file = "coralspprich_PR.RData")  }
+if (varint == "percov")  {  save(out1, file = "percoralcov_PR.RData")   }
 
 }
 

indicator_processing/non_automated/PRCRMP_fish.R → indicator_processing/non_automated/CRMP_processing/PRCRMP_fish.R

@@ -29,6 +29,13 @@
 
 rm(list = ls())
 
+directory <- rprojroot::find_rstudio_root_file()
+
+setwd(directory)
+setwd("indicator_data/PRCRMP")
+dir()
+
+
 # download data -------------------------------
 
 urlf <- "https://www.nodc.noaa.gov/archive/arc0147/0204647/5.5/data/0-data/PRCRMP_Fish-Invert_Abundance_data_1999-2023_(updated_11-30-2023).csv"
@@ -38,14 +45,12 @@ urlfi <- "https://www.nodc.noaa.gov/archive/arc0147/0204647/5.5/data/0-data/PRCR
 siz <- read.csv(urlfi, stringsAsFactors = F)
 
 
-fish <- read.csv("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/PRCRMP_Fish-Invert_Abundance_data_1999-2023_(updated_11-30-2023).csv", 
-            stringsAsFactors = F)
+fish <- read.csv("PRCRMP_Fish-Invert_Abundance_data_1999-2023_(updated_11-30-2023).csv", stringsAsFactors = F)
 
-siz <- read.csv("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/PRCRMP_Fish-Invert_Size-Freq._data_2004-2023_(updated_11-30-2023).csv", 
-            stringsAsFactors = F)
+siz <- read.csv("PRCRMP_Fish-Invert_Size-Freq._data_2004-2023_(updated_11-30-2023).csv", stringsAsFactors = F)
+
+met <- read.csv("PRCRMP_Site_Classification_Database_(11-25-2023).csv", stringsAsFactors = F)
 
-met <- read.csv("C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/PRCRMP_Site_Classification_Database_(11-25-2023).csv", 
-                stringsAsFactors = F)
 sitelis <- met$Site.Name
 sitelis
 
@@ -225,10 +230,15 @@ points(yrs, tapply(fish$dens, fish$YEAR, mean, na.rm = T), col = 2)
 fin <- data.frame(cbind(yrs, ind_norm, indse_norm))
 fin
 
-save(fin, file = "C:/Users/mandy.karnauskas/Desktop/Caribbean-ESR/indicator_data/PRCRMP/fish_density_PR.RData")
+save(fin, file = "fish_density_PR.RData")
 
 ###########################  END  ###############################
 
+
+
+
+
+
 ##############   CODE BELOW IS ABORTED   #######################
 # CODE FUNCTIONS BUT NOT SUFFICIENT DATA TO RUN ANALYSIS
 # size-based indicators -------------------------------------