From 92d1d382967397d670ec93a18e64cd41bcee947f Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Tue, 26 Sep 2023 22:41:09 +0530 Subject: [PATCH 01/28] moved test_that block --- .../test.download.AmerifluxLBL.R | 32 ++++---- .../integrationTests/test.download.ERA5.R | 74 +++++++++---------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R index e46d1d4830f..659f016e5dd 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R @@ -31,23 +31,23 @@ test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, l lon.in = lon.in, sitename = sitename ) - }) - - # checking if the file is downloaded - test_that("Downloaded files are present at the desired location", { - expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv"))) - }) - test_that("Downloaded data files have the right format", { - firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE) - lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE) - - # checking if first line of CSV has the sitename - expect_true(grepl(sitename, firstline[1])) - - # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates - expect_false(grepl("[A-Za-z]", firstline[4])) - expect_false(grepl("[A-Za-z]", lastline[1])) + # checking if the file is downloaded + test_that("Downloaded files are present at the desired location", { + expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv"))) + }) + + test_that("Downloaded data files have the right format", { + firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE) + lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE) + + # checking if first line of CSV has the sitename + expect_true(grepl(sitename, firstline[1])) + + # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates + expect_false(grepl("[A-Za-z]", firstline[4])) + expect_false(grepl("[A-Za-z]", lastline[1])) + }) }) } diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R index 90f37c8e0c0..fbcff615fd6 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R @@ -37,51 +37,51 @@ test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_typ product_types = product_types, reticulate_python = reticulate_python ) - }) - - test_that("All the required files are downloaded and stored at desired location", { - expect_true(file.exists(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc"))) - expect_true(file.exists(paste0(tmpdir, "/era5.2m_temperature.nc"))) - expect_true(file.exists(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc"))) - expect_true(file.exists(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc"))) - expect_true(file.exists(paste0(tmpdir, "/era5.surface_pressure.nc"))) - expect_true(file.exists(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc"))) - expect_true(file.exists(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc"))) - expect_true(file.exists(paste0(tmpdir, "/era5.total_precipitation.nc"))) - }) + + test_that("All the required files are downloaded and stored at desired location", { + expect_true(file.exists(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc"))) + expect_true(file.exists(paste0(tmpdir, "/era5.2m_temperature.nc"))) + expect_true(file.exists(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc"))) + expect_true(file.exists(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc"))) + expect_true(file.exists(paste0(tmpdir, "/era5.surface_pressure.nc"))) + expect_true(file.exists(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc"))) + expect_true(file.exists(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc"))) + expect_true(file.exists(paste0(tmpdir, "/era5.total_precipitation.nc"))) + }) - test_that("All ERA5 data files have the correct variable units", { - nc <- nc_open(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc")) - expect_equal(nc$var$d2m$units, "K") - nc_close(nc) + test_that("All ERA5 data files have the correct variable units", { + nc <- nc_open(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc")) + expect_equal(nc$var$d2m$units, "K") + nc_close(nc) - nc <- nc_open(paste0(tmpdir, "/era5.2m_temperature.nc")) - expect_equal(nc$var$t2m$units, "K") - nc_close(nc) + nc <- nc_open(paste0(tmpdir, "/era5.2m_temperature.nc")) + expect_equal(nc$var$t2m$units, "K") + nc_close(nc) - nc <- nc_open(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc")) - expect_equal(nc$var$u10$units, "m s**-1") - nc_close(nc) + nc <- nc_open(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc")) + expect_equal(nc$var$u10$units, "m s**-1") + nc_close(nc) - nc <- nc_open(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc")) - expect_equal(nc$var$v10$units, "m s**-1") - nc_close(nc) + nc <- nc_open(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc")) + expect_equal(nc$var$v10$units, "m s**-1") + nc_close(nc) - nc <- nc_open(paste0(tmpdir, "/era5.surface_pressure.nc")) - expect_equal(nc$var$sp$units, "Pa") - nc_close(nc) + nc <- nc_open(paste0(tmpdir, "/era5.surface_pressure.nc")) + expect_equal(nc$var$sp$units, "Pa") + nc_close(nc) - nc <- nc_open(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc")) - expect_equal(nc$var$ssrd$units, "J m**-2") - nc_close(nc) + nc <- nc_open(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc")) + expect_equal(nc$var$ssrd$units, "J m**-2") + nc_close(nc) - nc <- nc_open(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc")) - expect_equal(nc$var$strd$units, "J m**-2") - nc_close(nc) + nc <- nc_open(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc")) + expect_equal(nc$var$strd$units, "J m**-2") + nc_close(nc) - nc <- nc_open(paste0(tmpdir, "/era5.total_precipitation.nc")) - expect_equal(nc$var$tp$units, "m") - nc_close(nc) + nc <- nc_open(paste0(tmpdir, "/era5.total_precipitation.nc")) + expect_equal(nc$var$tp$units, "m") + nc_close(nc) + }) }) } From 0f19ee17c04da61248efb28828c9cd81de22fa5f Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Wed, 27 Sep 2023 01:10:58 +0530 Subject: [PATCH 02/28] updated function refs --- .../test.download.AmerifluxLBL.R | 18 +++++++++------ .../integrationTests/test.download.CRUNCEP.R | 22 +++++++++++-------- .../integrationTests/test.download.ERA5.R | 22 +++++++++++-------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R index 659f016e5dd..07412b4de31 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R @@ -1,19 +1,23 @@ library(testthat) +library(mockery) +library(PEcAn.DB) +library(PEcAn.logger) +library(withr) test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, lon.in) { # putting logger to debug mode - PEcAn.logger::logger.setUseConsole(TRUE, FALSE) - on.exit(PEcAn.logger::logger.setUseConsole(TRUE, TRUE), add = TRUE) - PEcAn.logger::logger.setLevel("DEBUG") + logger.setUseConsole(TRUE, FALSE) + on.exit(logger.setUseConsole(TRUE, TRUE), add = TRUE) + logger.setLevel("DEBUG") # mocking functions - mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame()) - mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1)) + stub(convert_input, 'dbfile.input.check', data.frame()) + stub(convert_input, 'db.query', data.frame(id = 1)) - withr::with_dir(tempdir(), { + with_dir(tempdir(), { tmpdir <- getwd() # calling download function - res <- PEcAn.DB::convert_input( + res <- convert_input( input.id = NA, outfolder = tmpdir, formatname = NULL, diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R index 8512b49f34d..832667c101e 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R @@ -1,19 +1,23 @@ library(testthat) library(ncdf4) +library(mockery) +library(PEcAn.DB) +library(PEcAn.logger) +library(withr) test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method, maxErrors, sleep) { # putting logger to debug mode - PEcAn.logger::logger.setUseConsole(TRUE, FALSE) - on.exit(PEcAn.logger::logger.setUseConsole(TRUE, TRUE), add = TRUE) - PEcAn.logger::logger.setLevel("DEBUG") + logger.setUseConsole(TRUE, FALSE) + on.exit(logger.setUseConsole(TRUE, TRUE), add = TRUE) + logger.setLevel("DEBUG") # mocking functions - mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame()) - mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1)) + stub(convert_input, 'dbfile.input.check', data.frame()) + stub(convert_input, 'db.query', data.frame(id = 1)) - withr::with_dir(tempdir(), { + with_dir(tempdir(), { tmpdir <- getwd() - PEcAn.DB::convert_input( + convert_input( input.id = NA, outfolder = tmpdir, formatname = NULL, @@ -44,8 +48,8 @@ test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method, test_that("NetCDF file contains lat and lon variables", { - mask_nc <- ncdf4::nc_open(paste0(tmpdir, "/cruncep_landwater_mask.nc")) - on.exit(ncdf4::nc_close(mask_nc), add = TRUE) + mask_nc <- nc_open(paste0(tmpdir, "/cruncep_landwater_mask.nc")) + on.exit(nc_close(mask_nc), add = TRUE) expect_true("land_water_mask" %in% names(mask_nc$var)) # Check the dimensions of "land_water_mask" variable diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R index fbcff615fd6..30c6e7af027 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R @@ -1,24 +1,28 @@ library(testthat) library(ncdf4) +library(mockery) +library(PEcAn.DB) +library(PEcAn.logger) +library(withr) test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_types, reticulate_python) { # putting logger to debug mode - PEcAn.logger::logger.setUseConsole(TRUE, FALSE) - on.exit(PEcAn.logger::logger.setUseConsole(TRUE, TRUE), add = TRUE) - PEcAn.logger::logger.setLevel("DEBUG") + logger.setUseConsole(TRUE, FALSE) + on.exit(logger.setUseConsole(TRUE, TRUE), add = TRUE) + logger.setLevel("DEBUG") # mocking functions - mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame()) - mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1)) + stub(convert_input, 'dbfile.input.check', data.frame()) + stub(convert_input, 'db.query', data.frame(id = 1)) # additional mocks needed since download.ERA5 does not return data as other download functions - mockery::stub(PEcAn.DB::convert_input, 'length', 2) - mockery::stub(PEcAn.DB::convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE))) + stub(convert_input, 'length', 2) + stub(convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE))) - withr::with_dir(tempdir(), { + with_dir(tempdir(), { tmpdir <- getwd() - PEcAn.DB::convert_input( + convert_input( input.id = NA, outfolder = tmpdir, formatname = NULL, From 0baebb5aba368711d0d545d0f1a0820245a51bf6 Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Wed, 27 Sep 2023 09:46:28 +0530 Subject: [PATCH 03/28] added namespaces --- .../test.download.AmerifluxLBL.R | 15 ++++++--------- .../integrationTests/test.download.CRUNCEP.R | 15 ++++++--------- .../integrationTests/test.download.ERA5.R | 19 ++++++++----------- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R index 07412b4de31..372f5bfb11d 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R @@ -1,20 +1,17 @@ library(testthat) -library(mockery) library(PEcAn.DB) -library(PEcAn.logger) -library(withr) test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, lon.in) { # putting logger to debug mode - logger.setUseConsole(TRUE, FALSE) - on.exit(logger.setUseConsole(TRUE, TRUE), add = TRUE) - logger.setLevel("DEBUG") + PEcAn.logger::logger.setUseConsole(TRUE, FALSE) + on.exit(PEcAn.logger::logger.setUseConsole(TRUE, TRUE), add = TRUE) + PEcAn.logger::logger.setLevel("DEBUG") # mocking functions - stub(convert_input, 'dbfile.input.check', data.frame()) - stub(convert_input, 'db.query', data.frame(id = 1)) + mockery::stub(convert_input, 'dbfile.input.check', data.frame()) + mockery::stub(convert_input, 'db.query', data.frame(id = 1)) - with_dir(tempdir(), { + withr::with_dir(tempdir(), { tmpdir <- getwd() # calling download function res <- convert_input( diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R index 832667c101e..396c7a0aae7 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R @@ -1,21 +1,18 @@ library(testthat) library(ncdf4) -library(mockery) library(PEcAn.DB) -library(PEcAn.logger) -library(withr) test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method, maxErrors, sleep) { # putting logger to debug mode - logger.setUseConsole(TRUE, FALSE) - on.exit(logger.setUseConsole(TRUE, TRUE), add = TRUE) - logger.setLevel("DEBUG") + PEcAn.logger::logger.setUseConsole(TRUE, FALSE) + on.exit(PEcAn.logger::logger.setUseConsole(TRUE, TRUE), add = TRUE) + PEcAn.logger::logger.setLevel("DEBUG") # mocking functions - stub(convert_input, 'dbfile.input.check', data.frame()) - stub(convert_input, 'db.query', data.frame(id = 1)) + mockery::stub(convert_input, 'dbfile.input.check', data.frame()) + mockery::stub(convert_input, 'db.query', data.frame(id = 1)) - with_dir(tempdir(), { + withr::with_dir(tempdir(), { tmpdir <- getwd() convert_input( input.id = NA, diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R index 30c6e7af027..bf84c31dd85 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R @@ -1,26 +1,23 @@ library(testthat) library(ncdf4) -library(mockery) library(PEcAn.DB) -library(PEcAn.logger) -library(withr) test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_types, reticulate_python) { # putting logger to debug mode - logger.setUseConsole(TRUE, FALSE) - on.exit(logger.setUseConsole(TRUE, TRUE), add = TRUE) - logger.setLevel("DEBUG") + PEcAn.logger::logger.setUseConsole(TRUE, FALSE) + on.exit(PEcAn.logger::logger.setUseConsole(TRUE, TRUE), add = TRUE) + PEcAn.logger::logger.setLevel("DEBUG") # mocking functions - stub(convert_input, 'dbfile.input.check', data.frame()) - stub(convert_input, 'db.query', data.frame(id = 1)) + mockery::stub(convert_input, 'dbfile.input.check', data.frame()) + mockery::stub(convert_input, 'db.query', data.frame(id = 1)) # additional mocks needed since download.ERA5 does not return data as other download functions - stub(convert_input, 'length', 2) - stub(convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE))) + mockery::stub(convert_input, 'length', 2) + mockery::stub(convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE))) - with_dir(tempdir(), { + withr::with_dir(tempdir(), { tmpdir <- getwd() convert_input( input.id = NA, From dfb1dae78e82053af92d46560348c227e235ab42 Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Wed, 27 Sep 2023 09:48:25 +0530 Subject: [PATCH 04/28] added ns --- .../inst/integrationTests/test.download.CRUNCEP.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R index 396c7a0aae7..cfad888104b 100644 --- a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R +++ b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R @@ -45,8 +45,8 @@ test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method, test_that("NetCDF file contains lat and lon variables", { - mask_nc <- nc_open(paste0(tmpdir, "/cruncep_landwater_mask.nc")) - on.exit(nc_close(mask_nc), add = TRUE) + mask_nc <- ncdf4::nc_open(paste0(tmpdir, "/cruncep_landwater_mask.nc")) + on.exit(ncdf4::nc_close(mask_nc), add = TRUE) expect_true("land_water_mask" %in% names(mask_nc$var)) # Check the dimensions of "land_water_mask" variable From 1aa068f07676a1a4b7822448ee61f78221584f9c Mon Sep 17 00:00:00 2001 From: meetagrawal09 Date: Mon, 2 Oct 2023 19:27:54 +0530 Subject: [PATCH 05/28] updated edit url --- book_source/_bookdown.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book_source/_bookdown.yml b/book_source/_bookdown.yml index 45407cef0bc..fb22b3b1e6e 100644 --- a/book_source/_bookdown.yml +++ b/book_source/_bookdown.yml @@ -1,2 +1,2 @@ rmd_subdir: true -edit: https://github.com/tonygardella/pecan/edit/release/vtonydoc/book_source/%s +edit: https://github.com/PecanProject/pecan/edit/develop/book_source/%s From ac5caa7eb2762dfcfe223b2eae5a918fcf48efed Mon Sep 17 00:00:00 2001 From: Chris Black Date: Tue, 26 Sep 2023 23:48:38 -0700 Subject: [PATCH 06/28] wip --- scripts/check_with_errors.R | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/check_with_errors.R b/scripts/check_with_errors.R index 4a4e8701658..f90a75532f4 100755 --- a/scripts/check_with_errors.R +++ b/scripts/check_with_errors.R @@ -7,6 +7,8 @@ log_level <- Sys.getenv("LOGLEVEL", unset = NA) die_level <- Sys.getenv("DIELEVEL", unset = NA) redocument <- as.logical(Sys.getenv("REBUILD_DOCS", unset = NA)) runtests <- as.logical(Sys.getenv("RUN_TESTS", unset = TRUE)) +resave <- as.logical(Sys.getenv("RESAVE_CHECKS", unset = FALSE)) +if (resave) die_level <- "never" old_file <- file.path(pkg, "tests", "Rcheck_reference.log") if (file.exists(old_file)) { @@ -71,7 +73,7 @@ if (log_notes && n_notes > 0) { # such that it's not yet practical to break the build on every warning. # Cleaning this up is a long-term goal, but will take time. # Meanwhile, we compare against a cached historic check output to enforce that -# no *new* warnings are added. As historic warnings are removed, we will update +# no *new* warnings are added. As historic warnings are removed, we update # the cached results to ensure they stay gone. # # To compare checks, we take a two-level approach: @@ -83,16 +85,16 @@ if (log_notes && n_notes > 0) { ### # To update reference files after fixing an old warning: # * Run check_with_errors.R to be sure the check is currently passing -# * Delete the file you want to update -# * Uncomment this section -# * run `DIELEVEL=never Rscript scripts/check_with_errors.R path/to/package` -# * recomment this section -# * Commit updated file -# if (!file.exists(old_file)) { -# cat("No reference check file found. Saving current results as the new standard\n") -# cat(chk$stdout, file = old_file) -# quit("no") -# } +# * run `RESAVE_CHECKS=true Rscript scripts/check_with_errors.R path/to/package` +# * Commit updated /tests/Rcheck_reference.log file +if (resave) { + cat("Saving current check results as the new standard\n") + if (file.exists(old_file)) { + cat("**Overwriting** existing saved check output\n") + } + cat(chk$stdout, file = old_file) + quit("no") +} ### # everything beyond this point is comparing to old version From 38293ded78dd4fb176cc6a79eba6f55301f23d11 Mon Sep 17 00:00:00 2001 From: Shashank Singh Date: Mon, 18 Sep 2023 02:33:07 +0000 Subject: [PATCH 07/28] title in description to upper case --- modules/data.mining/DESCRIPTION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data.mining/DESCRIPTION b/modules/data.mining/DESCRIPTION index 74176560466..7ac5389be56 100644 --- a/modules/data.mining/DESCRIPTION +++ b/modules/data.mining/DESCRIPTION @@ -1,6 +1,6 @@ Package: PEcAn.data.mining Type: Package -Title: PEcAn functions used for exploring model residuals and structures +Title: PEcAn Functions Used for Exploring Model Residuals and Structures Description: (Temporary description) PEcAn functions used for exploring model residuals and structures Version: 1.7.2 Date: 2021-10-04 @@ -21,4 +21,4 @@ LazyLoad: yes LazyData: FALSE Collate: Encoding: UTF-8 -RoxygenNote: 6.1.1 +RoxygenNote: 7.1.2 From 9e9a9d30959ee7bed4f4b235b03db246a36e2dfd Mon Sep 17 00:00:00 2001 From: Shashank Singh Date: Sun, 8 Oct 2023 15:16:01 +0000 Subject: [PATCH 08/28] log file update --- .../data.mining/tests/Rcheck_reference.log | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 modules/data.mining/tests/Rcheck_reference.log diff --git a/modules/data.mining/tests/Rcheck_reference.log b/modules/data.mining/tests/Rcheck_reference.log new file mode 100644 index 00000000000..7746a8b418c --- /dev/null +++ b/modules/data.mining/tests/Rcheck_reference.log @@ -0,0 +1,49 @@ +* using log directory ‘/tmp/Rtmpyv5R27/PEcAn.data.mining.Rcheck’ +* using R version 4.1.3 (2022-03-10) +* using platform: x86_64-pc-linux-gnu (64-bit) +* using session charset: UTF-8 +* using options ‘--no-manual --as-cran’ +* checking for file ‘PEcAn.data.mining/DESCRIPTION’ ... OK +* checking extension type ... Package +* this is package ‘PEcAn.data.mining’ version ‘1.7.2’ +* package encoding: UTF-8 +* checking package namespace information ... OK +* checking package dependencies ... OK +* checking if this is a source package ... OK +* checking if there is a namespace ... OK +* checking for executable files ... OK +* checking for hidden files and directories ... OK +* checking for portable file names ... OK +* checking for sufficient/correct file permissions ... OK +* checking serialization versions ... OK +* checking whether package ‘PEcAn.data.mining’ can be installed ... OK +* checking installed package size ... OK +* checking package directory ... OK +* checking for future file timestamps ... OK +* checking DESCRIPTION meta-information ... NOTE +Malformed Description field: should contain one or more complete sentences. +* checking top-level files ... OK +* checking for left-over files ... OK +* checking index information ... OK +* checking package subdirectories ... OK +* checking whether the package can be loaded ... OK +* checking whether the package can be loaded with stated dependencies ... OK +* checking whether the package can be unloaded cleanly ... OK +* checking whether the namespace can be loaded with stated dependencies ... OK +* checking whether the namespace can be unloaded cleanly ... OK +* checking loading without being on the library search path ... OK +* checking examples ... NONE +* checking for unstated dependencies in ‘tests’ ... OK +* checking tests ... + Running ‘testthat.R’ + OK +* checking for non-standard things in the check directory ... OK +* checking for detritus in the temp directory ... OK +* DONE + +Status: 1 NOTE +See + ‘/tmp/Rtmpyv5R27/PEcAn.data.mining.Rcheck/00check.log’ +for details. + + From 11c4d454771fccb0e05b9b1df230261c113ce74f Mon Sep 17 00:00:00 2001 From: Shashank Singh Date: Sun, 15 Oct 2023 13:42:21 +0000 Subject: [PATCH 09/28] mend --- .../data.remote/tests/Rcheck_reference.log | 93 +++---------------- 1 file changed, 13 insertions(+), 80 deletions(-) diff --git a/modules/data.remote/tests/Rcheck_reference.log b/modules/data.remote/tests/Rcheck_reference.log index 822a9411303..afd295d0b58 100644 --- a/modules/data.remote/tests/Rcheck_reference.log +++ b/modules/data.remote/tests/Rcheck_reference.log @@ -1,55 +1,12 @@ -* using log directory ‘/home/tanishq010/pecan/modules/PEcAn.data.remote.Rcheck’ -* using R version 4.2.1 (2022-06-23) +* using log directory ‘/tmp/RtmpqErjWC/PEcAn.data.remote.Rcheck’ +* using R version 4.1.3 (2022-03-10) * using platform: x86_64-pc-linux-gnu (64-bit) * using session charset: UTF-8 -* using options ‘--no-tests --no-manual --as-cran’ +* using options ‘--no-manual --as-cran’ * checking for file ‘PEcAn.data.remote/DESCRIPTION’ ... OK * checking extension type ... Package * this is package ‘PEcAn.data.remote’ version ‘1.7.2.9000’ * package encoding: UTF-8 -* checking CRAN incoming feasibility ... WARNING -Maintainer: ‘Bailey Morrison ’ - -New submission - -Version contains large components (1.7.2.9000) - -License components with restrictions and base license permitting such: - BSD_3_clause + file LICENSE -File 'LICENSE': - University of Illinois/NCSA Open Source License - - Copyright (c) 2012, University of Illinois, NCSA. All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal with the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimers. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - Neither the names of University of Illinois, NCSA, nor the names - of its contributors may be used to endorse or promote products - derived from this Software without specific prior written permission. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR - ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE SOFTWARE. - -Strong dependencies not in mainstream repositories: - PEcAn.DB, PEcAn.utils, PEcAn.logger, PEcAn.remote - -The Date field is over a month old. * checking package namespace information ... OK * checking package dependencies ... OK * checking if this is a source package ... OK @@ -63,11 +20,7 @@ The Date field is over a month old. * checking installed package size ... OK * checking package directory ... OK * checking for future file timestamps ... OK -* checking DESCRIPTION meta-information ... NOTE -Author field differs from that derived from Authors@R - Author: ‘Mike Dietze, Bailey Morrison’ - Authors@R: ‘Mike Dietze [aut], Bailey Morrison [aut, cre], University of Illinois, NCSA [cph]’ - +* checking DESCRIPTION meta-information ... OK * checking top-level files ... OK * checking for left-over files ... OK * checking index information ... OK @@ -80,7 +33,6 @@ Author field differs from that derived from Authors@R * checking whether the namespace can be loaded with stated dependencies ... OK * checking whether the namespace can be unloaded cleanly ... OK * checking loading without being on the library search path ... OK -* checking use of S3 registration ... OK * checking dependencies in R code ... WARNING 'library' or 'require' calls not declared from: ‘PEcAn.DB’ ‘doParallel’ ‘raster’ @@ -88,41 +40,17 @@ Author field differs from that derived from Authors@R ‘PEcAn.DB’ ‘doParallel’ ‘raster’ ‘rgdal’ Please use :: or requireNamespace() instead. See section 'Suggested packages' in the 'Writing R Extensions' manual. -Namespace in Imports field not imported from: ‘stringr’ - All declared Imports should be used. +Missing or unexported objects: + ‘PEcAn.utils::download_file’ ‘PEcAn.utils::need_packages’ * checking S3 generic/method consistency ... OK * checking replacement functions ... OK * checking foreign function calls ... OK * checking R code for possible problems ... NOTE -call_MODIS: no visible global function definition for ‘write.csv’ download.LandTrendr.AGB: no visible binding for global variable ‘k’ -download.NLCD: no visible global function definition for ‘dbfile.check’ -download.NLCD: no visible global function definition for ‘db.query’ -download.NLCD: no visible global function definition for - ‘dbfile.insert’ -download.thredds.AGB : get_data: no visible global function definition - for ‘write.csv’ -download.thredds.AGB: no visible global function definition for - ‘foreach’ -download.thredds.AGB: no visible global function definition for - ‘stopCluster’ -extract_NLCD: no visible global function definition for ‘dbfile.check’ -extract_NLCD: no visible global function definition for ‘db.query’ -extract_NLCD: no visible global function definition for ‘raster’ -extract_NLCD: no visible global function definition for ‘SpatialPoints’ -extract_NLCD: no visible global function definition for ‘CRS’ -extract_NLCD: no visible global function definition for ‘spTransform’ -extract_NLCD: no visible global function definition for ‘crs’ -extract_NLCD: no visible global function definition for ‘extract’ grid2netcdf: no visible binding for global variable ‘years’ grid2netcdf: no visible binding for global variable ‘yieldarray’ Undefined global functions or variables: - CRS SpatialPoints crs db.query dbfile.check dbfile.insert - download.file extract foreach k raster spTransform stopCluster - write.csv -Consider adding - importFrom("utils", "download.file", "write.csv") -to your NAMESPACE file. + k years yieldarray * checking Rd files ... OK * checking Rd metadata ... OK * checking Rd line widths ... OK @@ -148,4 +76,9 @@ Extensions’ manual. * checking for detritus in the temp directory ... OK * DONE -Status: 3 WARNINGs, 2 NOTEs +Status: 2 WARNINGs, 1 NOTE +See + ‘/tmp/RtmpqErjWC/PEcAn.data.remote.Rcheck/00check.log’ +for details. + + From ab57f797185f4dbed15a4cee32ce325d622694df Mon Sep 17 00:00:00 2001 From: Shashank Singh Date: Mon, 18 Sep 2023 03:18:53 +0000 Subject: [PATCH 10/28] fix warnings/notes in data.remote package --- modules/data.remote/R/NLCD.R | 18 +++++++++--------- modules/data.remote/R/call_MODIS.R | 2 +- modules/data.remote/R/download.thredds.R | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/data.remote/R/NLCD.R b/modules/data.remote/R/NLCD.R index dfc712f2e09..f1a8a6b868a 100644 --- a/modules/data.remote/R/NLCD.R +++ b/modules/data.remote/R/NLCD.R @@ -24,9 +24,9 @@ download.NLCD <- function(outdir, year = 2011, con = NULL) { ## before downloading, check if the file already exists on this host if (!is.null(con)) { library(PEcAn.DB) - chk <- dbfile.check(type = "Input", id = input.id, con = con) + chk <- PEcAn.DB::dbfile.check(type = "Input", id = input.id, con = con) if (nrow(chk) > 0) { - machines <- db.query(paste("SELECT * from machines where id in (", + machines <- PEcAn.DB::db.query(paste("SELECT * from machines where id in (", paste(chk$machine_id, sep = ","), ")"), con) if (PEcAn.remote::fqdn() %in% machines$hostname) { ## record already exists on this host @@ -49,7 +49,7 @@ download.NLCD <- function(outdir, year = 2011, con = NULL) { prefix <- table(sapply(strsplit(dir(data_dir), ".", fixed = TRUE), function(x) { x[1] })) prefix <- names(which.max(prefix)) site.id <- 1000000676 - return(dbfile.insert(data_dir, in.prefix = prefix, type = "Input", input.id, con, + return(PEcAn.DB::dbfile.insert(data_dir, in.prefix = prefix, type = "Input", input.id, con, reuse = TRUE)) } return(data_dir) @@ -80,9 +80,9 @@ extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 201 } else { print(paste("Year not yet supported: ", year)) } - chk <- dbfile.check(type = "Input", id = input.id, con = con) + chk <- PEcAn.DB::dbfile.check(type = "Input", id = input.id, con = con) if (nrow(chk) > 0) { - machines <- db.query(paste("SELECT * from machines where id in (", + machines <- PEcAn.DB::db.query(paste("SELECT * from machines where id in (", paste(chk$machine_id, sep = ","), ")"), con) if (PEcAn.remote::fqdn() %in% machines$hostname) { ## record already exists on this host @@ -104,14 +104,14 @@ extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 201 print(paste("File not found:", filename)) return(NULL) } - nlcd <- raster(filename) + nlcd <- raster::raster(filename) # transform points - sites <- SpatialPoints(coords = coords, proj4string = CRS("+proj=longlat +datum=WGS84")) - sites <- spTransform(sites, crs(nlcd)) + sites <- sp::SpatialPoints(coords = coords, proj4string = sp::CRS("+proj=longlat +datum=WGS84")) + sites <- sp::spTransform(sites, raster::crs(nlcd)) # extract - sum.raw <- table(extract(nlcd, sites, buffer = buffer)) + sum.raw <- table(raster::extract(nlcd, sites, buffer = buffer)) summ <- prop.table(sum.raw) mydf <- data.frame(cover = names(summ), percent = as.vector(summ), count = as.vector(sum.raw)) diff --git a/modules/data.remote/R/call_MODIS.R b/modules/data.remote/R/call_MODIS.R index 61e41a6f845..b4791cd004e 100755 --- a/modules/data.remote/R/call_MODIS.R +++ b/modules/data.remote/R/call_MODIS.R @@ -283,7 +283,7 @@ call_MODIS <- function(var, product, fname <- paste(site_info$site_id[i], "/", product, "_", band, "_", start_date, "-", end_date, "_unfiltered.csv", sep = "") } fname <- file.path(outdir, fname) - write.csv(site, fname, row.names = FALSE) + utils::write.csv(site, fname, row.names = FALSE) } } diff --git a/modules/data.remote/R/download.thredds.R b/modules/data.remote/R/download.thredds.R index 09d44ac1337..6e61e92ffa9 100755 --- a/modules/data.remote/R/download.thredds.R +++ b/modules/data.remote/R/download.thredds.R @@ -69,7 +69,7 @@ download.thredds.AGB <- function(outdir = NULL, site_ids, run_parallel = FALSE, # option to save output dataset to directory for user. if (!(is.null(outdir))) { - write.csv(output, file = paste0(outdir, "THREDDS_", sub("^([^.]*).*", "\\1",basename(files[1])), "_site_", site, ".csv"), row.names = FALSE) + utils::write.csv(output, file = paste0(outdir, "THREDDS_", sub("^([^.]*).*", "\\1",basename(files[1])), "_site_", site, ".csv"), row.names = FALSE) } return(output) @@ -86,8 +86,8 @@ download.thredds.AGB <- function(outdir = NULL, site_ids, run_parallel = FALSE, PEcAn.logger::logger.info(paste0("Running in parallel with: ", ncores)) cl = parallel::makeCluster(ncores) doParallel::registerDoParallel(cl) - data = foreach(i = seq_along(mylat), .combine = rbind) %dopar% get_data(i) - stopCluster(cl) + data = foreach::foreach(i = seq_along(mylat), .combine = rbind) %dopar% get_data(i) + parallel::stopCluster(cl) } else { # setup sequential run From 248e78ccc872f5b084053a39f6a5813cb7c698e7 Mon Sep 17 00:00:00 2001 From: Shashank Singh Date: Wed, 25 Oct 2023 05:07:13 +0000 Subject: [PATCH 11/28] add log file --- modules/data.remote/tests/Rcheck_reference.log | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data.remote/tests/Rcheck_reference.log b/modules/data.remote/tests/Rcheck_reference.log index afd295d0b58..e5c565fd010 100644 --- a/modules/data.remote/tests/Rcheck_reference.log +++ b/modules/data.remote/tests/Rcheck_reference.log @@ -1,4 +1,4 @@ -* using log directory ‘/tmp/RtmpqErjWC/PEcAn.data.remote.Rcheck’ +* using log directory ‘/tmp/Rtmpr2UgRH/PEcAn.data.remote.Rcheck’ * using R version 4.1.3 (2022-03-10) * using platform: x86_64-pc-linux-gnu (64-bit) * using session charset: UTF-8 @@ -78,7 +78,7 @@ Extensions’ manual. Status: 2 WARNINGs, 1 NOTE See - ‘/tmp/RtmpqErjWC/PEcAn.data.remote.Rcheck/00check.log’ + ‘/tmp/Rtmpr2UgRH/PEcAn.data.remote.Rcheck/00check.log’ for details. From 202f656a1e8d2a49ef0310f75f2e69322706437b Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 11:34:12 +0200 Subject: [PATCH 12/28] changed rgdal function to terra in GEFS_helpers --- .../data.atmosphere/R/GEFS_helper_functions.R | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/modules/data.atmosphere/R/GEFS_helper_functions.R b/modules/data.atmosphere/R/GEFS_helper_functions.R index 99b0a4db6e6..c63768daf11 100644 --- a/modules/data.atmosphere/R/GEFS_helper_functions.R +++ b/modules/data.atmosphere/R/GEFS_helper_functions.R @@ -243,25 +243,30 @@ process_gridded_noaa_download <- function(lat_list, for(hr in 1:length(curr_hours)){ file_name <- paste0(base_filename2, curr_hours[hr]) + grib_file_name <- paste0(working_directory,"/", file_name,".grib") - if(file.exists(paste0(working_directory,"/", file_name,".grib"))){ - grib <- rgdal::readGDAL(paste0(working_directory,"/", file_name,".grib"), silent = TRUE) - lat_lon <- sp::coordinates(grib) + if(file.exists(grib_file_name)){ + grib_data <- terra::rast(grib_file_name) + + ## Convert to data frame + grib_data_df <- terra::as.data.frame(grib_data, xy=TRUE) + lat_lon <- grib_data_df[, c("x", "y")] + for(s in 1:length(site_id)){ index <- which(lat_lon[,2] == lats[s] & lat_lon[,1] == lons[s]) - pressfc[s, hr] <- grib$band1[index] - tmp2m[s, hr] <- grib$band2[index] - rh2m[s, hr] <- grib$band3[index] - ugrd10m[s, hr] <- grib$band4[index] - vgrd10m[s, hr] <- grib$band5[index] + pressfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; Pressure [Pa]`[index] + tmp2m[s, hr] <- grib_data_df$`2[m] HTGL=Specified height level above ground; Temperature [C]`[index] + rh2m[s, hr] <- grib_data_df$`2[m] HTGL=Specified height level above ground; Relative humidity [%]`[index] + ugrd10m[s, hr] <- grib_data_df$`10[m] HTGL=Specified height level above ground; u-component of wind [m/s]`[index] + vgrd10m[s, hr] <- grib_data_df$`10[m] HTGL=Specified height level above ground; v-component of wind [m/s]`[index] if(curr_hours[hr] != "000"){ - apcpsfc[s, hr] <- grib$band6[index] - tcdcclm[s, hr] <- grib$band7[index] - dswrfsfc[s, hr] <- grib$band8[index] - dlwrfsfc[s, hr] <- grib$band9[index] + apcpsfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; 03 hr Total precipitation [kg/(m^2)]`[index] + tcdcclm[s, hr] <- grib_data_df$`RESERVED(10) (Reserved); Total cloud cover [%]`[index] + dswrfsfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; Downward Short-Wave Rad. Flux [W/(m^2)]`[index] + dlwrfsfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; Downward Long-Wave Rad. Flux [W/(m^2)]`[index] } } } From a5c65a6a177eb9a9d480529f459448e5ba2ae8ef Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 11:59:03 +0200 Subject: [PATCH 13/28] replace rgdal with sf --- modules/data.atmosphere/R/download.NARR_site.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data.atmosphere/R/download.NARR_site.R b/modules/data.atmosphere/R/download.NARR_site.R index 7ecfbca6bae..703546874e2 100644 --- a/modules/data.atmosphere/R/download.NARR_site.R +++ b/modules/data.atmosphere/R/download.NARR_site.R @@ -465,9 +465,9 @@ latlon2narr <- function(nc, lat.in, lon.in) { #' @inheritParams get_NARR_thredds #' @return `sp::SpatialPoints` object containing transformed x and y #' coordinates, in km, which should match NARR coordinates -#' @importFrom rgdal checkCRSArgs +#' @importFrom sf st_crs # ^not used directly here, but needed by sp::CRS. - # sp lists rgdal in Suggests rather than Imports, + # sp lists sf in Suggests rather than Imports, # so importing it here to ensure it's available at run time #' @author Alexey Shiklomanov #' @export From f5756af4e69d80f7c8f215b7644c1ece96386777 Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 12:02:54 +0200 Subject: [PATCH 14/28] roxygenize --- modules/data.atmosphere/DESCRIPTION | 3 ++- modules/data.atmosphere/NAMESPACE | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/data.atmosphere/DESCRIPTION b/modules/data.atmosphere/DESCRIPTION index a29287049c5..9b9d5706e3a 100644 --- a/modules/data.atmosphere/DESCRIPTION +++ b/modules/data.atmosphere/DESCRIPTION @@ -49,10 +49,11 @@ Imports: raster, REddyProc, reshape2, - rgdal, rlang (>= 0.2.0), + sf, sp, stringr (>= 1.1.0), + terra, testthat (>= 2.0.0), tibble, tidyr, diff --git a/modules/data.atmosphere/NAMESPACE b/modules/data.atmosphere/NAMESPACE index 7183782887f..9dd14faa440 100644 --- a/modules/data.atmosphere/NAMESPACE +++ b/modules/data.atmosphere/NAMESPACE @@ -113,5 +113,5 @@ export(write_noaa_gefs_netcdf) import(dplyr) import(tidyselect) importFrom(magrittr,"%>%") -importFrom(rgdal,checkCRSArgs) importFrom(rlang,.data) +importFrom(sf,st_crs) From 9624d3d4946c775f5e5180c8ecb81647ce9cb95f Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 12:20:06 +0200 Subject: [PATCH 15/28] change maptools to suntools --- modules/data.atmosphere/DESCRIPTION | 2 +- .../R/met_temporal_downscale.Gaussian_ensemble.R | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/data.atmosphere/DESCRIPTION b/modules/data.atmosphere/DESCRIPTION index 9b9d5706e3a..b07be4c42d9 100644 --- a/modules/data.atmosphere/DESCRIPTION +++ b/modules/data.atmosphere/DESCRIPTION @@ -36,7 +36,6 @@ Imports: jsonlite, lubridate (>= 1.6.0), magrittr, - maptools, MASS, mgcv, ncdf4 (>= 1.15), @@ -53,6 +52,7 @@ Imports: sf, sp, stringr (>= 1.1.0), + suntools, terra, testthat (>= 2.0.0), tibble, diff --git a/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R b/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R index 1082ffb60e3..a241d1fbe5f 100644 --- a/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R +++ b/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R @@ -216,11 +216,11 @@ met_temporal_downscale.Gaussian_ensemble <- function(in.path, in.prefix, outfold day <- as.POSIXct(sprintf("%s 12:00:00", date), tz = tz) sequence <- seq(from = day, length.out = span, by = "days") - sunrise <- maptools::sunriset(lon.lat, sequence, direction = "sunrise", + sunrise <- suntools::sunriset(lon.lat, sequence, direction = "sunrise", POSIXct.out = TRUE) - sunset <- maptools::sunriset(lon.lat, sequence, direction = "sunset", + sunset <- suntools::sunriset(lon.lat, sequence, direction = "sunset", POSIXct.out = TRUE) - solar_noon <- maptools::solarnoon(lon.lat, sequence, POSIXct.out = TRUE) + solar_noon <- suntools::solarnoon(lon.lat, sequence, POSIXct.out = TRUE) data.frame(date = as.Date(sunrise$time), sunrise = as.numeric(format(sunrise$time, "%H%M")), solarnoon = as.numeric(format(solar_noon$time, "%H%M")), From c3221122c313f79b3ca41f9aa63ae1bfc9d7255b Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 12:29:43 +0200 Subject: [PATCH 16/28] add suntools remove maptools --- docker/depends/pecan.depends.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index c208e5e5525..69728e55fbc 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -76,7 +76,6 @@ wanted <- c( 'magic', 'magrittr', 'maps', -'maptools', 'markdown', 'MASS', 'Matrix', @@ -130,6 +129,7 @@ wanted <- c( 'stats', 'stringi', 'stringr', +'suntools', 'swfscMisc', 'terra', 'testthat', From b24df9751dbcfb57bb349cd65734a1be21ed9a09 Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 14:05:05 +0200 Subject: [PATCH 17/28] rgdal to sf in gis.functions --- modules/data.land/DESCRIPTION | 1 - modules/data.land/R/gis.functions.R | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/data.land/DESCRIPTION b/modules/data.land/DESCRIPTION index 5ec23616b0f..eb187ac1667 100644 --- a/modules/data.land/DESCRIPTION +++ b/modules/data.land/DESCRIPTION @@ -61,7 +61,6 @@ Suggests: PEcAn.settings, redland, raster, - rgdal, RPostgreSQL, testthat (>= 1.0.2) License: BSD_3_clause + file LICENSE diff --git a/modules/data.land/R/gis.functions.R b/modules/data.land/R/gis.functions.R index 8f9434aea3f..0fa83941f88 100644 --- a/modules/data.land/R/gis.functions.R +++ b/modules/data.land/R/gis.functions.R @@ -68,14 +68,13 @@ shp2kml <- function(dir, ext, kmz = FALSE, proj4 = NULL, color = NULL, NameField # Read in shapefile(s) & get coordinates/projection info shp.file <- # readShapeSpatial(file.path(dir,i),verbose=TRUE) coordinates(test) <- ~X+Y - layers <- rgdal::ogrListLayers(file.path(dir, i)) - info <- rgdal::ogrInfo(file.path(dir, i), layers) + layers <- sf::st_layers(file.path(dir, i)) # shp.file <- readOGR(file.path(dir,i),layer=layers) # no need to read in file # Display vector info to the console print("") - print(paste0("Input layers: ", layers)) - print(paste0("Input projection info: ", info$p4s)) + print(paste0("Input layers: ", layers$name)) + print(paste0("Input projection info: ", layers$crs[[1]]$input)) print("") # Write out kml/kmz using plotKML package if (is.null(color)){ color <- 'grey70' } @@ -133,6 +132,7 @@ get.attributes <- function(file, coords) { #library(fields) #require(rgdal) + # note that OGR support is now provided by the sf and terra packages among others # print('NOT IMPLEMENTED YET') subset_layer(file,coords) } # get.attributes From 87559d7c6130b629ca45cfe8fe68db02126515f4 Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 14:25:19 +0200 Subject: [PATCH 18/28] removed rgdal call from data.remote as well --- modules/data.remote/DESCRIPTION | 1 - modules/data.remote/R/NLCD.R | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/data.remote/DESCRIPTION b/modules/data.remote/DESCRIPTION index 68e3ee22e89..c994d1cd476 100644 --- a/modules/data.remote/DESCRIPTION +++ b/modules/data.remote/DESCRIPTION @@ -36,7 +36,6 @@ Suggests: dplyr, ggplot2, lubridate, - rgdal, reshape, testthat (>= 1.0.2), tibble diff --git a/modules/data.remote/R/NLCD.R b/modules/data.remote/R/NLCD.R index dfc712f2e09..98fc995fa1a 100644 --- a/modules/data.remote/R/NLCD.R +++ b/modules/data.remote/R/NLCD.R @@ -69,8 +69,7 @@ download.NLCD <- function(outdir, year = 2011, con = NULL) { ##' @description Based on codes from Christy Rollinson and from Max Joseph (http://mbjoseph.github.io/2014/11/08/nlcd.html) extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 2011) { library(raster) - require(rgdal) - + if (!is.null(con)) { library(PEcAn.DB) if (year == 2001) { From c97edb155c0465a2666afd0f95df9590b26427fd Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 16:34:00 +0200 Subject: [PATCH 19/28] attempt to change raster with terra --- modules/data.remote/R/NLCD.R | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/data.remote/R/NLCD.R b/modules/data.remote/R/NLCD.R index 98fc995fa1a..6c7e1ff71cb 100644 --- a/modules/data.remote/R/NLCD.R +++ b/modules/data.remote/R/NLCD.R @@ -103,21 +103,21 @@ extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 201 print(paste("File not found:", filename)) return(NULL) } - nlcd <- raster(filename) + nlcd <- terra::rast(filename) # transform points - sites <- SpatialPoints(coords = coords, proj4string = CRS("+proj=longlat +datum=WGS84")) - sites <- spTransform(sites, crs(nlcd)) + sites <- terra::vect(coords, geom=c("long", "lat"), crs="+proj=longlat +datum=WGS84") + sites <- terra::buffer(x, width=buffer) # extract - sum.raw <- table(extract(nlcd, sites, buffer = buffer)) + sum.raw <- table(terra::extract(nlcd, sites)) summ <- prop.table(sum.raw) - mydf <- data.frame(cover = names(summ), percent = as.vector(summ), count = as.vector(sum.raw)) + mydf <- data.frame(cover.name = colnames(summ), percent = as.vector(summ), count = as.vector(sum.raw)) + mydf <- mydf[mydf$count!=0,] - # land cover number to name conversions - cover.table <- nlcd@data@attributes[[1]] - cover.names <- cover.table[as.numeric(as.character(mydf$cover)) + 1, grep("Land", names(cover.table))] - mydf$cover.name <- cover.names + # land cover name to number conversions + nlcd_levels <- terra::levels(nlcd)[[1]] + mydf$cover <- nlcd_levels$value[nlcd_levels$`Land Cover Class` %in% mydf$cover.name] return(mydf) } # extract_NLCD From 1114f56796a157b55a12d4cfa4b5afb31fa5e9fb Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 16:39:38 +0200 Subject: [PATCH 20/28] add warning --- modules/data.remote/R/NLCD.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/data.remote/R/NLCD.R b/modules/data.remote/R/NLCD.R index 6c7e1ff71cb..01005ddc479 100644 --- a/modules/data.remote/R/NLCD.R +++ b/modules/data.remote/R/NLCD.R @@ -103,6 +103,9 @@ extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 201 print(paste("File not found:", filename)) return(NULL) } + + # WARNING: the following extraction previously used raster and sp package functions + # this new implementation with terra functions has not been thoroughly tested nlcd <- terra::rast(filename) # transform points From bfb03a83141bf131b23bc490c416f6e9962f838e Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 16:50:46 +0200 Subject: [PATCH 21/28] remove library call to raster --- modules/data.remote/R/NLCD.R | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/data.remote/R/NLCD.R b/modules/data.remote/R/NLCD.R index 01005ddc479..a0416cdebbb 100644 --- a/modules/data.remote/R/NLCD.R +++ b/modules/data.remote/R/NLCD.R @@ -68,7 +68,6 @@ download.NLCD <- function(outdir, year = 2011, con = NULL) { ##' ##' @description Based on codes from Christy Rollinson and from Max Joseph (http://mbjoseph.github.io/2014/11/08/nlcd.html) extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 2011) { - library(raster) if (!is.null(con)) { library(PEcAn.DB) From 20315900ada53664babefe13f957899f45fd7cc8 Mon Sep 17 00:00:00 2001 From: istfer Date: Wed, 1 Nov 2023 16:51:45 +0200 Subject: [PATCH 22/28] fix typo --- modules/data.remote/R/NLCD.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data.remote/R/NLCD.R b/modules/data.remote/R/NLCD.R index a0416cdebbb..0e891aec2eb 100644 --- a/modules/data.remote/R/NLCD.R +++ b/modules/data.remote/R/NLCD.R @@ -109,7 +109,7 @@ extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 201 # transform points sites <- terra::vect(coords, geom=c("long", "lat"), crs="+proj=longlat +datum=WGS84") - sites <- terra::buffer(x, width=buffer) + sites <- terra::buffer(sites, width=buffer) # extract sum.raw <- table(terra::extract(nlcd, sites)) From 8263c71e923cac88e92431b4be36c72d96d0a898 Mon Sep 17 00:00:00 2001 From: istfer Date: Thu, 2 Nov 2023 17:09:10 +0200 Subject: [PATCH 23/28] remove library and require calls from data.remote --- modules/data.remote/R/NLCD.R | 4 ++-- modules/data.remote/R/download.thredds.R | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/data.remote/R/NLCD.R b/modules/data.remote/R/NLCD.R index 0e891aec2eb..32517eff2ae 100644 --- a/modules/data.remote/R/NLCD.R +++ b/modules/data.remote/R/NLCD.R @@ -23,7 +23,7 @@ download.NLCD <- function(outdir, year = 2011, con = NULL) { ## before downloading, check if the file already exists on this host if (!is.null(con)) { - library(PEcAn.DB) + chk <- dbfile.check(type = "Input", id = input.id, con = con) if (nrow(chk) > 0) { machines <- db.query(paste("SELECT * from machines where id in (", @@ -70,7 +70,7 @@ download.NLCD <- function(outdir, year = 2011, con = NULL) { extract_NLCD <- function(buffer, coords, data_dir = NULL, con = NULL, year = 2011) { if (!is.null(con)) { - library(PEcAn.DB) + if (year == 2001) { input.id <- 1000000482 } else if (year == 2011) { diff --git a/modules/data.remote/R/download.thredds.R b/modules/data.remote/R/download.thredds.R index 09d44ac1337..04fc4b99923 100755 --- a/modules/data.remote/R/download.thredds.R +++ b/modules/data.remote/R/download.thredds.R @@ -82,7 +82,7 @@ download.thredds.AGB <- function(outdir = NULL, site_ids, run_parallel = FALSE, } else { ncores <- parallel::detectCores() -1 } - require(doParallel) + PEcAn.logger::logger.info(paste0("Running in parallel with: ", ncores)) cl = parallel::makeCluster(ncores) doParallel::registerDoParallel(cl) From 9137628fa99dd82ab1d1e98b059092d879fbdbcb Mon Sep 17 00:00:00 2001 From: istfer Date: Thu, 2 Nov 2023 18:56:38 +0200 Subject: [PATCH 24/28] CI updates --- modules/data.remote/DESCRIPTION | 3 ++- modules/data.remote/tests/Rcheck_reference.log | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/data.remote/DESCRIPTION b/modules/data.remote/DESCRIPTION index c994d1cd476..617f49b1338 100644 --- a/modules/data.remote/DESCRIPTION +++ b/modules/data.remote/DESCRIPTION @@ -20,7 +20,6 @@ Imports: PEcAn.utils, purrr, XML, - raster, sp, MODISTools (>= 1.1.0), reticulate, @@ -28,6 +27,7 @@ Imports: magrittr, PEcAn.remote, stringr (>= 1.1.0), + terra, doParallel, parallel, foreach @@ -36,6 +36,7 @@ Suggests: dplyr, ggplot2, lubridate, + raster, reshape, testthat (>= 1.0.2), tibble diff --git a/modules/data.remote/tests/Rcheck_reference.log b/modules/data.remote/tests/Rcheck_reference.log index 822a9411303..182f76f6d12 100644 --- a/modules/data.remote/tests/Rcheck_reference.log +++ b/modules/data.remote/tests/Rcheck_reference.log @@ -82,8 +82,6 @@ Author field differs from that derived from Authors@R * checking loading without being on the library search path ... OK * checking use of S3 registration ... OK * checking dependencies in R code ... WARNING -'library' or 'require' calls not declared from: - ‘PEcAn.DB’ ‘doParallel’ ‘raster’ 'library' or 'require' calls in package code: ‘PEcAn.DB’ ‘doParallel’ ‘raster’ ‘rgdal’ Please use :: or requireNamespace() instead. From 00a12222c8e5a6eab8ad7ebfb3e4ab7a73065de9 Mon Sep 17 00:00:00 2001 From: istfer Date: Fri, 3 Nov 2023 09:07:22 +0200 Subject: [PATCH 25/28] generate dependencies --- docker/depends/pecan.depends.R | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index 69728e55fbc..5b03d7297e5 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -112,7 +112,6 @@ wanted <- c( 'reshape', 'reshape2', 'reticulate', -'rgdal', 'rjags', 'rjson', 'rlang', From a4c7e14697afc172710a235b31a25e9e4c663c11 Mon Sep 17 00:00:00 2001 From: istfer Date: Sun, 5 Nov 2023 09:32:13 +0200 Subject: [PATCH 26/28] remote install suntools --- docker/depends/pecan.depends.R | 2 +- modules/data.atmosphere/DESCRIPTION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index 5b03d7297e5..3e292529a6b 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -9,6 +9,7 @@ Sys.setenv(RLIB = rlib) # install remotes first in case packages are references in dependencies remotes::install_github(c( +'adokter/suntools', 'araiho/linkages_package', 'chuhousen/amerifluxr', 'ebimodeling/biocro@0.951', @@ -128,7 +129,6 @@ wanted <- c( 'stats', 'stringi', 'stringr', -'suntools', 'swfscMisc', 'terra', 'testthat', diff --git a/modules/data.atmosphere/DESCRIPTION b/modules/data.atmosphere/DESCRIPTION index b07be4c42d9..8f0482e9e41 100644 --- a/modules/data.atmosphere/DESCRIPTION +++ b/modules/data.atmosphere/DESCRIPTION @@ -52,7 +52,6 @@ Imports: sf, sp, stringr (>= 1.1.0), - suntools, terra, testthat (>= 2.0.0), tibble, @@ -70,6 +69,7 @@ Suggests: progress, reticulate Remotes: + github::adokter/suntools, github::chuhousen/amerifluxr, github::ropensci/geonames, github::ropensci/nneo From ad8adce78cccd7b5b457f96f475eb5d488972920 Mon Sep 17 00:00:00 2001 From: istfer Date: Sun, 5 Nov 2023 09:52:49 +0200 Subject: [PATCH 27/28] working with CI --- docker/depends/pecan.depends.R | 1 + modules/data.atmosphere/DESCRIPTION | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R index 3e292529a6b..e49b8fb72fb 100644 --- a/docker/depends/pecan.depends.R +++ b/docker/depends/pecan.depends.R @@ -129,6 +129,7 @@ wanted <- c( 'stats', 'stringi', 'stringr', +'suntools', 'swfscMisc', 'terra', 'testthat', diff --git a/modules/data.atmosphere/DESCRIPTION b/modules/data.atmosphere/DESCRIPTION index 8f0482e9e41..097a134349e 100644 --- a/modules/data.atmosphere/DESCRIPTION +++ b/modules/data.atmosphere/DESCRIPTION @@ -52,6 +52,7 @@ Imports: sf, sp, stringr (>= 1.1.0), + suntools, terra, testthat (>= 2.0.0), tibble, From 6900a0f40410792c6aa11a1579e480691a15e2b6 Mon Sep 17 00:00:00 2001 From: istfer Date: Sun, 5 Nov 2023 10:09:52 +0200 Subject: [PATCH 28/28] rcheck ref --- modules/data.atmosphere/tests/Rcheck_reference.log | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data.atmosphere/tests/Rcheck_reference.log b/modules/data.atmosphere/tests/Rcheck_reference.log index 1f7a42f6634..be30baa7ac2 100644 --- a/modules/data.atmosphere/tests/Rcheck_reference.log +++ b/modules/data.atmosphere/tests/Rcheck_reference.log @@ -76,13 +76,13 @@ Found the following (possibly) invalid URLs: The Date field is over a month old. * checking package namespace information ... OK * checking package dependencies ... WARNING -Imports includes 39 non-default packages. +Imports includes 40 non-default packages. Importing from so many packages makes the package vulnerable to any of them becoming unavailable. Move as many as possible to Suggests and use conditionally. * checking package dependencies ... NOTE -Imports includes 39 non-default packages. +Imports includes 40 non-default packages. Importing from so many packages makes the package vulnerable to any of them becoming unavailable. Move as many as possible to Suggests and use conditionally.