From fce6beee81f6b1af586431ec6de0c841fc112d08 Mon Sep 17 00:00:00 2001 From: Diego H Date: Wed, 14 Sep 2022 14:31:22 +0000 Subject: [PATCH] Add batch download of series --- NAMESPACE | 1 + R/thematic.R | 139 +++++++++++++++++++++++++++++++++ inst/WORDLIST | 17 ++-- man/bde_series_full_load.Rd | 3 +- man/bde_series_load.Rd | 3 +- man/bde_themes_download.Rd | 70 +++++++++++++++++ tests/testthat/test-catalogs.R | 36 ++++----- tests/testthat/test-thematic.R | 17 ++++ 8 files changed, 257 insertions(+), 29 deletions(-) create mode 100644 R/thematic.R create mode 100644 man/bde_themes_download.Rd create mode 100644 tests/testthat/test-thematic.R diff --git a/NAMESPACE b/NAMESPACE index 5a98711c..52aa77cb 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -16,6 +16,7 @@ export(bde_parse_dates) export(bde_rose_pal) export(bde_series_full_load) export(bde_series_load) +export(bde_themes_download) export(bde_vivid_pal) export(scale_color_bde_c) export(scale_color_bde_d) diff --git a/R/thematic.R b/R/thematic.R new file mode 100644 index 00000000..24fd7439 --- /dev/null +++ b/R/thematic.R @@ -0,0 +1,139 @@ +#' Download thematic classified files +#' +#' Download several series based on the thematic classification of BdE. +#' +#' @export +#' +#' @family series +#' +#' +#' @encoding UTF-8 +#' +#' @param themes A single value indicating the themes to be downloaded +#' or `"ALL"` as a shorthand. See **Details**. +#' +#' +#' @inheritParams bde_series_full_load +#' +#' +#' @return Invisible. This function is called for its side effects. +#' +#' @source [Time-series bulk data download](https://www.bde.es/webbde/en/estadis/infoest/descarga_series_temporales.html) +#' +#' @description +#' +#' By the time, this function just batch-download all the series included +#' in the +#' [Theme-based classification](https://www.bde.es/webbde/en/estadis/infoest/descarga_series_temporales.html) +#' provided by Bank of Spain. However, this is useful as the function +#' [bde_series_load_all()] and [bde_series_load()] would search also in the +#' downloaded files, providing a faster response to the end user. +#' +#' @seealso [bde_series_load_all()] +#' +#' +#' @details +#' Accepted values for `themes` are: +#' +#' ```{r, echo=FALSE} +#' +#' t <- tibble::tribble( +#' ~themes, ~Description, +#' "**TE_TIPOS**", "Interest rate statistics", +#' "**TE_TIPOSCAM**", "Exchange rate statistics", +#' "**TE_CF**", "Financial accounts statistics", +#' "**TE_IFINA**", "Financial corporations statistics", +#' "**TE_DEU**", "General government statistics", +#' "**TE_CENBAL**", "Non-Financial corporations statistics", +#' "**TE_SECEXT**", "External statistics", +#' "**TE_MERCA**", "Financial market statistics", +#' "**TE_ESTECON**", "General economic statistics", +#' ) +#' +#' +#' knitr::kable(t) + +#' +#' +#' ``` +#' +#' Use `"ALL"` as a shorthand for updating all the catalogs at a glance. +#' +#' If this function has been called, [bde_series_load_all()] and +#' [bde_series_load()] would try to find the series on the downloaded files. +#' +#' @examplesIf bde_check_access() +#' \dontrun{ +#' bde_themes_download() +#' } +bde_themes_download <- function(themes = "ALL", cache_dir = NULL, + verbose = FALSE) { + # Validate + valid_themes <- c( + "TE_TIPOS", "TE_TIPOSCAM", "TE_CF", + "TE_IFINA", "TE_DEU", "TE_CENBAL", + "TE_SECEXT", "TE_MERCA", "TE_ESTECON", + "ALL" + ) + stopifnot( + themes %in% valid_themes, + length(themes) == 1, + is.null(cache_dir) || is.character(cache_dir) + ) + + # nocov start + if (!bde_check_access()) { + message("tidyBdE> Offline") + return(invisible()) + } + # nocov end + + if (themes == "ALL") { + themes_batch <- setdiff(valid_themes, "ALL") + } else { + themes_batch <- themes + } + + themes_batch <- paste0(themes_batch, ".zip") + + # Helper function to download and unzip files + unzip_themes <- function(x, cache_dir = cache_dir, + verbose = verbose) { + baseurl <- "https://www.bde.es/webbde/es/estadis/infoest/series/" + fullurl <- paste0(baseurl, x) + cache_dir <- bde_hlp_cachedir(cache_dir) + local_zip <- file.path(cache_dir, x) + + + bde_hlp_download(fullurl, local_zip, verbose = verbose) + + # Extract all files + exdir <- bde_hlp_cachedir(cache_dir = cache_dir, suffix = "TE") + if (verbose) { + message("tidyBdE> Extracting ", x, " content on ", exdir) + } + unzip(local_zip, exdir = exdir, junkpaths = FALSE) + + if (verbose) { + message("tidyBdE> Removing ", local_zip) + } + unlink(local_zip) + return(NULL) + } + + # Apply to all + r <- lapply(themes_batch, unzip_themes, + cache_dir = cache_dir, + verbose = verbose + ) + + + if (verbose) { + exdir <- bde_hlp_cachedir(cache_dir, suffix = "TE") + nfiles <- length(list.files(exdir)) + + message("tidyBdE> Sucess! ", nfiles, " files available in ", exdir) + } + + return(invisible()) +} diff --git a/inst/WORDLIST b/inst/WORDLIST index a2e0db47..91cc5243 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -1,41 +1,40 @@ Banco BdE +CENBAL CMD CatastRo Catastro -DESCRIPCIÓN +DEU DOI -DTCCBCEGBPEUR Datos Descripcion +ESTECON Económicos España Euribor Eurostat -FRECUENCIA GBP Geográfico Herrero +IFINA IGN INE Indicadores Instituto -LABORABLE +MERCA MMM MMMMYYYY MicroDatosEs -NOMBRE Nacional Numero -NÚMERO ORCID Precompile -SECUENCIAL -SERIE +SECEXT SSM Sede +TIPOS +TIPOSCAM Tipo -UNIDADES YYYY bde cambio diff --git a/man/bde_series_full_load.Rd b/man/bde_series_full_load.Rd index 11a0017c..00e19d9a 100644 --- a/man/bde_series_full_load.Rd +++ b/man/bde_series_full_load.Rd @@ -75,6 +75,7 @@ bde_series_full_load("TI_1_1.csv") } \seealso{ Other series: -\code{\link{bde_series_load}()} +\code{\link{bde_series_load}()}, +\code{\link{bde_themes_download}()} } \concept{series} diff --git a/man/bde_series_load.Rd b/man/bde_series_load.Rd index 1158d998..30202d79 100644 --- a/man/bde_series_load.Rd +++ b/man/bde_series_load.Rd @@ -96,6 +96,7 @@ bde_series_load(c(573234, 573214), \code{\link[=bde_catalog_search]{bde_catalog_search()}}, \code{\link[=bde_indicators]{bde_indicators()}} Other series: -\code{\link{bde_series_full_load}()} +\code{\link{bde_series_full_load}()}, +\code{\link{bde_themes_download}()} } \concept{series} diff --git a/man/bde_themes_download.Rd b/man/bde_themes_download.Rd new file mode 100644 index 00000000..76510a5a --- /dev/null +++ b/man/bde_themes_download.Rd @@ -0,0 +1,70 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/thematic.R +\encoding{UTF-8} +\name{bde_themes_download} +\alias{bde_themes_download} +\title{Batch download of files included on the thematic classification} +\source{ +\href{https://www.bde.es/webbde/en/estadis/infoest/descarga_series_temporales.html}{Time-series bulk data download} +} +\usage{ +bde_themes_download(themes = "ALL", cache_dir = NULL, verbose = FALSE) +} +\arguments{ +\item{themes}{A single value indicating the themes to be downloaded +or \code{"ALL"} as a shorthand. See \strong{Details}.} + +\item{cache_dir}{A path to a cache directory. The directory can also be set +via options with \code{options(bde_cache_dir = "path/to/dir")}.} + +\item{verbose}{Logical \code{TRUE} or \code{FALSE}, display information useful for +debugging.} +} +\value{ +Invisible. This function is called for its side effects. +} +\description{ +By the time, this function just batch-download all the series included +in the +\href{https://www.bde.es/webbde/en/estadis/infoest/descarga_series_temporales.html}{Theme-based classification} +provided by Bank of Spain. However, this is useful as the function +\code{\link[=bde_series_load_all]{bde_series_load_all()}} and \code{\link[=bde_series_load]{bde_series_load()}} would search also in the +downloaded files, providing a faster response to the end user. +} +\details{ +Download several series based on the thematic classification of BdE. + +Accepted values for \code{themes} are:\tabular{ll}{ + themes \tab Description \cr + \strong{TE_TIPOS} \tab Interest rate statistics \cr + \strong{TE_TIPOSCAM} \tab Exchange rate statistics \cr + \strong{TE_CF} \tab Financial accounts statistics \cr + \strong{TE_IFINA} \tab Financial corporations statistics \cr + \strong{TE_DEU} \tab General government statistics \cr + \strong{TE_CENBAL} \tab Non-Financial corporations statistics \cr + \strong{TE_SECEXT} \tab External statistics \cr + \strong{TE_MERCA} \tab Financial market statistics \cr + \strong{TE_ESTECON} \tab General economic statistics \cr +} + + +Use \code{"ALL"} as a shorthand for updating all the catalogs at a glance. + +If this function has been called, \code{\link[=bde_series_load_all]{bde_series_load_all()}} and +\code{\link[=bde_series_load]{bde_series_load()}} would try to find the series on the downloaded files. +} +\examples{ +\dontshow{if (bde_check_access()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +\dontrun{ +bde_themes_download() +} +\dontshow{\}) # examplesIf} +} +\seealso{ +\code{\link[=bde_series_load_all]{bde_series_load_all()}} + +Other series: +\code{\link{bde_series_full_load}()}, +\code{\link{bde_series_load}()} +} +\concept{series} diff --git a/tests/testthat/test-catalogs.R b/tests/testthat/test-catalogs.R index 4a2e181e..f5dd89b6 100644 --- a/tests/testthat/test-catalogs.R +++ b/tests/testthat/test-catalogs.R @@ -1,12 +1,12 @@ test_that("Catalogs", { skip_on_cran() skip_if_bde_offline() - + # Test load catalogs---- expect_error(bde_catalog_load("aa")) expect_error(bde_catalog_search("xxxxxxx", catalog = "IE")) - - + + expect_message(bde_catalog_load("TC", cache_dir = tempdir(), verbose = TRUE)) expect_message(bde_catalog_load("TC", cache_dir = NULL, verbose = TRUE)) expect_message(bde_catalog_load( @@ -14,36 +14,36 @@ test_that("Catalogs", { cache_dir = file.path(tempdir(), "aa"), verbose = TRUE )) - + expect_silent(bde_catalog_load("ALL")) - - - + + + # Test update catalogs---- - + expect_error(bde_catalog_update("aa")) - - + + expect_message(bde_catalog_update("TC", - cache_dir = tempdir(), - verbose = TRUE + cache_dir = tempdir(), + verbose = TRUE )) expect_silent(bde_catalog_update("ALL", cache_dir = tempdir())) expect_silent(bde_catalog_update("TC", cache_dir = tempdir())) - + # Testing options cache dir init_cache_dir <- getOption("bde_cache_dir") options(bde_cache_dir = file.path(tempdir(), "test")) expect_message(bde_catalog_update("TC", verbose = TRUE)) # Reset options(bde_cache_dir = init_cache_dir) - - + + # Test bde_catalog_search---- - + expect_error(bde_catalog_search()) - - + + expect_silent(bde_catalog_search("Euro", catalog = "TC")) }) diff --git a/tests/testthat/test-thematic.R b/tests/testthat/test-thematic.R new file mode 100644 index 00000000..3b18cdd9 --- /dev/null +++ b/tests/testthat/test-thematic.R @@ -0,0 +1,17 @@ +test_that("Thematic", { + skip_on_cran() + skip_if_bde_offline() + + # Test load catalogs---- + expect_error(bde_themes_download("aa")) + + expect_message(bde_themes_download("TE_DEU", + cache_dir = tempdir(), + verbose = TRUE + )) + expect_message(bde_themes_download("TE_DEU", + cache_dir = NULL, + verbose = TRUE + )) + expect_silent(bde_themes_download("ALL")) +})