diff --git a/DESCRIPTION b/DESCRIPTION index 9ea1ffa1..597f5787 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -53,7 +53,8 @@ Imports: rappdirs, future, promises, - methods, + methods, + qs, joyn Remotes: PIP-Technical-Team/wbpip@master diff --git a/NEWS.md b/NEWS.md index 0b47c452..631c6609 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,8 @@ # pipapi (development version) ## New features + +- [Read data in `_aux` folder from qs files](https://github.com/PIP-Technical-Team/pipapi/issues/271) - [Add new endpoint to support country profile downloads](https://github.com/PIP-Technical-Team/pipapi/pull/283) - [Add separate arguments to select release and ppp version](https://github.com/PIP-Technical-Team/pipapi/issues/265) - [Allow direct selection of non-official WB regions](https://github.com/PIP-Technical-Team/pipapi/pull/287) @@ -12,6 +14,7 @@ # pipapi 1.1.0 ## Enhancements + - [Sort the rows by `country_code` and `reporting_year` in the pip response.](https://github.com/PIP-Technical-Team/pipapi/issues/248) - [Fix casing making pip call case insensitive](https://github.com/PIP-Technical-Team/pipapi/issues/120) - [Add unit tests for newly created fg_remove_duplicates() and sub-functions](https://github.com/PIP-Technical-Team/pipapi/issues/226) @@ -21,6 +24,7 @@ - [Better error message when passing more than one dataset as `lkup` in `pip` call](https://github.com/PIP-Technical-Team/pipapi/issues/263) ## New features + - Region codes can now be passed directly to the `country` query parameter to return all countries pertaining to the specified region - the `/valid-params` endpoint gains an `endpoint` parameter that allows to only diff --git a/R/create_lkups.R b/R/create_lkups.R index 8c53d225..229e61b9 100644 --- a/R/create_lkups.R +++ b/R/create_lkups.R @@ -78,11 +78,11 @@ create_lkups <- function(data_dir, versions) { paths_ids <- tools::file_path_sans_ext(paths) # TEMP FIX to add country and region name - cts_path <- fs::path(data_dir, "_aux/countries.fst") - reg_path <- fs::path(data_dir, "_aux/regions.fst") + cts_path <- fs::path(data_dir, "_aux/countries.qs") + reg_path <- fs::path(data_dir, "_aux/regions.qs") - countries <- fst::read_fst(cts_path,as.data.table = TRUE) - regions <- fst::read_fst(reg_path,as.data.table = TRUE) + countries <- qs::qread(cts_path) %>% data.table::as.data.table() + regions <- qs::qread(reg_path) %>% data.table::as.data.table() data.table::setnames(countries, 'region', 'region_name') @@ -186,20 +186,20 @@ create_lkups <- function(data_dir, versions) { dist_stats <- fst::read_fst(dist_stats_path, as.data.table = TRUE) # Load pop_region - pop_region_path <- fs::path(data_dir, "_aux/pop_region.fst") - pop_region <- fst::read_fst(pop_region_path,as.data.table = TRUE) + pop_region_path <- fs::path(data_dir, "_aux/pop_region.qs") + pop_region <- qs::qread(pop_region_path) %>% data.table::as.data.table() # Load country profiles lkups - cp_lkups_path <- fs::path(data_dir, "_aux/country_profiles.rds") - cp_lkups <- readRDS(cp_lkups_path) + cp_lkups_path <- fs::path(data_dir, "_aux/country_profiles.qs") + cp_lkups <- qs::qread(cp_lkups_path) %>% data.table::as.data.table() # Load poverty lines table - pl_lkup_path <- fs::path(data_dir, "_aux/poverty_lines.fst") - pl_lkup <- fst::read_fst(pl_lkup_path, as.data.table = TRUE) + pl_lkup_path <- fs::path(data_dir, "_aux/poverty_lines.qs") + pl_lkup <- qs::qread(pl_lkup_path) %>% data.table::as.data.table() # Load list with censor tables - censored_path <- fs::path(data_dir, "_aux/censored.rds") - censored <- readRDS(censored_path) + censored_path <- fs::path(data_dir, "_aux/censored.qs") + censored <- qs::qread(censored_path) %>% data.table::as.data.table() # Files with country and region information ----- @@ -279,7 +279,7 @@ create_lkups <- function(data_dir, versions) { ) # Create list of available auxiliary data tables - aux_tables <- list.files(fs::path(data_dir, "_aux"),pattern = "\\.fst$") + aux_tables <- list.files(fs::path(data_dir, "_aux"),pattern = "\\.qs$") aux_tables <- tools::file_path_sans_ext(aux_tables) aux_tables <- sort(aux_tables) valid_years <- valid_years(data_dir) diff --git a/R/get_aux_table.R b/R/get_aux_table.R index 314a33f7..290496c0 100644 --- a/R/get_aux_table.R +++ b/R/get_aux_table.R @@ -19,8 +19,8 @@ get_aux_table <- function(data_dir, table, long_format = FALSE) { # Strip all "non-word" characters from user input sanitized_table <- gsub("\\W", "", table) - out <- fst::read_fst(sprintf( - "%s/_aux/%s.fst", + out <- qs::qread(sprintf( + "%s/_aux/%s.qs", data_dir, sanitized_table )) diff --git a/README.Rmd b/README.Rmd index c190c382..a00934cb 100644 --- a/README.Rmd +++ b/README.Rmd @@ -49,8 +49,8 @@ The folder structure looks like this: ```{bash eval=FALSE, echo=TRUE} data-folder-root/ ├─ _aux/ -│ ├─ pop_regions.fst -│ ├─ pop.fst +│ ├─ pop_regions.qs +│ ├─ pop.qs │ ├─ ... ├─ estimations/ │ ├─ prod_svy_estimation.fst diff --git a/README.md b/README.md index 55582988..f0d270f4 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ In order to use `pip()` you’ll need to have access to a PIP ``` bash data-folder-root/ ├─ _aux/ -│ ├─ pop_regions.fst -│ ├─ pop.fst +│ ├─ pop_regions.qs +│ ├─ pop.qs │ ├─ ... ├─ estimations/ │ ├─ prod_svy_estimation.fst