Read data from qs files

DESCRIPTION

@@ -53,7 +53,8 @@ Imports:
     rappdirs,
     future,
     promises,
-    methods,
+    methods, 
+    qs,
     joyn
 Remotes:
     PIP-Technical-Team/wbpip@master

NEWS.md

@@ -1,6 +1,8 @@
 # pipapi (development version)
 
 ## New features
+
+- [Read data in `_aux` folder from qs files](https://github.com/PIP-Technical-Team/pipapi/issues/271)
 - [Add new endpoint to support country profile downloads](https://github.com/PIP-Technical-Team/pipapi/pull/283)
 - [Add separate arguments to select release and ppp version](https://github.com/PIP-Technical-Team/pipapi/issues/265)
 - [Allow direct selection of non-official WB regions](https://github.com/PIP-Technical-Team/pipapi/pull/287)
@@ -12,6 +14,7 @@
 # pipapi 1.1.0
 
 ## Enhancements
+
 - [Sort the rows by `country_code` and `reporting_year` in the pip response.](https://github.com/PIP-Technical-Team/pipapi/issues/248)
 - [Fix casing making pip call case insensitive](https://github.com/PIP-Technical-Team/pipapi/issues/120)
 - [Add unit tests for newly created fg_remove_duplicates() and sub-functions](https://github.com/PIP-Technical-Team/pipapi/issues/226)
@@ -21,6 +24,7 @@
 - [Better error message when passing more than one dataset as `lkup` in `pip` call](https://github.com/PIP-Technical-Team/pipapi/issues/263)
 
 ## New features
+
 - Region codes can now be passed directly to the `country` query parameter to 
 return all countries pertaining to the specified region
 - the `/valid-params` endpoint gains an `endpoint` parameter that allows to only

R/create_lkups.R

@@ -78,11 +78,11 @@ create_lkups <- function(data_dir, versions) {
   paths_ids <- tools::file_path_sans_ext(paths)
 
   # TEMP FIX to add country and region name
-  cts_path <- fs::path(data_dir, "_aux/countries.fst")
-  reg_path <- fs::path(data_dir, "_aux/regions.fst")
+  cts_path <- fs::path(data_dir, "_aux/countries.qs")
+  reg_path <- fs::path(data_dir, "_aux/regions.qs")
 
-  countries <-  fst::read_fst(cts_path,as.data.table = TRUE)
-  regions   <-  fst::read_fst(reg_path,as.data.table = TRUE)
+  countries <-  qs::qread(cts_path) %>% data.table::as.data.table()
+  regions   <-  qs::qread(reg_path) %>% data.table::as.data.table()
 
   data.table::setnames(countries, 'region', 'region_name')
 
@@ -186,20 +186,20 @@ create_lkups <- function(data_dir, versions) {
   dist_stats      <- fst::read_fst(dist_stats_path, as.data.table = TRUE)
 
   # Load pop_region
-  pop_region_path <- fs::path(data_dir, "_aux/pop_region.fst")
-  pop_region      <- fst::read_fst(pop_region_path,as.data.table = TRUE)
+  pop_region_path <- fs::path(data_dir, "_aux/pop_region.qs")
+  pop_region      <- qs::qread(pop_region_path) %>% data.table::as.data.table()
 
   # Load country profiles lkups
-  cp_lkups_path   <- fs::path(data_dir, "_aux/country_profiles.rds")
-  cp_lkups        <- readRDS(cp_lkups_path)
+  cp_lkups_path   <- fs::path(data_dir, "_aux/country_profiles.qs")
+  cp_lkups        <- qs::qread(cp_lkups_path) %>% data.table::as.data.table()
 
   # Load poverty lines table
-  pl_lkup_path    <- fs::path(data_dir, "_aux/poverty_lines.fst")
-  pl_lkup         <- fst::read_fst(pl_lkup_path, as.data.table = TRUE)
+  pl_lkup_path    <- fs::path(data_dir, "_aux/poverty_lines.qs")
+  pl_lkup         <- qs::qread(pl_lkup_path) %>% data.table::as.data.table()
 
   # Load list with censor tables
-  censored_path   <- fs::path(data_dir, "_aux/censored.rds")
-  censored        <- readRDS(censored_path)
+  censored_path   <- fs::path(data_dir, "_aux/censored.qs")
+  censored        <- qs::qread(censored_path) %>% data.table::as.data.table()
 
 
   # Files with country and region information -----
@@ -279,7 +279,7 @@ create_lkups <- function(data_dir, versions) {
     )
 
   # Create list of available auxiliary data tables
-  aux_tables <- list.files(fs::path(data_dir, "_aux"),pattern = "\\.fst$")
+  aux_tables <- list.files(fs::path(data_dir, "_aux"),pattern = "\\.qs$")
   aux_tables <- tools::file_path_sans_ext(aux_tables)
   aux_tables <- sort(aux_tables)
   valid_years <- valid_years(data_dir)

R/get_aux_table.R

@@ -19,8 +19,8 @@ get_aux_table <- function(data_dir, table, long_format = FALSE) {
   # Strip all "non-word" characters from user input
   sanitized_table <- gsub("\\W", "", table)
 
-  out <- fst::read_fst(sprintf(
-    "%s/_aux/%s.fst",
+  out <- qs::qread(sprintf(
+    "%s/_aux/%s.qs",
     data_dir,
     sanitized_table
   ))

README.Rmd

@@ -49,8 +49,8 @@ The folder structure looks like this:
 ```{bash eval=FALSE, echo=TRUE}
 data-folder-root/
 ├─ _aux/
-│  ├─ pop_regions.fst
-│  ├─ pop.fst
+│  ├─ pop_regions.qs
+│  ├─ pop.qs
 │  ├─ ...
 ├─ estimations/
 │  ├─ prod_svy_estimation.fst

README.md

@@ -40,8 +40,8 @@ In order to use `pip()` you’ll need to have access to a PIP
 ``` bash
 data-folder-root/
 ├─ _aux/
-│  ├─ pop_regions.fst
-│  ├─ pop.fst
+│  ├─ pop_regions.qs
+│  ├─ pop.qs
 │  ├─ ...
 ├─ estimations/
 │  ├─ prod_svy_estimation.fst