From 1967298393d20e6db9e8360d7c025b6c0c88c160 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Mon, 3 Jun 2019 15:49:56 -0400 Subject: [PATCH] Add checking methods: `check_code`, `check_result`, and `test_result` (#10) - Broken. Needs to be fixed * Fix #7, #8: Compare user and solution based on expressions * ignore all html tutorial files * roxygen version * white space * add `result` to convey results from grading * rename strict_check.R to check_strict.R and upgrade how function is handled * add a check_result method and partially generalize grade_learnr * use check_code vs check_strict * check result should not check code * add checkmate helpers * pull out praise and encourage and make random helper methods * make sure grading_code is called from envir_result * copy extra args * report error an error occured while checking answer * add helper methods for getting user and solution code * add a results fn * add a `test_result` method * if the tutorial is trying to be run from not inside rstudio IDE, browse the URL * open file _url_ if rstudio isn't available * update grading-demo doc * do not require solution code * more documentation * Update DESCRIPTION Use GPL-3 (cuz shiny). Add learnr remote to get at `envir_prep` * the pkg is grader not teachr * documentation and passing R cmd check. Tests still fail * pass tests * white space * fix tests * don't use tidyverse in tests * result message should be "" if NULL is supplied. helps with printing * make sure error result message is a character * correct the order * use glue_data over glue * todo solved * use answer not found obj * tests for ex checking methods * documentation * ignore all the things * use travis * use codecov * use pkgdown * update description * use a readme with a bunch of badges * evaluate all *-check code except for the last value. Evaluate that like normal. Move all code inside trycatch * add custom check code example * Require testthat for checkmate expect_* methods * use github links in description * update authors * bump learnr version * `results` does not exist anymore * add `grader_args`/`learnr_args` and use `graded()` for computed result * import %||% from rlang * add possible correct/incorrect methods * document --- .Rbuildignore | 6 + .gitignore | 2 + .travis.yml | 51 +++ DESCRIPTION | 28 +- LICENSE | 21 -- NAMESPACE | 18 +- R/check_code.R | 138 ++++++++ R/check_result.R | 66 ++++ R/checkmate.R | 11 + R/correct.R | 30 ++ R/detect_mistakes.R | 118 +++---- R/grade_learnr.R | 207 ++++++----- R/grading_demo.R | 27 +- R/message_generators.R | 49 +-- R/order_calls.R | 37 +- R/praise.R | 69 ++++ R/result.R | 39 +++ R/strict_check.R | 80 ----- R/test_result.R | 151 ++++++++ R/test_solutions.R | 60 ++-- R/utils.R | 5 +- R/view_tutorial.R | 36 +- README.md | 31 +- _pkgdown.yml | 1 + codecov.yml | 12 + inst/tutorials/grading-demo/grading-demo.Rmd | 279 ++++++++++++++- inst/tutorials/grading-demo/grading-demo.html | 146 -------- .../recording-demo/recording-demo.Rmd | 2 +- man/add_tutorial.Rd | 4 +- man/check_code.Rd | 62 ++++ man/check_result.Rd | 41 +++ man/correct.Rd | 17 + man/figures/missing_sqrt.png | Bin 0 -> 42714 bytes man/get_code.Rd | 30 ++ man/grade_learnr.Rd | 15 +- man/grading_demo.Rd | 13 +- man/order_calls.Rd | 8 - man/praise.Rd | 18 + man/result.Rd | 25 ++ man/strict_check.Rd | 56 --- man/test.Rd | 44 +++ man/test_result.Rd | 52 +++ man/test_solutions.Rd | 4 - tests/testthat.R | 4 +- tests/testthat/test_check_code.R | 198 +++++++++++ tests/testthat/test_check_result.R | 107 ++++++ tests/testthat/test_detect_mistakes.R | 325 +++++++++--------- tests/testthat/test_strict_check.R | 197 ----------- tests/testthat/test_test_result.R | 94 +++++ tests/testthat/test_unpipe.R | 9 +- 50 files changed, 2081 insertions(+), 962 deletions(-) create mode 100644 .travis.yml delete mode 100644 LICENSE create mode 100644 R/check_code.R create mode 100644 R/check_result.R create mode 100644 R/checkmate.R create mode 100644 R/correct.R create mode 100644 R/praise.R create mode 100644 R/result.R delete mode 100644 R/strict_check.R create mode 100644 R/test_result.R create mode 100644 _pkgdown.yml create mode 100644 codecov.yml delete mode 100644 inst/tutorials/grading-demo/grading-demo.html create mode 100644 man/check_code.Rd create mode 100644 man/check_result.Rd create mode 100644 man/correct.Rd create mode 100644 man/figures/missing_sqrt.png create mode 100644 man/get_code.Rd delete mode 100644 man/order_calls.Rd create mode 100644 man/praise.Rd create mode 100644 man/result.Rd delete mode 100644 man/strict_check.Rd create mode 100644 man/test.Rd create mode 100644 man/test_result.Rd create mode 100644 tests/testthat/test_check_code.R create mode 100644 tests/testthat/test_check_result.R delete mode 100644 tests/testthat/test_strict_check.R create mode 100644 tests/testthat/test_test_result.R diff --git a/.Rbuildignore b/.Rbuildignore index 91114bf2..161056f4 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,2 +1,8 @@ ^.*\.Rproj$ ^\.Rproj\.user$ +^inst/tutorials/*/*.html$ +^_pkgdown\.yml$ +^docs$ +^pkgdown$ +^\.travis\.yml$ +^codecov\.yml$ diff --git a/.gitignore b/.gitignore index 1c8b5c67..628ab4a1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ NEXT_STEPS.Rmd tmp */rsconnect/ inst/tutorials/*/rsconnect/ +inst/tutorials/*/*.html +docs/* diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..2a07ab28 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,51 @@ +# R for travis: see documentation at https://docs.travis-ci.com/user/languages/r + +language: r +sudo: false +cache: packages + +notifications: + email: + on_success: change + on_failure: change + +jobs: + include: + - stage: "R CMD check" + name: "Old Release" + r: oldrel + + + - name: "Release" + r: release + after_success: + # code coverage + - Rscript -e 'if(!require("covr")) install.packages("covr")' + - Rscript -e 'covr::codecov()' + + + - name: "Devel" + r: devel + + + - stage: "Deploy" + name: "pkgdown" + r: release + if: | + branch IN (master, travis, grade_exercise_answer) AND \ + type = push AND \ + repo = rstudio-education/grader + before_deploy: # build site and copy over files + - Rscript -e 'if(!require("pkgdown")) install.packages("pkgdown")' + - Rscript -e 'pkgdown::build_site()' + deploy: + provider: pages # github pages + local-dir: docs # deploy from docs folder + skip-cleanup: true # keep pkgdown output + keep-history: false # overwrite all git history + on: + # deploy on any branch as condition is done above + all_branches: true + github-token: + secure: + "BuNSl2Tp1GGS2ikn0zTzqUXu9bC6KaQBOYV6G8y8msidJAj/1M1sjhLO8BmP4moQ7XsoxAG1Zd+pvuE+vWVIQ/A7oMZU/WXQiymLtUXajSM4lj6+RwK9OS4/BiXia0B1rEUUmUJbX4C+yRBKHU9prNcuyZhG7RO6DTJPwodspL2ABPf4yap1Dfptj3O0ZheziC4NqPqndSGdjZos6sgvIGjEUNfjXQmBv9irpSfpZmzMzWNQvnkF9aCnsgxNcjLwClY73EfJTq3z9FN9EGpk8gRw+M2mhIKwT3N2P6XB4BCHtlIgXCw2TwwJpmuj1qLm2JvUdWx3LfBidxcv5pd9PKQffoYkdaOsX00kImEZtxH4xMCAzsKALLa4B8JyMls7lwEZsEAyU6JYDBraWIXM7s1GOu2lTcqQBpRk6bFU46FGaFfNglPgYbynP3PwGo12W+UsXK6LvuvnbfQKsNzXE2t7WEuMDOQraBU3NoZSQUwX37bAL28Iq5Vyx6XpgwHcayY+vRkNTZXGd7HK/refdhecpAAWi5SRx8I2ExEZ+9qFpw0KijWA2yJ/ksCfSX85EcIZka8PfvSgYYZWzcnD1iuHB+sLb6074h5ePJ+YIQRjZJAeXrOiyHHvmXRfKXUvPymX99VkSiqbVEFMfIendeMw6AcYynP+6WI+pqRf6EQ=" diff --git a/DESCRIPTION b/DESCRIPTION index 865f224c..62c6b51e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,15 +2,19 @@ Package: grader Type: Package Title: Tools for "grading" student exercises in learnr tutorials Version: 0.1.0 -Author: Garrett Grolemund -Maintainer: Garrett Grolemund -Description: Learn more about learnr tutorials at rstudio.github.io/learnr/ -License: MIT +Author@R: c( + person("Garrett", "Grolemund", role = c("aut", "cre"), email = "garrett@rstudio.com"), + person("Barret", "Schloerke", role = c("aut"), email = "barret@rstudio.com"), + person(family = "RStudio, Inc.", role = c("cph", "fnd")), + person(family = "Ajax.org B.V.", role=c("ctb", "cph"), comment= "Ace library") + ) +Description: + Pairing with the 'learnr' R package, 'grader' provides multiple methods to grade 'learnr' exercises. To learn more about 'learnr' tutorials, please visit . +License: GPL-3 Encoding: UTF-8 LazyData: true -Suggests: testthat -RoxygenNote: 6.0.1 -Imports: +RoxygenNote: 6.1.1 +Imports: purrr, crayon, clisymbols, @@ -20,6 +24,12 @@ Imports: glue, knitr, callr, - learnr, + learnr (>= 0.9.2.9001), stringr, - readr + readr, + checkmate, + testthat +Remotes: + rstudio/learnr +URL: https://rstudio-education.github.io/grader/, https://rstudio.github.io/learnr/, https://github.com/rstudio-education/grader +BugReports: https://github.com/rstudio-education/grader/issues diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 5969605d..00000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2018 rstudio-education - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/NAMESPACE b/NAMESPACE index 608e46af..14f6f751 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,8 +1,24 @@ # Generated by roxygen2: do not edit by hand export(add_tutorial) +export(check_code) +export(check_result) +export(correct) +export(get_code) +export(get_solution_code) +export(get_user_code) export(grade_learnr) +export(graded) export(grading_demo) -export(strict_check) +export(incorrect) +export(is_rrect) +export(random_encourage) +export(random_praise) +export(result) +export(test) +export(test_result) export(test_solutions) +export(tests) export(view_tutorial) +importFrom(rlang,"%||%") +importFrom(utils,browseURL) diff --git a/R/check_code.R b/R/check_code.R new file mode 100644 index 00000000..99ab0cbe --- /dev/null +++ b/R/check_code.R @@ -0,0 +1,138 @@ +#' Check code structure +#' +#' \code{check_code()} compares user code to a solution (i.e. model code) and +#' describes the first way that the user code differs. If the user code exactly +#' matches the solution, \code{check_code()} returns a customizable success +#' message. +#' +#' \code{check_code()} provides a *strict* check in that the user code must +#' exactly match the solution. It is not enough for the user code to be +#' equivalent to the solution code (e.g. to return the same result as the +#' solution). +#' +#' You can provide solution code for \code{check_code()} to use in two ways: +#' +#' 1. Pass code as a character string or a quoted expression to the solution +#' argument of \code{check_code()} +#' +#' 2. Make a "-solution" code chunk for the exercise to be checked in a learnr +#' document. There is no need to supply a solution argument for +#' \code{check_code()} if you call it from the "-check" chunk of the same +#' exercise. Likewise, there is no need to supply a user argument when you call +#' \code{check_code()} from a learnr document (learnr will provide the code +#' that the student submits when it runs \code{check_code()}. +#' +#' For best results, name all arguments provided in the solution code. +#' +#' @param correct A character string to display if the student answer matches +#' the solution code. +#' This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = TRUE, message = NULL)}. +#' @param incorrect A character string to display if the student answer matches +#' the solution code. +#' This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = FALSE, message = "")} where message is the error found while comparing the user solution to the known solution. +#' @param solution (Optional) solution code surrounded by \code{quote()}, +#' \code{rlang::quo()}, or provided as a character string. +#' @param user (Optional) student code to check against the solution surrounded +#' by \code{quote()}, \code{rlang::quo()}, or provided as a character string. +#' +#' @return a \code{grader_result} structure from \code{\link{result}}. +#' If the student answer differs from the +#' solution code, the message will describe the first way that the answer +#' differs, and it will ask the student to try again. If the answer matches +#' the solution code, the message will be the content of the \code{success} +#' argument. +#' +#' @export +#' @examples +#' \dontrun{grading_demo()} +check_code <- function( + correct = "{random_praise()} Correct!", + incorrect = "{message} {random_encourage()}", + grader_args = list(), # provided by `grade_learnr` + learnr_args = list() # provided by `grade_learnr` +) { + chkm8_single_character(correct) + chkm8_single_character(incorrect) + + user <- grader_args$user_quo + solution <- grader_args$solution_quo + + is_same_info <- code_is_same(user, solution) + + if (is_same_info$correct) { + return( + graded( + correct = TRUE, + message = glue::glue_data( + list( + correct = TRUE, + message = NULL + ), + correct + ) + ) + ) + } + + message <- glue::glue_data( + list( + correct = FALSE, + message = is_same_info$message + ), + incorrect + ) + if (uses_pipe(user)) { + message <- glue::glue_data( + list( + user = user, + message = message + ), + "I see that you are using pipe operators (e.g. %>%), ", + "so I want to let you know that this is how I am interpretting your code ", + "before I check it:\n\n{deparse(unpipe_all(user))}\n\n{message}" + ) + } + + return( + graded( + correct = FALSE, + message = message + ) + ) +} + + + +code_is_same <- function(user = NULL, solution = NULL) { + + # Sometimes no solution is provided, but that + # means there is nothing to check against + if (is.null(solution)) { + stop("No solution is provided for this exercise.") + } + + # Sometimes no user code is provided, + # that means there is nothing to check + if (is.null(user)) { + stop("I didn't receive your code. Did you write any?") + } + + # MUST call user first to avoid "poisoning" the envir with solution information + user_code <- rlang::get_expr(user) + solution_code <- rlang::get_expr(solution) + + # Correct answers are all alike + if (suppressWarnings(user_code == solution_code)) { + return(graded(correct = TRUE, message = NULL)) + } + + message <- detect_mistakes(user, solution) + if (is.null(message)) { + # found no errors + return(graded(correct = TRUE, message = NULL)) + } + + return( + graded(correct = FALSE, message = message) + ) +} diff --git a/R/check_result.R b/R/check_result.R new file mode 100644 index 00000000..c4d33407 --- /dev/null +++ b/R/check_result.R @@ -0,0 +1,66 @@ + +#' Check result of exercise code +#' +#' \code{check_result()} compares the final result of the user code to known \code{results}. +#' If the user result exactly matches a known \code{result}, \code{check_result} +#' returns the matching message value. +#' +#' @param results A \code{\link{results}} object that contains possible \code{\link{result}} values to compare against. +#' @param correct A character string to display if the student answer matches +#' a known answer. +#' This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = TRUE, message = "")}. where message is the matched result message. +#' @param incorrect A character string to display if the student answer matches +#' a known answer. +#' This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = FALSE, message = "")}. where message is the matched result message. +#' @param empty_msg A character string to display as a message if the user code is NULL. +#' @param ... ignored +#' @param user (Optional) student code to check against the \code{results} surrounded +#' by \code{quote()}, \code{rlang::quo()}, or provided as a character string. +#' +#' @return a \code{grader_result} structure from \code{\link{result}} containing a formatted \code{correct} or \code{incorrect} message. +#' +#' @export +#' @examples +#' \dontrun{grading_demo()} +check_result <- function( + ..., + correct = "{paste0(random_praise(), if (nchar(message) > 0) \" \", message)}", + incorrect = "{paste0(message, if(nchar(message) > 0) \" \", random_encourage())}", + empty_msg = "I did not notice a result. Does your code return one?", + grader_args = list(), # provided by `grade_learnr` + learnr_args = list() # provided by `grade_learnr` +) { + results <- list(...) + chkm8_item_class(results, "grader_result") + chkm8_single_character(correct) + chkm8_single_character(incorrect) + chkm8_single_character(empty_msg) + + if (!any(vapply(results, `[[`, logical(1), "correct"))) { + stop("At least one correct result must be provided") + } + + user_answer <- learnr_args$last_value + + # init final answer as not found + final_result <- graded(correct = FALSE, "Answer not found") + for (resu in results) { + if (identical(resu$x, user_answer)) { + final_result <- resu + break + } + } + + message <- glue::glue_data( + list( + correct = final_result$correct, + message = final_result$message + ), + {if (final_result$correct) correct else incorrect} + ) + + return(graded( + correct = final_result$correct, + message = message + )) +} diff --git a/R/checkmate.R b/R/checkmate.R new file mode 100644 index 00000000..0680841a --- /dev/null +++ b/R/checkmate.R @@ -0,0 +1,11 @@ + +chkm8_class <- checkmate::assert_class +chkm8_item_class <- function(x, class, name = checkmate::vname(x)) { + lapply(seq_along(x), function(i) { + chkm8_class(x[[i]], class, .var.name = paste0(name, "[[", i, "]]")) + }) +} + +chkm8_single_character <- function(x, name = checkmate::vname(x)) { + checkmate::assert_character(x, null.ok = TRUE, len = 1, any.missing = FALSE, .var.name = name) +} diff --git a/R/correct.R b/R/correct.R new file mode 100644 index 00000000..bcdd5cd9 --- /dev/null +++ b/R/correct.R @@ -0,0 +1,30 @@ +# TODO determine if API should use `graded(correct, message)` or `correct(message)`/`incorrect(message)` +# TODO document +#' Correct or Incorrect result +#' @export +#' @rdname correct +correct <- function(message = NULL) { + structure( + list(message = message), + class = c("grader_correct", "grader_rrect") + ) +} +#' @export +#' @rdname correct +incorrect <- function(message = NULL) { + structure( + list(message = message), + class = c("grader_incorrect", "grader_rrect") + ) +} + + +#' @export +#' @rdname correct +is_rrect <- function(x) { + inherits(x, "grader_rrect") +} + + +correct_fn <- correct +incorrect_fn <- incorrect diff --git a/R/detect_mistakes.R b/R/detect_mistakes.R index fb7d273e..80a5e759 100644 --- a/R/detect_mistakes.R +++ b/R/detect_mistakes.R @@ -1,33 +1,34 @@ detect_mistakes <- function(user, - solution) { - - # code should be checked in the opposite order - # of evaluation (e.g. from the outside in for - # nested notation), whether or not the student + solution, env = parent.frame()) { + force(env) + + # code should be checked in the opposite order + # of evaluation (e.g. from the outside in for + # nested notation), whether or not the student # (and/or teacher) used a pipe - user <- rev(order_calls(unpipe_all(user))) - solution <- rev(order_calls(unpipe_all(solution))) - + user <- rev(order_calls(unpipe_all(user), env = env)) + solution <- rev(order_calls(unpipe_all(solution), env = env)) + max_length <- max(length(user), length(solution)) - + for (i in seq_len(max_length)) { - + # Did the user miss something? if (i > length(user)) { - return(missing_argument(this = user[[i-1]], - that = solution[[i]], + return(missing_argument(this_call = user[[i-1]], + that = solution[[i]], that_name = names(solution[i]))) } - + # Did the user write too much? if (i > length(solution)) { - return(surplus_argument(this_call = user[[i - 1]][1], + return(surplus_argument(this_call = user[[i - 1]][1], this_name = names(user[i]), - this = ifelse(is.call(user[[i]]), - renest(user[i:length(user)]), + this = ifelse(is.call(user[[i]]), + renest(user[i:length(user)]), user[[i]]))) } - + # Does the user code not match the solution code? if (length(user[[i]]) != length(solution[[i]]) || user[[i]] != solution[[i]]) { @@ -36,82 +37,82 @@ detect_mistakes <- function(user, } NULL } - + isolate_mismatch <- function(user, solution, i) { - - # We've honed in on the error when we can narrow - # it down to a single incorrect user element + + # We've honed in on the error when we can narrow + # it down to a single incorrect user element # matched to a single correct solution element if (length(user[[i]]) == 1 && length(solution[[i]]) == 1) { - + wrong <- prep_snippet(user, i) right <- prep_snippet(solution, i, .solution = TRUE) - - return(wrong_value(this = wrong, + + return(wrong_value(this = wrong, that = right, this_name = names(user[i]), that_name = names(solution[i]))) - # If we cannot do this, we are working with two - # multipart calls and we need to identify which - # elements of the calls do not align (here we - # rely heavily on the fact that both calls have + # If we cannot do this, we are working with two + # multipart calls and we need to identify which + # elements of the calls do not align (here we + # rely heavily on the fact that both calls have # been previously standardized) } else { - + user_call <- user[[i]] solution_call <- solution[[i]] - - # First check that the calls match. + + # First check that the calls match. if (user_call[[1]] != solution_call[[1]]) { wrong <- renest(user[i:length(user)]) right <- ifelse(is_infix(solution_call[1]), renest(solution[i:length(solution)]), solution_call[1]) - - return(wrong_value(this = wrong, + + return(wrong_value(this = wrong, that = right, this_name = names(user[i]), that_name = names(solution[i]))) } - + # Then inspect the first argument (which appears on the next line) if (length(user) == i && length(solution) > i) { - return(missing_argument(this_call = user[[i]], - that = solution[[i + 1]], + return(missing_argument(this_call = user[[i]], + that = solution[[i + 1]], that_name = names(solution[i + 1]))) - + } else if (length(user) > i && length(solution) == i) { - return(surplus_argument(this_call = user[[i]][1], + return(surplus_argument(this_call = user[[i]][1], this_name = names(user[i + 1]), - this = ifelse(is.call(user[[i + 1]]), - renest(user[(i + 1):length(user)]), + this = ifelse(is.call(user[[i + 1]]), + renest(user[(i + 1):length(user)]), user[[i + 1]]))) - + } else if (length(user) > i && length(solution) > i) { if (!names_match(user, solution, i + 1)) { return(wrong_value(this = user[[i + 1]], this_name = names(user[i + 1]), that = solution[[i + 1]], that_name = names(solution[i + 1]))) - + } else if (user[[i + 1]] != solution[[i + 1]]) { if (is_infix(user_call[1])) { return(detect_mistakes(user_call, solution_call)) } else { - return(detect_mistakes(renest(user[(i + 1):length(user)]), + return(detect_mistakes(renest(user[(i + 1):length(user)]), renest(solution[(i + 1):length(solution)]))) - } + } } } - + # Then iterate through the remaining arguments one at a time - # here we are iterating across a call instead of down a call + # here we are iterating across a call instead of down a call # stack as in detect_mistakes - + max_length <- max(length(user_call), length(solution_call)) - + for (j in seq_len(max_length)) { if (j == 1) next @@ -128,7 +129,7 @@ isolate_mismatch <- function(user, solution, i) { this = user_call[[j]], this_name = names(user_call[j]))) } - + # Do the argument names match? if (!names_match(user_call, solution_call, j)) { return(wrong_value(this = user_call[[j]], @@ -153,17 +154,17 @@ named <- function(vec) { } real_name <- function(name) { - !is.null(name) && name != "" + !is.null(name) && name != "" } names_match <- function(user, solution, i) { - + if (is.null(names(user))) return(TRUE) user_names <- names(user) - + if (is.null(names(solution))) return(TRUE) solution_names <- names(solution) - + if (real_name(user_names[i])) { if (real_name(solution_names[i])) { if (user_names[i] != solution_names[i]) { @@ -180,19 +181,18 @@ names_match <- function(user, solution, i) { } prep_snippet <- function(code, i, .solution = FALSE) { - - # errors that involve an infix operator make more + + # errors that involve an infix operator make more # sense if the explanation refers to the operator if (i == 2 && is_infix(code[[1]]) && length(code[[1]]) == 1) { paste(deparse(code[[1]][[1]]), deparse(code[[2]])) - - # Return the internal arguments of user code, but + + # Return the internal arguments of user code, but # not solution code (that could give away too much) } else if (is.call(code[[i]])) { ifelse(.solution, code[[i]][1], renest(code[i:length(code)])) } else { code[[i]] } - -} +} diff --git a/R/grade_learnr.R b/R/grade_learnr.R index 6f736d20..92f7d83e 100644 --- a/R/grade_learnr.R +++ b/R/grade_learnr.R @@ -5,12 +5,12 @@ #' For exercise checking, learnr tutorials require a function that learnr can #' use in the background to run the code in each "-check" chunk and to format #' the results into a format that learnr can display. The function must accept a -#' specific set of inputs and return a specific type of output. Users are not -#' intended to use the function themselves, but to pass it to the +#' specific set of inputs and return a specific type of output (see \code{\link{result}}). +#' Users are not intended to use the function themselves, but to pass it to the #' \code{exercise.checker} knitr chunk option within the setup chunk of the #' tutorial. #' -#' The grader package provides \code{grade_learnr()} for this purpose. To enable +#' The grader package provides \code{grade_learnr} for this purpose. To enable #' exercise checking in your learnr tutorial, set #' \code{tutorial_options(exercise.checker = grade_learnr)} in the setup chunk #' of your tutorial. @@ -20,11 +20,13 @@ #' #' @param label Label for exercise chunk #' @param solution_code R code submitted by the user -#' @param user_code Code provided within the “-solution” chunk for the exercise. -#' @param check_code Code provided within the “-check” chunk for the exercise. -#' @param envir_result The R environment after the execution of the chunk. +#' @param user_code Code provided within the “-solution” chunk for the exercise. +#' @param check_code Code provided within the “-check” chunk for the exercise. +#' @param envir_result The R environment after the execution of the chunk. #' @param evaluate_result The return value from the \code{evaluate::evaluate} function. -#' @param ... Unused (include for compatibility with parameters to be added in the future) +#' @param envir_prep A copy of the R environment before the execution of the chunk. +#' @param last_value The last value from evaluating the exercise. +#' @param ... Extra arguments supplied by learnr #' #' @return An R list which contains several fields indicating the result of the check. #' @export @@ -37,60 +39,9 @@ grade_learnr <- function(label = NULL, check_code = NULL, envir_result = NULL, evaluate_result = NULL, + envir_prep = NULL, + last_value = NULL, ...) { - - # Praise messages - .praise <- c("Absolutely fabulous!", - "Amazing!", - "Awesome!", - "Beautiful!", - "Bravo!", - "Cool job!", - "Delightful!", - "Excellent!", - "Fantastic!", - "Great work!", - "I couldn't have done it better myself.", - "Impressive work!", - "Lovely job!", - "Magnificent!", - "Nice job!", - "Out of this world!", - "Resplendent!", - "Smashing!", - "Someone knows what they're doing :)", - "Spectacular job!", - "Splendid!", - "Success!", - "Super job!", - "Superb work!", - "Swell job!", - "Terrific!", - "That's a first-class answer!", - "That's glorious!", - "That's marvelous!", - "Very good!", - "Well done!", - "What first-rate work!", - "Wicked smaht!", - "Wonderful!", - "You aced it!", - "You rock!", - "You should be proud.", - ":)") - - # Encouragement messages - .encourage <- c("Please try again.", - "Give it another try.", - "Let's try it again.", - "Try it again; next time's the charm!", - "Don't give up now, try it one more time.", - "But no need to fret, try it again.", - "Try it again. I have a good feeling about this.", - "Try it again. You get better each time.", - "Try it again. Perseverence is the key to success.", - "That's okay: you learn more from mistakes than successes. Let's do it one more time.") - # Sometimes no user code is provided, but # that means there is nothing to check. Also, @@ -117,14 +68,7 @@ grade_learnr <- function(label = NULL, # Sometimes no solution is provided, but that # means there is nothing to check against. Also, # you do not want to parse NULL - if (is.null(solution_code)) { - return(list( - message = "No solution is provided for this exercise.", - correct = TRUE, - type = "info", - location = "append" - )) - } else { + if (!is.null(solution_code)) { solution_code <- parse(text = solution_code) if (length(solution_code) == 0) { return(list( @@ -136,29 +80,114 @@ grade_learnr <- function(label = NULL, } } - # Run checking code to get feedback - grading_code <- pryr::standardise_call(parse(text = check_code)[[1]]) - grading_code$user <- rlang::as_quosure(user_code[[length(user_code)]]) - grading_code$solution <- rlang::as_quosure(solution_code[[length(solution_code)]]) + had_error_checking <- FALSE + checked_result <- tryCatch( + { + # Run checking code to get feedback + parsed_check_code <- parse(text = check_code) + if (length(parsed_check_code) > 1) { + # don't eval the last one to avoid bad check calls + for (i in 1:(length(parsed_check_code) - 1)) { + eval(parsed_check_code[[i]], envir_prep) + } + } + grading_code <- pryr::standardise_call(parsed_check_code[[length(parsed_check_code)]], envir_prep) - feedback <- eval(grading_code) + ## TODO - barret try to no force check fn to be last part of code - # Check that the student submission was correct - if (feedback == grading_code$success) { - result <- list( - message = paste(sample(.praise, 1), feedback), - correct = TRUE, - type = "success", - location = "append" - ) - } else { - result <- list( - message = paste(feedback, sample(.encourage, 1)), - correct = FALSE, - type = "error", - location = "append" - ) + # # set args to . args for the environment + # envir_prep$.grader_args <- grader_args + # envir_prep$.learnr_args <- learnr_args + + # get all grader_args + grader_args <- list( + user_quo = rlang::as_quosure(user_code[[length(user_code)]], envir_result) + ) + + if (!is.null(solution_code)) { + grader_args$solution_quo <- rlang::as_quosure(solution_code[[length(solution_code)]], envir_prep) + } + + # copy in all learnr arguments + learnr_args <- list(...) + learnr_args$label <- label + learnr_args$solution_code <- solution_code + learnr_args$user_code <- user_code + learnr_args$check_code <- check_code + learnr_args$envir_result <- envir_result + learnr_args$evaluate_result <- evaluate_result + learnr_args$envir_prep <- envir_prep + learnr_args$last_value <- last_value + + # copy in all grader arguments + grading_code$grader_args <- grader_args + grading_code$learnr_args <- learnr_args + + # set user answer for the environment to find + envir_prep$ans <- grader_args$user + + # eval code in a copy of the chunk's prepped environment + eval(grading_code, envir_prep) + }, + error = function(e) { + # prevent the error from being re-thrown + message("", e) + had_error_checking <<- TRUE + graded( + correct = FALSE, + message = "Error occured while checking the submission" + ) + } + ) + if (!checkmate::test_class(checked_result, "grader_result")) { + stop("`grade_learnr` should receive a `graded` value from every `-check` chunk") } - result + + message_type <- + if (had_error_checking) { + "warning" + } else { + if (checked_result$correct) { + "success" + } else { + "error" + } + } + + ret <- list( + message = checked_result$message, + correct = checked_result$correct, + type = message_type, + location = "append" + ) + + ret } + + +#' Get Code +#' +#' Helper methods around \code{rlang::\link[rlang]{eval_tidy}} to extract user code and solution code. +#' @seealso \code{\link{check_result}}, \code{\link{test_result}}, and \code{\link{check_code}} +#' @export +#' @rdname get_code +#' @param user,solution,expr An expression or quosure to evaluate. +#' @param name Name to print if a \code{NULL} expression is provided. +#' @inheritParams rlang::eval_tidy +get_user_code <- function(user = NULL, env = rlang::caller_env()) { + get_code(user, "user", env = env) +} +#' @export +#' @rdname get_code +get_solution_code <- function(solution = NULL, env = rlang::caller_env()) { + get_code(solution, "solution", env = env) +} +#' @export +#' @rdname get_code +get_code <- function(expr = NULL, name = "", env = rlang::caller_env()) { + if (is.null(expr)) { + stop("'", name, "' not provided") + } + rlang::eval_tidy(expr, env = env) +} diff --git a/R/grading_demo.R b/R/grading_demo.R index 149f4271..23f7628f 100644 --- a/R/grading_demo.R +++ b/R/grading_demo.R @@ -2,19 +2,24 @@ #' #' If you are using the RStudio IDE, \code{grading_demo()} opens an example learnr #' file that demonstrates how to use the grader package to check student code. -#' +#' #' The tutorial sets the learnr exercise. checker option to -#' \code{grade_learnr()} in the document's setup chunk. It then uses -#' \code{strict_check()} to check the single exercise in the tutorial. To use -#' \code{strict_check()}, follow the exercise chunk with a chunk whose label -#' matches the label of the exercise chunk but includes the suffix -#' \code{-check}. Call \code{strict_check()} in that chunk. To ensure that -#' \code{strict_check()} can provide formative feedback, also provide the -#' solution code in a chunk suffixed \code{-solution}. -#' +#' \code{grade_learnr()} in the document's setup chunk. +#' It then uses three different exercise checking methods: \code{\link{check_result}}, \code{\link{test_result}}, and \code{\link{check_code}}. +#' To use a checking method, follow the exercise chunk with a chunk whose label +#' matches the label of the exercise chunk (ex: \code{myexercise}) but includes the suffix +#' \code{-check} (ex: \code{myexercise-check}). Call any checking method in that chunk. To ensure that +#' checking method can provide informative feedback, you may provide custom \code{correct} and \code{incorrect} messages. +#' #' If you are not using RStudio IDE, you can access the demo file at \code{system.file("extdata", "grading-demo/grading-demo.Rmd", package = "grader")}. #' #' @export +#' @importFrom utils browseURL grading_demo <- function() { - rstudioapi::navigateToFile(system.file("tutorials", "grading-demo/grading-demo.Rmd", package = "grader")) -} \ No newline at end of file + grading_demo_path <- system.file("tutorials", "grading-demo/grading-demo.Rmd", package = "grader") + if (rstudioapi::isAvailable()) { + rstudioapi::navigateToFile(grading_demo_path) + } else { + browseURL(paste0('file://', grading_demo_path)) + } +} diff --git a/R/message_generators.R b/R/message_generators.R index febfec17..eaafb7d0 100644 --- a/R/message_generators.R +++ b/R/message_generators.R @@ -7,13 +7,13 @@ missing_argument <- function(this_call, that, that_name = NULL) { this_call <- prep(this_call) that <- prep(that) - - if (!is.null(that_name) && that_name != "") + + if (!is.null(that_name) && that_name != "") that <- paste(that_name, "=", that) - - if (grepl("\\(\\)", that)) + + if (grepl("\\(\\)", that)) that <- paste("a call to", that) - + glue::glue("Your call to {this_call} should include {that} ", "as one of its arguments. You may have ", "referred to it in a different way, or left out an important ", @@ -24,34 +24,43 @@ missing_argument <- function(this_call, that, that_name = NULL) { surplus_argument <- function(this_call, this, this_name = NULL) { this_call <- prep(this_call) this <- prep(this) - if (!is.null(this_name) && this_name != "") + if (!is.null(this_name) && this_name != "") this <- paste(this_name, "=", this) - - glue::glue("I did not expect your call to {this_call} to ", - "include {this}. You ", - "may have included an unnecessary argument, or you ", - "may have left out or misspelled an important ", - "argument name.") + + glue::glue_data( + list( + this = this, + this_call = this_call + ), + "I did not expect your call to {this_call} to ", + "include {this}. You ", + "may have included an unnecessary argument, or you ", + "may have left out or misspelled an important ", + "argument name." + ) } # wrong value wrong_value <- function(this, that, that_name = NULL, this_name = NULL) { this <- prep(this) that <- prep(that) - - if (!is.null(that_name) && that_name != "") + + if (!is.null(that_name) && that_name != "") that <- paste(that_name, "=", that) - if (!is.null(this_name) && this_name != "") + if (!is.null(this_name) && this_name != "") this <- paste(this_name, "=", this) - - if (grepl("\\(\\)", that)) + + if (grepl("\\(\\)", that)) that <- paste("a call to", that) - - glue::glue("I expected {that} where you wrote {this}.") + + glue::glue_data( + list(this = this, that = that), + "I expected {that} where you wrote {this}." + ) } prep <- function(text) { if (is.call(text)) text <- text[1] if (!is.character(text)) text <- deparse(text) text -} \ No newline at end of file +} diff --git a/R/order_calls.R b/R/order_calls.R index 2ca79f16..c390d5e2 100644 --- a/R/order_calls.R +++ b/R/order_calls.R @@ -1,19 +1,20 @@ #' Order calls #' #' Turns a quoted object into a list of symbols that would represent the call as -#' a pipe if you placed a \code{%>%} between each element of the list. This +#' a pipe if you placed a \code{\%>\%} between each element of the list. This #' let's checking code evaluate the elements in the same order that R would. #' -order_calls <- function(code) { +#' @noRd +order_calls <- function(code, env = parent.frame()) { if (is.name(code) || is.call(code) || is.atomic(code)) { code <- list(code) } if (is.call(code[[1]]) && length(code[[1]]) != 1) { - code[[1]] <- standardize_call(code[[1]]) + code[[1]] <- standardize_call(code[[1]], env = env) code <- c(pre_pipe(code[[1]], name = names(code[1])), code[-1]) - code <- order_calls(code) + code <- order_calls(code, env = env) } code <- purrr::discard(code, is.null) purrr::map(code, remove_null_from_call) @@ -22,13 +23,13 @@ order_calls <- function(code) { pre_pipe <- function(code, name = "") { if (is.call(code)) { new <- list(code[[2]], code[-2]) - + name2 <- names(code)[[2]] if (is.null(name2) || name2 == "") arg_name <- "" else arg_name <- name2 if (is.null(name)) name <- "" names(new) <- c(arg_name, name) - + new } else { code @@ -39,21 +40,21 @@ repipe <- function(lst, .call = FALSE) { text <- purrr::map(lst, deparse) text <- purrr::reduce(text, paste, sep = " %>% ") text <- gsub("\\(NULL\\)", "()", text) - + if(.call) parse(text = text)[[1]] else text } renest <- function(lst, .call = FALSE) { lst <- rev(lst) - + nest <- function(b, a) { if (is.call(a)) { if (length(a) > 1) { - - # double check that the function does + + # double check that the function does # not contain a placeholder NULL argument - if (length(a) != 2 || !is.null(a[[2]])) { + if (length(a) != 2 || !is.null(a[[2]])) { if (!is.null(names(a))) { names_a <- names(a) names_a <- c(names(a)[1], "", names(a)[2:length(a)]) @@ -63,7 +64,7 @@ renest <- function(lst, .call = FALSE) { a[3:(length(a) + 1)] <- a[2:length(a)] } } - } + } a[[2]] <- b } a @@ -75,18 +76,18 @@ renest <- function(lst, .call = FALSE) { } # Modified from pryr::standardise_call -# Returns a version of the call that has -# arguments in a standard order and +# Returns a version of the call that has +# arguments in a standard order and # argument names supplied for each argument after the first standardize_call <- function(code, env = parent.frame()) { stopifnot(is.call(code)) f <- eval(code[[1]], env) if (!is.null(args(f))) { call <- match.call(args(f), code) - - # because checking code should follow practice - # of not naming the first argument (unless the - # user deliberately does so) and not naming the + + # because checking code should follow practice + # of not naming the first argument (unless the + # user deliberately does so) and not naming the # arguments of infix operators first_arg <- names(as.list(args(f)))[1] if (is_infix(code)) { diff --git a/R/praise.R b/R/praise.R new file mode 100644 index 00000000..16088d38 --- /dev/null +++ b/R/praise.R @@ -0,0 +1,69 @@ +# Praise messages +.praise <- c("Absolutely fabulous!", + "Amazing!", + "Awesome!", + "Beautiful!", + "Bravo!", + "Cool job!", + "Delightful!", + "Excellent!", + "Fantastic!", + "Great work!", + "I couldn't have done it better myself.", + "Impressive work!", + "Lovely job!", + "Magnificent!", + "Nice job!", + "Out of this world!", + "Resplendent!", + "Smashing!", + "Someone knows what they're doing :)", + "Spectacular job!", + "Splendid!", + "Success!", + "Super job!", + "Superb work!", + "Swell job!", + "Terrific!", + "That's a first-class answer!", + "That's glorious!", + "That's marvelous!", + "Very good!", + "Well done!", + "What first-rate work!", + "Wicked smaht!", + "Wonderful!", + "You aced it!", + "You rock!", + "You should be proud.", + ":)") + +# Encouragement messages +.encourage <- c("Please try again.", + "Give it another try.", + "Let's try it again.", + "Try it again; next time's the charm!", + "Don't give up now, try it one more time.", + "But no need to fret, try it again.", + "Try it again. I have a good feeling about this.", + "Try it again. You get better each time.", + "Try it again. Perseverence is the key to success.", + "That's okay: you learn more from mistakes than successes. Let's do it one more time.") + +#' Random praise and encouragement +#' +#' Generate a random praise or encouragement phrase. This can be used in conjunction with \code{glue::\link[glue]{glue}} to generate praise or encouragement within feedback to users. +#' +#' @examples +#' replicate(5, glue::glue("Random praise: {random_praise()}")) +#' replicate(5, glue::glue("Random encouragement: {random_encourage()}")) +#' @export +#' @rdname praise +random_praise <- function() { + sample(.praise, 1) +} +#' @export +#' @rdname praise +random_encourage <- function() { + sample(.encourage, 1) +} diff --git a/R/result.R b/R/result.R new file mode 100644 index 00000000..7729d5c0 --- /dev/null +++ b/R/result.R @@ -0,0 +1,39 @@ +#' Result +#' +#' The \code{result} method returns an object containing information about what has been graded or what could be graded against. +#' @param x object graded or being compared against +#' @param message possible message value to be displayed +#' @param correct a boolean that determines if the result is a correct result +#' @export +#' @examples +#' result(1, "Custom message for value 1.") +#' result(2, "Custom message for value 2.", correct = TRUE) +#' result(3, "Custom message for value 3.") +#' result(4, "Custom message for value 4.", correct = TRUE) +#' \dontrun{grading_demo()} +result <- function(x, message = NULL, correct = FALSE) { + chkm8_single_character(message) + checkmate::expect_logical(correct, any.missing = FALSE, len = 1, null.ok = FALSE) + + structure(class = "grader_result", list( + x = x, + message = message %||% "", + correct = correct + )) +} + + +# TODO-document +# Should be what EVERY check_* returns +#' @export +graded <- function(message, correct) { + chkm8_single_character(message) + checkmate::expect_logical(correct, any.missing = FALSE, len = 1, null.ok = FALSE) + + structure(class = "grader_graded", + list( + message = message %||% "", + correct = correct + ) + ) +} diff --git a/R/strict_check.R b/R/strict_check.R deleted file mode 100644 index 973e379d..00000000 --- a/R/strict_check.R +++ /dev/null @@ -1,80 +0,0 @@ -#' strict_check -#' -#' Strict exercise checking -#' -#' \code{strict_check()} compares user code to a solution (i.e. model code) and -#' describes the first way that the user code differs. If the user code exactly -#' matches the solution, \code{strict_check()} returns a customizable success -#' message. -#' -#' \code{strict_check()} provides a *strict* check in that the user code must -#' exactly match the solution. It is not enough for the user code to be -#' equivalent to the solution code (e.g. to return the same result as the -#' solution). -#' -#' You can provide solution code for \code{strict_check()} to use in two ways: -#' -#' 1. Pass code as a character string or a quoted expression to the solution -#' argument of \code{strict_check()} -#' -#' 2. Make a "-solution" code chunk for the exercise to be checked in a learnr -#' document. There is no need to supply a solution argument for -#' \code{strict_check()} if you call it from the "-check" chunk of the same -#' exercise. Likewise, there is no need to supply a user argument when you call -#' \code{strict_check()} from a learnr document (learnr will provide the code -#' that the student submits when it runs \code{strict_check()}. -#' -#' For best results, name all arguments provided in the solution code. -#' -#' @param success A character string to display if the student answer matches -#' the solution code -#' @param solution (Optional) solution code surrounded by \code{quote()}, -#' \code{rlang::quo()}, or provided as a character string. -#' @param user (Optional) student code to check against the solution surrounded -#' by \code{quote()}, \code{rlang::quo()}, or provided as a character string. -#' -#' @return (character) A message. If the student answer differs from the -#' solution code, the message will describe the first way that the answer -#' differs, and it will ask the student to try again. If the answer matches -#' the solution code, the message will be the content of the \code{success} -#' argument. -#' -#' @export -#' -#' @examples -#' \dontrun{grading_demo()} -strict_check <- function(success = NULL, - solution = NULL, - user = NULL) { - - # Sometimes no solution is provided, but that - # means there is nothing to check against - if (is.null(solution)) { - stop("No solution is provided for this exercise.") - - # Sometimes no user code is provided, - # that means there is nothing to check - } else if (is.null(user)) { - stop("I didn't receive your code. Did you write any?") - - # Correct answers are all alike - } else if (suppressWarnings(user == solution)) { - return(success) - - # But incorrect answers are each incorrect in their own way - # (and we should let the student know how their answer is - # incorrect) - } else { - message <- detect_mistakes(user, solution) - if (is.null(message)) { - return(success) - } else { - if (uses_pipe(user)) { - message <- glue::glue("I see that you are using pipe operators (e.g. %>%), ", - "so I want to let you know that this is how I am interpretting your code ", - "before I check it:\n\n{deparse(unpipe_all(user))}\n\n{message}") - } - return(message) - } - } -} \ No newline at end of file diff --git a/R/test_result.R b/R/test_result.R new file mode 100644 index 00000000..bb5ece35 --- /dev/null +++ b/R/test_result.R @@ -0,0 +1,151 @@ +#' Test the result of exercise code +#' +#' \code{test_result()} executes tests against the final result of the user code. +#' If a test throws an error, the test fails and the submitted answer will be marked incorrect. +#' +#' @param tests A \code{\link{tests}} object that contains all \code{\link{test}} functions to check the user's result. +#' @param correct A character string to display if all tests pass. +#' This character string will be run through \code{glue::\link[glue]{glue_data}} with +#' \itemize{ +#' \item \code{num_correct}: Number of correct tests. (Equals \code{num_total}) +#' \item \code{num_total}: Number of tests +#' \item \code{errors}: Vector of errors found. (\code{NULL}) +#' } +#' @param incorrect A character string to display if at least one test fails. +#' This character string will be run through \code{glue::\link[glue]{glue_data}} with +#' \itemize{ +#' \item \code{num_correct}: Number of correct tests +#' \item \code{num_total}: Number of tests +#' \item \code{errors}: Vector of errors found +#' } +#' @param empty_msg A character string to display as a message if the user code is NULL. +#' @param ... ignored +#' @param user (Optional) student code to check against the \code{results} surrounded +#' by \code{quote()}, \code{rlang::quo()}, or provided as a character string. +#' +#' @return a \code{grader_result} structure from \code{\link{result}} containing a formatted \code{correct} or \code{incorrect} message. +#' @seealso \code{tests}, \code{test} +#' @export +#' @examples +#' \dontrun{grading_demo()} +test_result <- function( + ..., + correct = "{num_correct}/{num_total} correct! {random_praise()}", + incorrect = paste0( + "{num_correct}/{num_total} correct. ", + "Fix this first: '{errors[1]}'. ", + "{random_encourage()}" + ), + empty_msg = "I did not notice a result. Does your code return one?", + grader_args = list(), # provided by `grade_learnr` + learnr_args = list() # provided by `grade_learnr` +) { + chkm8_item_class(tests, "grader_test") + chkm8_single_character(correct) + chkm8_single_character(incorrect) + chkm8_single_character(empty_msg) + + user_answer <- get_user_code(user) + if (is.null(user_answer)) { + return(graded(correct = FALSE, message = empty_msg)) + } + + results <- lapply(tests$fns, function(test_fn) { + tryCatch( + { + test_fn(user_answer) + graded( + correct = TRUE, + message = NULL + ) + }, + error = function(e) { + graded( + correct = FALSE, + message = as.character(e) + ) + } + ) + }) + + is_corrects <- vapply(results, `[[`, logical(1), "correct") + is_correct <- all(is_corrects) + + message <- glue::glue_data( + list( + is_correct = is_correct, + num_correct = sum(is_corrects), + num_total = length(results), + errors = unlist(lapply(results, function(resu) { + if (!resu$correct) resu$message else NULL + })) + ), + {if (is_correct) correct else incorrect} + ) + + return(graded( + correct = is_correct, + message = message + )) +} + +# TODO do not use anymore in favor of `...` arg +#' Tests to check +#' +#' Collect a set of test to execute against a user's result value +#' @param ... a set of functions that will accept the evaluated user solution. If the test fails, it should throw an error with the message to display. +#' @export +#' @rdname test +#' @examples +#' +#' tests( +#' function(your_answer) { +#' checkmate::expect_function(your_answer, args = c("x")) +#' }, +#' test( +#' # use a custom error message +#' "Make sure your function returns a number!", +#' function(your_answer) { +#' checkmate::expect_number(your_answer(1)) +#' } +#' ), +#' function(your_answer) { +#' testthat::expect_equal(your_answer(0), NaN) +#' }, +#' function(your_answer) { +#' testthat::expect_equal(your_answer(1:10), sqrt(log(1:10))) +#' } +#' ) +#' +#' \dontrun{grading_demo()} +tests <- function(...) { + fns <- list(...) + lapply(fns, function(fn) { + checkmate::assert_function(fn) + if (length(formals(fn)) == 0) { + stop("The function must be able to accept the user submission") + } + }) + structure( + class = "grader_tests", + list( + fns = list(...) + ) + ) +} +#' @export +#' @rdname test +#' @param message Message to report back if the test throws an error. +#' @param fn function to execute against the user solution. If the test fails, it should throw an error to display the \code{message} provided. +test <- function(message, fn) { + function(x) { + tryCatch( + { + fn(x) + }, + error = function(e) { + stop(message, call. = FALSE) + } + ) + } +} diff --git a/R/test_solutions.R b/R/test_solutions.R index 2b8b963d..10fac966 100644 --- a/R/test_solutions.R +++ b/R/test_solutions.R @@ -1,14 +1,20 @@ +yoink <- function(pkg, fn) { + do.call("getFromNamespace", list(fn, pkg)) +} + # get all of the chunks with extract_chunks <- function(file) { + knitr__knit_code <- yoink("knitr", "knit_code") + knitr__split_file <- yoink("knitr", "split_file") knitr::pat_md() knitr::render_markdown() on.exit({ knitr::knit_patterns$restore() knitr::knit_hooks$restore() - knitr:::knit_code$restore() + knitr__knit_code$restore() }, add = TRUE) - knitr:::split_file(readLines(file, encoding = "UTF-8")) - knitr:::knit_code$get() + knitr__split_file(readLines(file, encoding = "UTF-8")) + knitr__knit_code$get() } #' Test Solutions @@ -27,10 +33,10 @@ extract_chunks <- function(file) { #' \code{test_solutions} will test that .Rmd file. #' @param show.answers TRUE or FALSE. Should solution results be printed in the #' output? -#' @param .params A list of parameters to use when evauating code in a -#' parameterized R Markdown document. This should be identical to the list -#' that you would use to render the document. -#' +# ' @param .params A list of parameters to use when evauating code in a +# ' parameterized R Markdown document. This should be identical to the list +# ' that you would use to render the document. +# ' #' @return \code{test_solutions} does not return a value; it prints an #' informative summary of the testing results. Each solution and setup block #' is listed by name alongside a result status. Within the RStudio IDE, chunks @@ -45,7 +51,7 @@ extract_chunks <- function(file) { test_solutions <- function(file = NULL, show.answers = FALSE ) { - + if (is.null(file)) { files <- dir() rmds <- files[grepl("(.Rmd|.rmd)$", files)] @@ -57,25 +63,25 @@ test_solutions <- function(file = NULL, file <- rmds } } - + # the functions that test the solutions are defined here so they can find the # results of the global setup chunk through lexical scoping safe_eval <- purrr::safely(purrr::quietly(eval)) - + safe_test <- function(label, chunks) { safe_eval(parse(text = chunks[[label]]), envir = parent.frame(1)) } - + test_solution <- function(label, chunks, show.answer = FALSE) { exercise <- sub("-solution", "", label) if (grepl("-solution$", label) && !(exercise %in% names(chunks))) { stop(paste(label, "not associated with an exercise chunk."), call. = FALSE) } - + # Does the chunk require a setup chunk? setup_option <- attr(chunks[[exercise]], "chunk_opts")$exercise.setup setup_suffix <- paste0(exercise, "-setup") - + if (!is.null(setup_option)) { if (!is.character(setup_option)) { print_author_error(label, "exercise.setup chunk_opt is not a single string.") @@ -88,44 +94,44 @@ test_solutions <- function(file = NULL, return() } else { setup <- safe_test(setup_option, chunks = chunks) - print_result(setup_option, - setup, + print_result(setup_option, + setup, show.answers = show.answer) } } else if (setup_suffix %in% names(chunks)) { setup <- safe_test(setup_suffix, chunks = chunks) - print_result(setup_suffix, + print_result(setup_suffix, setup, show.answers = show.answer) } - + result <- safe_test(label, chunks = chunks) - print_result(label, + print_result(label, result, show.answers = show.answer) } - + chunks <- extract_chunks(file) labels <- names(chunks) - + # Exercises have access to the computations # performed in the setup chunk if ("setup" %in% labels) { setup_result <- safe_test("setup", chunks = chunks) - print_result("setup", + print_result("setup", setup_result, show.answers = show.answers) } - + solutions <- grep("-solution$", labels, value = TRUE) - purrr::walk(solutions, - test_solution, - chunks = chunks, + purrr::walk(solutions, + test_solution, + chunks = chunks, show.answer = show.answers) } -print_result <- function(label, result, show.answers = FALSE) { +print_result <- function(label, result, show.answers = FALSE) { cat(label, crayon::silver(": "), sep = "") if (!is.null(result$error)) { cat( @@ -152,5 +158,3 @@ print_author_error <- function(label, message){ purple(message), "\n" ) } - - diff --git a/R/utils.R b/R/utils.R index 1d2aad8f..8e6333b2 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,5 +1,8 @@ .pipes <- c("%>%") +#' @importFrom rlang %||% +NULL + is_pipe <- function(x) { if (is.character(x)) x <- parse(text = x)[[1]] ifelse(is.call(x), as.character(x[[1]]) %in% .pipes, FALSE) @@ -10,4 +13,4 @@ is_pipe <- function(x) { is_infix <- function(x) { if (is.character(x)) x <- parse(text = x)[[1]] ifelse(is.call(x), as.character(x[[1]]) %in% .infixes, FALSE) -} \ No newline at end of file +} diff --git a/R/view_tutorial.R b/R/view_tutorial.R index 508c026d..3cf4dfe1 100644 --- a/R/view_tutorial.R +++ b/R/view_tutorial.R @@ -11,6 +11,7 @@ #' @param package A character string. The name of the package that contains a tutorial. #' #' @export +#' @importFrom utils browseURL view_tutorial <- function(name, package) { # launch in separate R session @@ -28,9 +29,9 @@ view_tutorial <- function(name, package) { supervise = TRUE, args = list(name = name, package = package) ) - - # Failed attempt to bypass IDE console scrolling - # error by using terminal instead of callr (does + + # Failed attempt to bypass IDE console scrolling + # error by using terminal instead of callr (does # not avoid the bug) # termID <- rstudioapi::terminalExecute( # 'R -e \'learnr::run_tutorial("solutions-demo", package = "grader", shiny_args = list(launch.browser = FALSE, port = 8000, host = "127.0.0.1"))\'', @@ -51,10 +52,14 @@ view_tutorial <- function(name, package) { } } - rstudioapi::viewer( - url = "http://localhost:8000", - height = "maximize" - ) + if (rstudioapi::isAvailable()) { + rstudioapi::viewer( + url = "http://localhost:8000", + height = "maximize" + ) + } else { + browseURL("http://localhost:8000") + } } @@ -66,7 +71,7 @@ view_tutorial <- function(name, package) { #' \code{remove_tutorial} should be easy to adapt to a an RStudio-specific #' startup file should RStudio implement one. #' -#' \code{add_tutorial()} inserts a call to \link[grader]{\code{view_tutorial()}} +#' \code{add_tutorial()} inserts a call to \code{\link{view_tutorial}()} #' into the .Rprofile file contained in the current working directory. As a #' result, R will launch the tutorial in the RStudio IDE viewer pane whenever #' the current project is opened or re-opened. @@ -80,7 +85,7 @@ view_tutorial <- function(name, package) { #' \code{view_tutorial()}, \code{add_tutorial} will override the call without a #' warning message. #' -#' @seealso \link[grader]{\code{view_tutorial}} +#' @seealso \code{\link{view_tutorial}} #' #' @param name A character string. The name of a tutorial saved in a package. #' @param package A character string. The name of the package that contains a @@ -91,7 +96,7 @@ add_tutorial <- function(name, package) { rprofile <- paste0(getwd(), "/.Rprofile") # load packages - + # Check that the .Rprofile does not already load a tutorial if (file.exists(rprofile)) { text <- readr::read_file(rprofile) @@ -107,29 +112,29 @@ add_tutorial <- function(name, package) { } cat(paste0( 'grader::view_tutorial(name = "', - name, '", package = "', package, - '") ## Learnr tutorial added on ', + name, '", package = "', package, + '") ## Learnr tutorial added on ', Sys.Date() ), file = rprofile, sep = "\n", append = TRUE ) - + # return message } remove_tutorial <- function(dir = NULL) { if (!is.null(dir)) dir <- getwd() rprofile <- paste0(dir, "/.Rprofile") - + if (file.exists(rprofile)) { text <- readr::read_lines(rprofile) } else { stop("Directory does not have a .Rprofile ", "file to remove tutorial from.") } - + tutorial_calls <- grepl("\\) ## Learnr tutorial added on ", text) if (!any(tutorial_calls)) { message("No tutorials detected to remove.") @@ -139,4 +144,3 @@ remove_tutorial <- function(dir = NULL) { message("Tutorial removed.") } } - \ No newline at end of file diff --git a/README.md b/README.md index 6ffb5234..e3f7b1b6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ # grader -Tools for teachers to use with learnr + + +[![Travis build status](https://travis-ci.org/rstudio-education/grader.svg?branch=master)](https://travis-ci.org/rstudio-education/grader) +[![Codecov test coverage](https://codecov.io/gh/rstudio-education/grader/branch/master/graph/badge.svg)](https://codecov.io/gh/rstudio-education/grader?branch=master) +[![CRAN version](http://www.r-pkg.org/badges/version/grader)](https://cran.r-project.org/package=grader) +[![grader downloads per month](http://cranlogs.r-pkg.org/badges/grader)](http://www.rpackages.io/package/grader) +[![DOI](https://zenodo.org/badge/126734088.svg)](https://zenodo.org/badge/latestdoi/126734088) + + +[![RStudio Community: Teaching](https://img.shields.io/badge/Community%20Support-Teaching-75aadb.svg?style=popout&logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDIyLjEuMCwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeD0iMHB4IiB5PSIwcHgiCgkgdmlld0JveD0iMCAwIDYyNS45IDYyNS45IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA2MjUuOSA2MjUuOTsiIHhtbDpzcGFjZT0icHJlc2VydmUiPgo8c3R5bGUgdHlwZT0idGV4dC9jc3MiPgoJLnN0MHtmaWxsOiM3NUFBREI7fQoJLnN0MXtmaWxsOiM0RDRENEQ7fQoJLnN0MntmaWxsOiNGRkZGRkY7fQoJLnN0M3tmaWxsOnVybCgjU1ZHSURfMV8pO30KCS5zdDR7ZmlsbDp1cmwoI1NWR0lEXzJfKTt9Cgkuc3Q1e2ZpbGw6dXJsKCNTVkdJRF8zXyk7fQoJLnN0NntmaWxsOnVybCgjU1ZHSURfNF8pO30KCS5zdDd7ZmlsbDp1cmwoI1NWR0lEXzVfKTt9Cgkuc3Q4e2ZpbGw6dXJsKCNTVkdJRF82Xyk7fQoJLnN0OXtmaWxsOnVybCgjU1ZHSURfN18pO30KCS5zdDEwe2ZpbGw6dXJsKCNTVkdJRF84Xyk7fQoJLnN0MTF7ZmlsbDp1cmwoI1NWR0lEXzlfKTt9Cgkuc3QxMntmaWxsOnVybCgjU1ZHSURfMTBfKTt9Cgkuc3QxM3tvcGFjaXR5OjAuMTg7ZmlsbDp1cmwoI1NWR0lEXzExXyk7fQoJLnN0MTR7b3BhY2l0eTowLjM7fQo8L3N0eWxlPgo8ZyBpZD0iR3JheV9Mb2dvIj4KPC9nPgo8ZyBpZD0iQmxhY2tfTGV0dGVycyI+CjwvZz4KPGcgaWQ9IkJsdWVfR3JhZGllbnRfTGV0dGVycyI+Cgk8Zz4KCgkJCTxlbGxpcHNlIHRyYW5zZm9ybT0ibWF0cml4KDAuNzA3MSAtMC43MDcxIDAuNzA3MSAwLjcwNzEgLTEyNy45MjY1IDMxNy4wMzE3KSIgY2xhc3M9InN0MCIgY3g9IjMxOC43IiBjeT0iMzEyLjkiIHJ4PSIzMDkuOCIgcnk9IjMwOS44Ii8+CgkJPGc+CgkJCTxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik00MjQuNyw0MTEuOGgzMy42djI2LjFoLTUxLjNMMzIyLDMxMC41aC00NS4zdjEwMS4zaDQ0LjN2MjYuMUgyMDkuNXYtMjYuMWgzOC4zVjE4Ny4zbC0zOC4zLTQuN3YtMjQuNwoJCQkJYzE0LjUsMy4zLDI3LjEsNS42LDQyLjksNS42YzIzLjgsMCw0OC4xLTUuNiw3MS45LTUuNmM0Ni4yLDAsODkuMSwyMSw4OS4xLDcyLjNjMCwzOS43LTIzLjgsNjQuOS02MC43LDc1LjZMNDI0LjcsNDExLjh6CgkJCQkgTTI3Ni43LDI4NS4zbDI0LjMsMC41YzU5LjMsMC45LDgyLjEtMjEuOSw4Mi4xLTUyLjNjMC0zNS41LTI1LjctNDkuNS01OC4zLTQ5LjVjLTE1LjQsMC0zMS4zLDEuNC00OC4xLDMuM1YyODUuM3oiLz4KCQk8L2c+Cgk8L2c+CjwvZz4KPGcgaWQ9IldoaXRlX0xldHRlcnMiPgo8L2c+CjxnIGlkPSJSX0JhbGwiPgo8L2c+Cjwvc3ZnPg==)](https://community.rstudio.com/c/teaching) +[![RStudio Ask a question: grader](https://img.shields.io/badge/Ask%20a%20question-grader-75aadb.svg?style=popout&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB2ZXJzaW9uPSIxLjEiIHg9IjBweCIgeT0iMHB4IiB2aWV3Qm94PSIwIDAgNjI1LjkgNjI1LjkiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDYyNS45IDYyNS45OyIgeG1sOnNwYWNlPSJwcmVzZXJ2ZSI+CjxzdHlsZSB0eXBlPSJ0ZXh0L2NzcyI+Cgkuc3Qwe2ZpbGw6Izc1QUFEQjt9Cgkuc3Qxe2ZpbGw6IzRENEQ0RDt9Cgkuc3Qye2ZpbGw6I0ZGRkZGRjt9Cgkuc3Qze2ZpbGw6dXJsKCNTVkdJRF8xXyk7fQoJLnN0NHtmaWxsOnVybCgjU1ZHSURfMl8pO30KCS5zdDV7ZmlsbDp1cmwoI1NWR0lEXzNfKTt9Cgkuc3Q2e2ZpbGw6dXJsKCNTVkdJRF80Xyk7fQoJLnN0N3tmaWxsOnVybCgjU1ZHSURfNV8pO30KCS5zdDh7ZmlsbDp1cmwoI1NWR0lEXzZfKTt9Cgkuc3Q5e2ZpbGw6dXJsKCNTVkdJRF83Xyk7fQoJLnN0MTB7ZmlsbDp1cmwoI1NWR0lEXzhfKTt9Cgkuc3QxMXtmaWxsOnVybCgjU1ZHSURfOV8pO30KCS5zdDEye2ZpbGw6dXJsKCNTVkdJRF8xMF8pO30KCS5zdDEze29wYWNpdHk6MC4xODtmaWxsOnVybCgjU1ZHSURfMTFfKTt9Cgkuc3QxNHtvcGFjaXR5OjAuMzt9Cjwvc3R5bGU+CjxnIGlkPSJHcmF5X0xvZ28iPgo8L2c+CjxnIGlkPSJCbGFja19MZXR0ZXJzIj4KPC9nPgo8ZyBpZD0iQmx1ZV9HcmFkaWVudF9MZXR0ZXJzIj4KCTxnPgoKCQkJPGVsbGlwc2UgdHJhbnNmb3JtPSJtYXRyaXgoMC43MDcxIC0wLjcwNzEgMC43MDcxIDAuNzA3MSAtMTI3LjkyNjUgMzE3LjAzMTcpIiBjbGFzcz0ic3QwIiBjeD0iMzE4LjciIGN5PSIzMTIuOSIgcng9IjMwOS44IiByeT0iMzA5LjgiLz4KCQk8Zz4KCQkJPHBhdGggY2xhc3M9InN0MiIgZD0iTTQyNC43LDQxMS44aDMzLjZ2MjYuMWgtNTEuM0wzMjIsMzEwLjVoLTQ1LjN2MTAxLjNoNDQuM3YyNi4xSDIwOS41di0yNi4xaDM4LjNWMTg3LjNsLTM4LjMtNC43di0yNC43ICAgICBjMTQuNSwzLjMsMjcuMSw1LjYsNDIuOSw1LjZjMjMuOCwwLDQ4LjEtNS42LDcxLjktNS42YzQ2LjIsMCw4OS4xLDIxLDg5LjEsNzIuM2MwLDM5LjctMjMuOCw2NC45LTYwLjcsNzUuNkw0MjQuNyw0MTEuOHogICAgICBNMjc2LjcsMjg1LjNsMjQuMywwLjVjNTkuMywwLjksODIuMS0yMS45LDgyLjEtNTIuM2MwLTM1LjUtMjUuNy00OS41LTU4LjMtNDkuNWMtMTUuNCwwLTMxLjMsMS40LTQ4LjEsMy4zVjI4NS4zeiIvPgoJCTwvZz4KCTwvZz4KPC9nPgo8ZyBpZD0iV2hpdGVfTGV0dGVycyI+CjwvZz4KPGcgaWQ9IlJfQmFsbCI+CjwvZz4KPC9zdmc+)](https://community.rstudio.com/new-topic?title=&category_id=13&tags=grader&body=%0A%0A%0A%20%20--------%0A%20%20%0A%20%20%3Csup%3EReferred%20here%20by%20%60grader%60%27s%20README%3C/sup%3E%0A&u=barret) + +Pairing with the `learnr` R package, `grader` provides multiple methods to grade `learnr` exercises. To learn more about `learnr` tutorials, please visit [https://rstudio.github.io/learnr/](https://rstudio.github.io/learnr/). + +## Installation + +You can install the released version of grader from [CRAN](https://CRAN.R-project.org) with: + +``` r +install.packages("grader") +``` + +## Grading Demo + +To view the latest grading demo of the different checking methods: + +``` r +library(grader) +grader::grading_demo() +``` diff --git a/_pkgdown.yml b/_pkgdown.yml new file mode 100644 index 00000000..6ef5f5a6 --- /dev/null +++ b/_pkgdown.yml @@ -0,0 +1 @@ +destination: docs diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..8f36b6cc --- /dev/null +++ b/codecov.yml @@ -0,0 +1,12 @@ +comment: false + +coverage: + status: + project: + default: + target: auto + threshold: 1% + patch: + default: + target: auto + threshold: 1% diff --git a/inst/tutorials/grading-demo/grading-demo.Rmd b/inst/tutorials/grading-demo/grading-demo.Rmd index 0f99da01..cdb90cee 100644 --- a/inst/tutorials/grading-demo/grading-demo.Rmd +++ b/inst/tutorials/grading-demo/grading-demo.Rmd @@ -2,6 +2,8 @@ title: "Grading demo" output: learnr::tutorial runtime: shiny_prerendered +# editor_options: +# preview: viewer --- ```{r setup, include=FALSE} @@ -13,32 +15,285 @@ knitr::opts_chunk$set(echo = FALSE) ``` -## Topic 1 +## Grading Types -### Exercise +### Check Exercise Result -Here is a number. You can do great things with it, like this: +`grader` can check for the final returned value. This grading approach does not inspect the code. It only inspects the final result. -* Take the square root of the log of the number. Then click Submit Answer. +See `?check_result` for more information. -```{r number, exercise = TRUE} -1 +Example code written below: + +````markdown +`r ''````{r result, exercise = TRUE} +3 +``` + +`r ''````{r result-hint-1} +"Is it an even number" +``` + +`r ''````{r result-hint-2} +"Enter an even number below 5" +``` + +`r ''````{r result-check} +check_result( + results( + result(1, "Custom message for value 1."), + result(2, "Custom message for value 2.", correct = TRUE), + result(3, "Custom message for value 3."), + result(4, "Custom message for value 4.", correct = TRUE) + ) +) +``` +```` + +----------------------- + +* Enter an small even number, then click Submit Answer. + +```{r result, exercise = TRUE} +3 +``` + +```{r result-hint-1} +"Is it an even number" +``` + +```{r result-hint-2} +"Enter an even number below 5" +``` + +```{r result-check} +check_result( + result(1, "Custom message for value 1."), + result(2, "Custom message for value 2.", correct = TRUE), + result(3, "Custom message for value 3."), + result(4, "Custom message for value 4.", correct = TRUE) +) +``` + +### Test Exercise Result + +`grader` can test against for the final returned value. This grading approach does not inspect the code. It only inspects the final result. + +Writers can include as many testing functions as they would like. The test functions should accept a single result value to test against. If any test fails, it should throw an error. This error message will be returned back to the user by default. See `?test_result` for more information. + +Example code written below: + +````markdown +`r ''````{r test_result, exercise = TRUE} +function(x) { + sqrt(log(x)) +} +``` + +`r ''````{r test_result-hint-1} +"function(x) { + sqrt(...) +}" +``` + +`r ''````{r test_result-hint-2} +"function(x) { + sqrt(log(...)) +}" +``` + +`r ''````{r test_result-check} +test_result( + tests( + function(your_answer) { + checkmate::expect_function(your_answer, args = c("x")) + }, + test( + # use a custom error message + "Make sure your function returns a number!", + function(your_answer) { + checkmate::expect_number(your_answer(1)) + } + ), + function(your_answer) { + testthat::expect_equal(your_answer(0), NaN) + }, + function(your_answer) { + testthat::expect_equal(your_answer(1:10), sqrt(log(1:10))) + } + ) +) +``` +```` + +--------------------------- + + +Please make a function in the exercise space below. It should: + +* Take a single `x` argument +* Take the `log` of `x` +* Take the `sqrt` of the logged value. + +Then click Submit Answer. + +```{r test_result, exercise = TRUE} +function(x) { + sqrt(log(x)) +} +``` + +```{r test_result-hint-1} +"function(x) { + sqrt(...) +}" +``` + +```{r test_result-hint-2} +"function(x) { + sqrt(log(...)) +}" +``` + +```{r test_result-check} +test_result( + tests( + function(your_answer) { + checkmate::expect_function(your_answer, args = c("x")) + }, + test( + # use a custom error message + "Make sure your function returns a number!", + function(your_answer) { + checkmate::expect_number(your_answer(1)) + } + ), + function(your_answer) { + testthat::expect_equal(your_answer(0), NaN) + }, + function(your_answer) { + testthat::expect_equal(your_answer(1:10), sqrt(log(1:10))) + } + ) +) ``` -```{r number-hint-1} + +### Check Exercise Code + +`grader` can test against an exact code match. This grading approach does not inspect the computed result. + +This check method requires a `*-solution` code chunk containing the solution to compare. Only the last solution expression and last user expression will be matched. + +See `?check_code` for more information. + +Example code written below: + +````markdown +`r ''````{r strict, exercise = TRUE} +2 +``` + +`r ''````{r strict-hint-1} "You can take the log with `log()`" ``` -```{r number-hint-2} +`r ''````{r strict-hint-2} "You can take the square root with `sqrt()`" ``` -```{r number-solution} -sqrt(log(1)) +`r ''````{r strict-solution} +sqrt(log(2)) ``` -```{r number-check} -strict_check("Good job. Don't worry, things will soon get harder.") +`r ''````{r strict-check} +check_code("Good job. Don't worry, things will soon get harder.") ``` +```` + +--------------------------- + +Here is a number. You can do great things with it, like this: +* Take the square root of the log of the number `2`. +Then click Submit Answer. + +```{r strict, exercise = TRUE} +2 +``` + +```{r strict-hint-1} +"You can take the log with `log()`" +``` + +```{r strict-hint-2} +"You can take the square root with `sqrt()`" +``` + +```{r strict-solution} +sqrt(log(2)) +``` + +```{r strict-check} +check_code("Good job. Don't worry, things will soon get harder.") +``` + + + + + +## Custom Checking Code + +`grader` can accept any checking method that returns a `grader::result` object. The example below returns a correct/incorrect answer with 50/50 probability. + + +````markdown +`r ''````{r custom, exercise = TRUE} +"Flip a coin" +``` + +`r ''````{r custom-check} +fifty_fifty_checker <- function( + correct = "Correct!", + incorrect = "May the odds ever in your favor!", + ..., + user +) { + is_correct <- (runif(1) < 0.5) + grader::result( + x = user, + correct = is_correct, + message = ifelse(is_correct, correct, incorrect) + ) +} +fifty_fifty_checker() +``` +```` + +--------------------------- + +Are you feeling lucky? + +If you are, click Submit Answer. + +```{r custom, exercise = TRUE} +"Flip a coin" +``` + +```{r custom-check} +fifty_fifty_checker <- function( + correct = "Correct!", + incorrect = "May the odds ever in your favor!", + ..., + user +) { + is_correct <- (runif(1) < 0.5) + grader::result( + x = user, + correct = is_correct, + message = ifelse(is_correct, correct, incorrect) + ) +} +fifty_fifty_checker() +``` diff --git a/inst/tutorials/grading-demo/grading-demo.html b/inst/tutorials/grading-demo/grading-demo.html deleted file mode 100644 index e3cc83e1..00000000 --- a/inst/tutorials/grading-demo/grading-demo.html +++ /dev/null @@ -1,146 +0,0 @@ - - - - - - - - - - - - - - - -Grading demo - - - - - - - - - - - - - - - -
-
- -
- -
-

Topic 1

-
-

Exercise

-

Here is a number. You can do great things with it, like this:

-
    -
  • Take the square root of the log of the number. Then click Submit Answer.
  • -
-
-
1
- -
-
-
"You can take the log with `log()`"
-
-
-
"You can take the square root with `sqrt()`"
-
-
-
sqrt(log(1))
-
-
-
strict_check("Good job. Don't worry, things will soon get harder.")
-
- - - - - - - - - -
-
- -
- -
-
-
-
- - -
- -
- - -
-
-
-
- - -
-
- - - - - - - - - - - - - diff --git a/inst/tutorials/recording-demo/recording-demo.Rmd b/inst/tutorials/recording-demo/recording-demo.Rmd index b0b0e511..d748bf54 100644 --- a/inst/tutorials/recording-demo/recording-demo.Rmd +++ b/inst/tutorials/recording-demo/recording-demo.Rmd @@ -43,7 +43,7 @@ sqrt(log(1)) ``` ```{r number-check} -strict_check("Good job. Don't worry, things will soon get harder.") +check_code("Good job. Don't worry, things will soon get harder.") ``` diff --git a/man/add_tutorial.Rd b/man/add_tutorial.Rd index 47950b19..85998d81 100644 --- a/man/add_tutorial.Rd +++ b/man/add_tutorial.Rd @@ -20,7 +20,7 @@ is an incorrect assumption. However, \code{add_tutorial} and startup file should RStudio implement one. } \details{ -\code{add_tutorial()} inserts a call to \link[grader]{\code{view_tutorial()}} +\code{add_tutorial()} inserts a call to \code{\link{view_tutorial}()} into the .Rprofile file contained in the current working directory. As a result, R will launch the tutorial in the RStudio IDE viewer pane whenever the current project is opened or re-opened. @@ -35,5 +35,5 @@ home directory. If one of these files contains a call to warning message. } \seealso{ -\link[grader]{\code{view_tutorial}} +\code{\link{view_tutorial}} } diff --git a/man/check_code.Rd b/man/check_code.Rd new file mode 100644 index 00000000..abbf4c62 --- /dev/null +++ b/man/check_code.Rd @@ -0,0 +1,62 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/check_code.R +\name{check_code} +\alias{check_code} +\title{Check code structure} +\usage{ +check_code(correct = "{random_praise()} Correct!", + incorrect = "{message} {random_encourage()}", grader_args = list(), + learnr_args = list()) +} +\arguments{ +\item{correct}{A character string to display if the student answer matches +the solution code. +This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = TRUE, message = NULL)}.} + +\item{incorrect}{A character string to display if the student answer matches +the solution code. +This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = FALSE, message = "")} where message is the error found while comparing the user solution to the known solution.} + +\item{solution}{(Optional) solution code surrounded by \code{quote()}, +\code{rlang::quo()}, or provided as a character string.} + +\item{user}{(Optional) student code to check against the solution surrounded +by \code{quote()}, \code{rlang::quo()}, or provided as a character string.} +} +\value{ +a \code{grader_result} structure from \code{\link{result}}. + If the student answer differs from the + solution code, the message will describe the first way that the answer + differs, and it will ask the student to try again. If the answer matches + the solution code, the message will be the content of the \code{success} + argument. +} +\description{ +\code{check_code()} compares user code to a solution (i.e. model code) and +describes the first way that the user code differs. If the user code exactly +matches the solution, \code{check_code()} returns a customizable success +message. +} +\details{ +\code{check_code()} provides a *strict* check in that the user code must +exactly match the solution. It is not enough for the user code to be +equivalent to the solution code (e.g. to return the same result as the +solution). + +You can provide solution code for \code{check_code()} to use in two ways: + +1. Pass code as a character string or a quoted expression to the solution +argument of \code{check_code()} + +2. Make a "-solution" code chunk for the exercise to be checked in a learnr +document. There is no need to supply a solution argument for +\code{check_code()} if you call it from the "-check" chunk of the same +exercise. Likewise, there is no need to supply a user argument when you call +\code{check_code()} from a learnr document (learnr will provide the code +that the student submits when it runs \code{check_code()}. + +For best results, name all arguments provided in the solution code. +} +\examples{ +\dontrun{grading_demo()} +} diff --git a/man/check_result.Rd b/man/check_result.Rd new file mode 100644 index 00000000..7d695f9c --- /dev/null +++ b/man/check_result.Rd @@ -0,0 +1,41 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/check_result.R +\name{check_result} +\alias{check_result} +\title{Check result of exercise code} +\usage{ +check_result(..., + correct = "{paste0(random_praise(), if (nchar(message) > 0) \\" \\", message)}", + incorrect = "{paste0(message, if(nchar(message) > 0) \\" \\", random_encourage())}", + empty_msg = "I did not notice a result. Does your code return one?", + grader_args = list(), learnr_args = list()) +} +\arguments{ +\item{...}{ignored} + +\item{correct}{A character string to display if the student answer matches +a known answer. +This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = TRUE, message = "")}. where message is the matched result message.} + +\item{incorrect}{A character string to display if the student answer matches +a known answer. +This character string will be run through \code{glue::\link[glue]{glue_data}} with \code{list(correct = FALSE, message = "")}. where message is the matched result message.} + +\item{empty_msg}{A character string to display as a message if the user code is NULL.} + +\item{results}{A \code{\link{results}} object that contains possible \code{\link{result}} values to compare against.} + +\item{user}{(Optional) student code to check against the \code{results} surrounded +by \code{quote()}, \code{rlang::quo()}, or provided as a character string.} +} +\value{ +a \code{grader_result} structure from \code{\link{result}} containing a formatted \code{correct} or \code{incorrect} message. +} +\description{ +\code{check_result()} compares the final result of the user code to known \code{results}. +If the user result exactly matches a known \code{result}, \code{check_result} +returns the matching message value. +} +\examples{ +\dontrun{grading_demo()} +} diff --git a/man/correct.Rd b/man/correct.Rd new file mode 100644 index 00000000..399992e7 --- /dev/null +++ b/man/correct.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/correct.R +\name{correct} +\alias{correct} +\alias{incorrect} +\alias{is_rrect} +\title{Correct or Incorrect result} +\usage{ +correct(message = NULL) + +incorrect(message = NULL) + +is_rrect(x) +} +\description{ +Correct or Incorrect result +} diff --git a/man/figures/missing_sqrt.png b/man/figures/missing_sqrt.png new file mode 100644 index 0000000000000000000000000000000000000000..c77796639587b10778fc06b7c9d1a4a7391eef67 GIT binary patch literal 42714 zcmeFZ^;cBi`}dE4lEMs)NTVRqAvJW1G%84!bl1?*-3THjA&9gz3^gdrJ%qwj^^C6Lw_|wtiDn6O;bor%&)* zy|f}DW3^1$yE=wDx){1mdKhv(7#J*4ffl>ok@~gtcHri7ZwG&Wj=J<7FRJsqrg&DF z;ri;4Xyb9v9k1uME__&6%s7eJSz1|H1QzQUzeNY`%0H4iA!N{RfE8kNOWd)DA-hA1 zYq4;rdYo~Gn8B0e!*Dm|D^ijbLJJj)-^m9ehbi5ny zlJA`B(3(nm*P?u{N#rw+m{eX*%16ovgswQCAs#4CBM%$T2YxCZUA_%I;k3e6sqe@{ z*C}-%TlML*vZeWr>@x9-|+*Zkci9*R3M2Hq-sSbmH-+u z?(Z-u`QFhn7XniG*xOO%PmC4P{Ur3mAmrY?0K0qP4(s?NF(gPLtIJ-QNaL)x_SmK< zVP2jp1p4Nt<^dKKn&n=2JL%2MP3y+ZO?ynx3D)wC6cGlCm(60V9SJee57{<4y6(EF zFGS6q9Jov^oXjk_pbpMJKVe`%prXKA2TONT2B?F*qnjvHoavu4M1l9Whq;*;{yD|n zPMk?sRf9p+$<>lUkV}Azhe-mTfq?ppGT;$*D$XU9XyV^Lr+c-Hg++Nqz%*n%D zoQdi7Mt}eQQ%_5%&HvoV(e1yF1w0`4?Kj-KTs+)=uMHH1+#VIxuz_0I>&n?USUS1^ z_mB`25rF)2{{QEj|GDG;DyjEBB?X@G|L>Cj>zn_UgmB+J!2f!n|FqUWM}g*&z=v@E zZF&j(`7*0~3=Am@WjSdb=$)PHyM7OUo%E?^7VcH(e9?}k;Oy~kM0e5B5nng9w9IzR z8xnAG8nLBn;XBxrJe`&k?j5K40NX3NOt+RXFMrU z?{FjkeX*#1vi+%VjRgkX`TJ6ol6lS={6FWUCcX>R!Rxf)|9b{FG4ZDe118hIuSYBl zj4x%JXI@~-N&WZ5Qs_?f_b!oAcz8-m??GA5NdMk)SG@ZT%Q z0o7Rlx1PYI{(q_w`LvMEdj%|p-l=ienkeD5Io@Ab;Ji5+;x^~vF3liiRdYXv-dvpy z*``xB<%SVZ&J3{T+H=DWWSS=T-@i6;**%=lRwvg>XxVR@Ek_;p(Ni|5)pMU6bP*f+ zs%G##nk5#@Ti%|k%w6WSo%mul75HaNalWa>b#>5eYG@@aYOaQ46}&C1-Lp|z+2&dn zh~5=y$h?DfuW>VpS+RLjUE;i3bu>$a|COMY1h?kn64C2a`gi!>MKN8%Cg=k%O-UcW zsCuN5)(ttM=6~C`YUu&)DU;)IJ6!6ipnKmc`=-wJ{TRh~k?K?Yml@v|yw|f0ePPFw z`n@7#b)6wNv+KEmdt1MY)pIwNp7URwE(ErmylM0EenpGQ_Cna;(EC*l=SsHsFE72Z z%EQ0X=FHOOWn#bDZbsf9PgIj{TiZ47jSi8m8!|)K!QL7 z-fUO4Kiu|4tu^V57pr@Xw>fzC=YalzRnobRGYkLJ6u!vs%l-Mhga?cZ?!WI}`NLRf z&RG^{OLUoOS2+2F@VC!L!i(8x=TPowF`SY{E;bxAy^lO+xDC1VGm_XM9bE%Qq~rm-ZJ{_oI2~Ve6x&4 z+UINqax*MRN2l{%#MD+ouPRxTEeLEnZH`m^V8$qxQM}C)L26n>?Ip5~z6L;;Z(xV+xa{ zuepbYhMrDKT->FUL{!hs8Q4vHNz(tWdKtsr#&!_tygcL5M;pT#aeu19aC_?nc+zIq zXTL2-w`b}~ff@~~QXX3^b_7ipDwCQ^LJfN3Y3<7O8|oeEfA>BRa@={gT(Vc38}Kcj zR_sj#5iNf75oDUu^2JAif!4tNUCzgQ@_Utog@pPDlF2^~Pme;ehacMfBH@R8O+Sd$Ek~SX-YkCcKHeOq+R76!(XStDP zLcG6$xt1;oX^`SuGQKW`8k9~3q?hZW@MN2GXe=AC6Zn*6tJkBi>45~j9;^LasY&ng z%qMBAhDf_qwDH*+<9hoHF~_|y8v82gPVv%LA%mPqy`A&nJXuO22G_+0yb0Bn1y zA?6g1X0L8;TsXm;8ciqa{zbpLX`4PjxvA3k)gwz1i2!58nSdXkq{GwiGb*){NeF>Y zpg%+^4*c3%CFB_>f@*P}aHRo5-beiP{$_EmBYL+Xx3mu!fKKX?SGB(1&i7jO=V0Xx zh#A8yXc{g$G>&Ez=)G4A$?&1G_=CpPG<^dr|FsOO=J=H?w-Faq*LK_4Qj8}sS5;1l zoOv`t)x~xNGWo6Ib1hfUKadEwQSrM%d%YM5j(0Czz?rGtOl?U^ zwbn=w4xM#zmh1D*$(%l*f7b>X)?=@j>KmzjyK3BiNaOsTb!y+W`fXR)`qs3bLyAWZ zHE9%ZZpq9&KrsBXHiZ&)_kHByXWL(2JpntJ0VW$OVE!sR-kqKDAaQ~nhU=sAW4K&% zaeJRu!hU)fwNb@aDkji*j;jiZ+_@7X*Lc3$mFs?4dtXSyY2_y)`@o%U-!$tM)}Orr z^hotBsjGtZIwyj5uHYkJe&(K7G89Qm#MUGE;Mch`B6lZw*9xlHXzdl=5bLVeYh{%8)6+CD^3kR`u zc(5HN*%eD-lX6hsa_H6HHgKw#%i=4=&7lBtbr*xIhPj$h$%d~v~cCKGZDpb zVtS*bi); z8e`}fYsS^Ys41-)W#cG9r={eoxhR>0d>viqaQjcAte}CoLe9%e)80#rUhh~@$yAt2 zk<9VVLSzs^Dj|Evtrf`S!p&7~=IL6dQ?vR_1Q9j&gYS)u8pK<@yr!QWSle@8Q)ktVSx5}So7@D|U7(`j>JFGs#kEV3!@=@#i$ zVzC{bQw+F^vq5G1^@Y!Bs8zRJx*WC3u-hCY%UOcS6Z$ELnyyv1_c+pH)&Bm0_S-mkRf*gpPq zhw4-n`|MG=!P1DxUP7Dy!NrlYY%U9FkQ^S%PSTNS1@tcIptP`af$(8B>HHSdtnmp- zrx%w~#sh&X@}tQcMY}FIWuSfVCoUqA29!gk-FLN)$@*Q%Kbnt= z$#JJ_NSn~J;||U=u&ytj7*2R1&)riI0BQ-Oqo)q7eqo=0fql@t^P<8mu6?iX2N z@d=0LxT()RFCVLiK@~yo+zX-cm)@!UZD!@mqa~#k&4a=VJTvKT(JXo#zdhCK75 zQy8?^y@H+Ee0>Rw0jAv|D!$0EF6h??Qp}k6Ij4P!YgAv1pBxBjM;H*Ao=+^s5h8om zT(I10GAt6Y+q_+|_i=fSRiN9QDGS%(+%CdSkR#Ecvy-0g*`*-dmHy&n$ZNshixoTL zF?R1^3O;E6g1f?y#PScYcs1LC9J`nmtw~&>BPgBRIJOb3Ep|ffoXkg!bie5|dj-DZ zy}YObV-C?w@ex}4UKdA4a7!jVJKRqDL+V6CJd@Q2p<0gmvsT$@vA11fA)^lkzxa}d zN{O%^k(Kd==>Ft_IydDG$}v!5;t@O%-E&q;Silo1J~;4@l!=LnDt2_r`a&gwW4b&r zLBnP0C_1OqUxJs}9&~7t7MTcN{jQBeD=fra1onb@MGpx|3ygu23Zx8w=eCQ4Z);C@ zZLNvnPD(eLc5kbgT@U0q_%n&;h&l2hfc?%oga(3!@ z3;2{_BhhzO$*l(bagTtm&ig%oJW4`CXLKEF#Wbs&*HH> zkViz-r<(j!1QbQ3UTqpf=LFR|6U!^trC1#jy|$5@ya=F^VX<`=nCr(IPc6#MUwzCs ztV&RDAB>Oi%j|o;^QUi7zH#C4r5>43J5ey5Y$(Mqyl;?<+d*)}bEu1%gBr`U0Voqf<{gj~Bs<04C>5Os zdpyay6KsWFNo15H7pBAHVNScidwyJyBP!WmnCp&V+Qq-zN>r<9GxNm2(Wc#ZKbXFA zbEpez(-dp7%gth5JeoMiyzEOvqpszvLhm_#YMaP+Oj^=JCTNDN zj6t-IvbXcX;)Ueamx4Jd%)QMO*cdT`AD&@KIvZ(FGN@l2&Nh%HZ$JK7g|eZW*x{t~ zKE8~7R<}gc)F^SFD^_s|qvn`bK8GSU&x7#Ys}whAi|&H*$yun5$kZ72y-(5b@sT-fU`#>u;@EuQf+vU=i_RrUHy-9 z-x+-L@!eO^`Ens=7=CU3Xf>R2Faq9a>LXjW_+)WYV1Ad#X(1zYTdJSWkM74as$iiV zZ!>Wj`&8LJvLU}|@$cX}e}Fw{?;HPC?>5?gd3+jTS_4Jso9FNW*2C~2j>`QgJ~IJM z?+hx0Foo{4nR-q^{`6e&2bItaphR@}TaXhL17XOi3K zraW0hny)R5eMVzYODJkLDLb9$Hxw6CtSJ*rHIVA=+BP2LnAnp{=&AkD)9efdRAT!`A^HJP&PMf*0T7?U8Ca`_Ao8q- zKloe@_0ze)dti6u-s66@t9D;SDr;{gf6RMUu z$D;6-@!iHlu)y30@BPSAqO;?xV(#?#P@>RGy!{>NA3vxQ56%~EuBR&N5yjknfh4EB zg6c;c*=*jL5b)8&zA-h3Gt}Hdjz$RT7(Qn~B`^OW*+P4Z>{=vdxP(hp?A%{&JE>oQ)@7QF zz(u;{hXhO6(i0isjsaHn8Y8ATxFveu-6BgY^5qP%CLXbZ&Jjuf&-#{iu;bHqryFIQ zn8+S}Y1&4vgV+nt(``m;PkcJ6#(^3l*}8!->DU7k&D6ut;R|;vmdbH3%6(Y9%0wct zf9hA@#S!SML2fOvk+~c;Q}p(G>y*tN31~@yD0HaG11w$wiS+E7CXg@HD9NTXL1l&w{x<>yfk zFdaX@#@T$#C|`s2ULJ`xHZ3Kes%!T+klInNCPv6O5;AYhP=X&sIjU6P%~rLw2OWgU zS40LT$2+Ec7pQO-RhmAlddWA-+RTMs96k&syBvnV9BgKv7`(D+(az04gcf=gsYhp`T(p_JjDaxlnr;Eql{){+4p|kw z_dVABsVjuM2iU~r008}Dfz49*yjN8}N~#ykK-QxUPJ+C!PBcCBzxWfCxtQgGliZ6R zrTGB))f)Vz5IC3Jrfa)O{$FnTT%BR>OY6^lnMePNo_dlbCJv5x{?h!5Zd32v!7Jf= zS7!M4EDKiP$d3JM+V}sGTPbQ#Vj??M|4i85BmeJ0|JSuq(A*A$bhDtCD2KYxox1p@Oxz- zWtnfmW6E&%QwR~eXX`DomN;9)YT5c)S_Y8co{G%quvAAL+DyGfC5s{|WmE4zTeR*SIc;)7R@TFtU)LAgmBIV+{&^1_8vz>4Y5QJlxE(bjwhJJxIsmB7 zY)w@a>eGy(N? z;)|x561iT9>l+@E4h$Lq8{Z;aQTGiE3(ovq9leS=;B%AjV6Dw~v_(q%-GMbZ^4AeM zX)osjYF}WJFlsyro#-N#G>E*LJ@QF9WCu>bU7PJWQ}W~6QI2kvNl>{ixUpiY?8*LD z>zhX`?-W^H?g|B5LSM%)CqTaG?gx!zdc+Aj&It>-EXN7GNuU#-h z7x%6$YE*m+_eQPB-?i@z5+F_F4@F7GLkJe`!>_zt;hD{{YnxP`T*G#w@ZN|^v!d3 zrs*4vkfVY1ymHFp-^7xaCQm~yL(A)D20uP?*vh#qQOSJfkX5`lDtY6MObx$JLt(EH zdAIUG{x4q)JiX2j%bmB--0KAt4D&xBQ6r z;IKbxScnT-(eNX37r4N&vm_sifi5J$o@VYVf z2;u}kUi5YGgKuBBQBOi=Ot5{a$odF^-WZ5=CVTb<03n~D`=YB{hRvZ~h8ALBuaAyZ zj_VX6>w%W2M~2hPEOv%C>`YfrHhb9JwjO@GF9PPh_Xap*&Y18~GO+0tTgY}2$~lI! zo@Y-Q`t8un&6r^&`qRn-y*9rlG2*5hxi~%%E0G)Fb-dHCO6kc2ukv%Bp5}^t^i$jB zg@%Rc5bkyh?O-%^{p(!XooH?Kni)Ymin^$^7^z;+m(+rj=p!Zn%fr5tAF{;dQ^LRy z@t4l&Nl};Bl;I3@Xxtpl5uY6qKQ&lh^Ew(}ZF%@qCIX1641H&h-!b}a!irl z!`m2nzbOOZhZ?>O^a!B=OR7Wfw@bpJNwt8fq=7c@^3ZeIG$O;wZQf(5DYM1+9AL3k z#sTL&M*hbyStURZi?cmH4rlQ9RW3<@^-ky962^i|R>BxvH*j?+P5WI%=WaoK>%rE) zAb$4QWG+Aw$HQI#@e+N2m+sH8H!K9S)B%nySNo+a>mYnq(E3Nm;W1{M+VdE%hC>?b1j*@}Lq1Dw@; zkYSWNx6w+oAz!Ym!4;p2#H!r&N@BUG>OQ({x+;?g&$c{)I#JFQkTgyDwdPN~oSH9IN2Djn&NYx;bY1Y?YMl!F@&{s8}@Jl{VeFYAsRJfho-lDfn(d3|DhM zt(Z*K`5{M==hA~|pY3u(%?5-|siDv2=WTMD%Iu#W<>%2F0mlsPVJ66r%#3uEg6ne1 zv2Lfg=$)8aA|W0%u)wXv(FcN<%E}$`^omEh#rS9l37ZcX*^OAnBh9!h_WUbty4HWpo}V#VeSHgp>ntau?q4mE*pt(AzsN#jmuT>{qd zM&%qrum=Jp3W*zcIo+LYSS-o@fuT?wmknKt;xdreiv8U>aL0LKv?X%0&_ya zoS`6|K&&+?KdLog#_; zvr%~9zW7QJmoXLOpqbvZP)SVb+|{AK6sJLtcqsZaLE@r+)HAmc#oJ9FRAE6_z35gs z9S>vp9)Sb0rV=uju}3_9@nNxz>7E(2kQ+9QkX|>~EE(`+8>!8W%D|5JG$ND|auhlA z`(pV~YN-#I;2Z~M$5Y1Uj^QSZkmcP3$!llhP2H}(fpPw{emkxcAhOcH6$qgeWu#|1 zCIG8W#2Ju{CKaDlbrP9IM#RCs04jwr%D&5Z>ql0xZlSQ$Dvy1{yf^M|D|QY;V30UQ z9UV@kP;Rz89N}CDLVF`eyOfqcVx4e=nmg7#-6Fir_(>_jU$hdn2!(((X~W-hjzggv^dn^*PB$m z5OQGcx~TAj@FLBRE59&!buDc8|9)ttLwIh9REL2Pe!h%_`D^lErs2wnbUs^zTfa|0 z=7uQJX5p3lr?HER-NWY3@Sh$Cx$QJIdO87~8Hl|^AUlNnaz35DG8;ad`p369Gn?fakv1DG6;+w2B!`4%KEw^Ult=j%HY zvX}{{l5~d_^;6AE8g$rvMoNe7k*tF>ZH$?!h_Gd|7=L$JwiTVOo(#6 z%M6>I^a9Pr`bUmBsVEMSn(8E;t$sf5_Laj?4Ujzx52uV(6PZ*^xnkVX4Ug=uOJEJH zB2g~8WED{Ij(noP<51Ok@9r;-L3BvpBYUJ5ls6X!k|QHLd~vAroz$6wk(?`HKXe=# zI3}{5wJej4Tzwb_BNBm0EclCQzny?tJN^3lQksa^+a^}+=$>()UlhK|{-(KDJ+bfI zONoJOc7msy-lXN3>r@E|fjrXK5#mp--`^c`y{hGwG!i>M9zV}ue|xw^<$v~M%O~cw zn&wb}k$Q#x{_izWU^n(kNBr_KNH(DADShL@$*c>j80vue(%wH>s+Y9?>|sgGh-m3z zcub_3DD9eYZE)CA@w9>rx8TIl?iWf`j!9nWs2#}N~q zq`Muue$MLQ`Vy{?ER43;@Z;zT>2alhCCzqNZ=S5di%oZGn!dAZs$=O%#ru=zG}anK7`CyK?h=0KTXK6Cp&`wss>cDdQ^Iz-laMjjbwp|urM<}?-yq>&SOKF zVUKF@N#e=63hcLE0C;g7X0+l6ezUR`$-`m$hiX4AuY$0(h_u&Ss*NP)E;i#O-XX8W z72v11n}tl>UF;-S*bt$+`HHQ8fwW;HI|T3|xu_1XyZGr&$kNZO`Gg)JSEEhi9pN&UYcja?vZ_eX!#(m{Y;Vm`E)!!tX@;y{ii3^65 z%-&*PA(^o~ZmzH9>%g4+kZfzQu!g8p@fo$l$9dk+dK=C&_!mvy&oa&QYXrFscU+%Y2Gr@UievVVsY4Hl#_qK242GeO{$ zL%9aak7J?ka9pfVJX(I)lTD@bVQ!~b9rHqC7ll3*A}}43D6vequCO{AWOv;SwMwfP zIa2ld-eKQ{^!~X^SlBfK$w4Y_Xbw-2kfR!GD4a%A=AAQZCFTM1nJ)Jwf~p9t6>B6X zaG@C*dlUkOM>P8w;WkVe2WAh?N#OqKq>~Pn!k_9X|1(POc=nL_*i`be?+bSxT-WS0 zSCxRu=WT!lEpxt;#^Ns-7H`>prcmB`kekE)kmEl1%yxNRjX&(I9-NpdMQ zV>F21CY6xzx8k=~lfW_)5OoBYhQYCKCL7tw|LRNM-sgPbrgnI7?}j0`aV-n^i~S!gy$@h%asW&7Gf`Kjw!iuv4(`9s zdpc$}oW3qufQ}auzs$YE5`#6mB1kN$Q{~f=R*v7+cR?btr#mF5jy)JH_1TZn%Ej4* zd>Py++cVCm&>zvQBD!h8Wi-fi8UkUFd4RCfn7h^sw~D+P+hoZ^$r4crDaypK@S_uz zVE#w$gOTz%KAUIWPaX^XtU_8YL2`Zl$MP^f61@=#=(yyR`=5_iXdOd4`PM}K zB+=eCirg!)^75#`5P+q6Ym6rQCl`~cC|M35H+gp7mk;qpTtYYdX?Mnp6VO+Ua)Uoa z$y20$%{|nwVZT;_$;HQivtUYmf=xCvl$xd?eyc)dh`}qvb6tAHrFR;g^z)&uQTc;c zg8h}LQBsdVUq0f!*P#86R`pvZ2As9n2YH9%k%-w~VkA4Bh(ih&=8@9pn6g0tI|I^M z-2{xBl3!0>J*Hh_&PvzjRA;gk&;O{20Pyc%%iY?*f0eaC_$pF)t@yLul4=gsmUb$oQ_?yjxEEF)pHEy*d2dY zwCvetI@D*(J9pjxN7HkS-lzAw-A@j(2xX08Tj@c#Wb5 z=Pw;F7cSwszIiHx7+Y%0*QzW88*S{rtI znJwYpoS7pT7yy4sFBt&)RcS1VJidyS4JWJy^P9-8Vtb8UcvBYG zV_#pEK0F0PZRduCm;8gp#_a)s+mY})jMB&&V339~XkhU^eQ5sb8en}pUkc)%TZ@&m z{Fi`dya1|r3`n*qzoKtt&ms5jyV7o`iS6oG|K4xEF|uxEQr!VW$@RPSvu^-R=ue~F z2s)qs-Q`;uB8yDG`6^}0QuNb*2#Zl`v&VK(D_7JW!=TmX&(HN{J^1>7qFP42ipxmK z36~e|iMnj=e^l$_8B*0UUkh=x=-`$d!6z=qIKR~GqCOY!j2Lymsm|qb$Py?8%A}0;nKFUx_Zlm7q{R|LH*@M ztv=F+6F8L=rdF`v`DTx&CU+zu1zD{39|$>~D+}#4Uern=t*aYWw`H&|^q!z!m5K*Ust7-UxjWf=gnMrQy$-m+*- za7Ge0pwO@O9wm<1IH>wU**{EpP54mwhj(xV?CAL|$<2D#^5>^YHqYVq;u9H$dB8-& z;2#4-L|y}w8zyKw(GQSE3CWfp#gu|jYmjVYcU5(us2euz$x4cPvojS!lH2w};FO_P z(^Y5~S$+&(*9l;evjEZBos`F(19YSSGTxR!&ZHap41k0IgDOGso$rDPoO|JZ66Y(V ztjQVbdt~rOIf?nL)#9gf<9OVQNuhRd>qyn-w>o0$Fm6R%y(e)ke+2m_<^|@)T{y}0maFh>u*LAfz>#dChRGj|{SM^|_`9P? zgvY*JpSItKfaO`EJ}v7WSiHbJ`U+zsa0;Q4mJ@WCnYhKQ>8E5yd+plAw4T$QstAC> zd)~6^LN}N!RQR9~dFL>A$SJ*_6_e132=8Ws+6mv`@N7AOawaQF2&RF{A%}g^i9;{U z`!4a5(GLA6p`}6`j4gSoT3bz|=RK(*D@2vnI&hjnj{3tpM)+iz9;wkUe?`c$s>~$> z0O2F00-ig%eL3VNaiGtt3qISNEw_s9Sq6KMd6w_=k389VRxB)#FBQ_Nh&P_t+#jZG zBuDGo=1`y<#b)qrO|$c+X&aGElhj4@J3{qcW{*BkHFcukki8Rvrg?mFQpSVbZ= z;cAdWIbb(?DYHK417$q04%@v2V@n~0_PMWRH$JoZUL8-(=ill+!%(fUF{7L!{WEN`Y+qoep8rw{#oM4Xa1I&@%J-9_XIcZ) zzR%>Cm_8Ray>-8EzmCl*9C7Y~Hz~gum%x1m_hPAYdoc}wqJ6k|XcMlv88kdUd>X~5 z8!w1BT=IsWAbg!pqcm0|&H7_#8y0gVC-%RVR=7gIMxi4-#_2$$>6hx%*Oxz{P^P_FXMUl$IVTsmHtYt60q-(Hxbd!dCS%MW3#jeRMnnQ)V6k0*L zzPoklr20rpK+kqN0c$kj7D8$jamYs28Ky zH{VRWC(4B}%BVvZ4wjI=j+!9yWHktTVa<9WZ1!OM1d#VDgmrEeqY{-1^$6;4iS5A} zzEk>!7k8oV4KUq$WsZff^fPLJ8cmrmDI@~@;*%u3SaM#GUuyEW@fj8)dyR*nK zroL!@&{4NZd~vePbVpCbD{n!p@JOQElxte!vDfc*0K)I+e+B+$6A23P5O<2!J;OFO zHfYqg$RVZ#&qO^i3bAgCh?lUJ7`??u*+eAFf3!$Yl}^^P^$s$bzST$Fr0T+>dVkKk z!Ze;`iy;{%yJ|WEA#a@-74d_j&K7B3R})GgFB#j_%H({E8xA&_b&pX-PmEuEmB5Sg zzOy2pg#0!jB2$(qynlc6&2;{*KM<`E@eK9yw#0^`hlLj0QJ~k*6r~kKr6m0b=ip&r z*4qmu!@`7V{K30v8y-kG${(Bxa^ri0liRg6HCFu3##UX7luguYUnB{rrZlKzS&Bqkf;WPI@COxzZ+*wHYvbR)Q@BxEG^OWQ7x0wjw%13 zA~9r@wD!sqsqShgyca?w+P|t8mfpE~i=B}o`{byhR-9NHcu+N=Ciq$XAwm+S3WP=P zb91HaOAK5BTLv%2z5}(x%Z?8`wtRGSBeX}~K?=2ajH1a?Hu~_gPyv55NR&kqpO_pW zDpMQz88E7*uTUo{zk_=uZO0-8Z52D*I~jja$NyP<7%)Lf-tjt+ve14WxSr{9`P7OO zQkgC8Tqp%?VK$iG^@td+QfV3uwv&|b{t%zB3q#&9^(858MIM{!N+>VmN}W)PbG38j z#!7TuobJ=}LpEC(5i|<8gai>{?Pp5_LfjV-7vb_0Om|1gonEM?_dVe_$Up8R=-s}{ z6H3U#7E^Zkr=DHeq5n8Hd?`{HzJ(xKC_WW+gX`+^^s<=s z;COl`4X9F}aw2bj>=E=XRt`4I2%5YhF?Ov#6!Q{bT|Vg{0vk2I^xI!aj=&Yta!+kH`VP{!LK>HNwg#evb+>~_`1ng6U)>|n$?37 zCP&rL>H$MwQ-bz`QnoKxL$IWGM8?=e%VKf}Zn}9l;v8(jRr$NS5b26U_ z3-KD`+zdcQBlrJp1;tO#8Ssyc%d~s@;)SB)sS+@m(YVr|e+qNB#!(>^Kd`<^_JCb6 z7WLOL1xDp295QhuJfk?)TQy_vF+YqrE2uf44RH58$~>51z=mf--NlRWu(Y1#J;j!3 zjxF@sQfmVT7nH>n`or{a4Z6Ld6Q;`Ba1q(e1BHN@&{J1sw)d$;^|#(g(4h3Exlo!2NGAEpzO1hIu^f|7McDj{z{ls;v`k`CktjA4op^3@VELj~E7U$G;fd z>aJex+l*R#)30oQOAX3P$$WLKYlwgK zZwf6#7*IzoTy6QknlBwL;P&rJTRm$3KI{`9@%FvU2lnUxmipuh=$rr8@ITf9n2K<0@-K*sTj7D+ zz~2s(`}7T@q=4=w*TP?anKrJ__wP*3gb`#>t8T#fsPJ#I0dJ}SOX^w6Z~C8OpmeD@ z3smDH9XL4!;56Mz8aRz$jU;sb=d`Zx*?{^^)|{I2EzV|;84NC+3!BpZ2prHK7`Ir;!`LC-tx>L^-i|S)W*=VNNhDEf=h#zHA$}6C5$M+{P z?E2k_4x-KZxq!^(e#Es)(Xf7gI!!vJRff|HEI@E5Dg~=WbmRTa$95UB=I{EfHJ)sY zFXMMR3EH8)?CqP(!2HwC1qg^L%>_$e)^UnW)WpqBRtI!h>D?@>O>YSfpsS8tC+R1? zxq2L}riSwL{U~^u&A>Wgzw`akXOhsDA$+z|{dHj~{Q5dMO?LX*zl+pl8Wq%y?N0+m zOQr!i)%UuQVY{YfI#Qk&z0;lpN53A4I_0C01K2N=|NSih$kL_C8d>4ITS&NyXUj%g zjrSWF_PBB}{|k3Ckhs2kn?@W(%K8RKM0Q9~eOA5Ouz(+C8N1n~Q zj|)u7$rnds6rgjw*QjYA&?*}7X&}pa8^EHpzB|=vDJdym=f7leFlc-p6wT8XZ}_^~ z?#(}2?}p`j6_If=n<|e-Z#t_WxbyQ$w5&mbG5U#H`fqW1_1e`^t(%aeiE~)%3^kf~ z_N+a9ruFLa%vTcJrgALqdEVK&+R}xGb~Tjcgp`?@3t&xkKhZ%2wT##Ht)g{hPF8!r z=3TLZw8#=tF6!$!F?o0IOY^x*^0n-bpAmwq8|V-s2iDi*kq5uqB1r-fZZCw;E(XrT zXxpz2lvy!$!70`U)8{>DUOy;??Wf`%%+B>Z-5nI_rD$=qG-P9ZUz7%n%+p<9t*d+g zbZVXhq{H3%Ct1)5F}DW*v0zBX@m*MM9shs(3W z5#Px+f3$0<;sYYi5+<-_agNW|TXiF#XrvU|_!P1O#LcUKgM$jORHnDOqiVD06Ceqz zGDZRc&W~w=~qXVDi4VZMpb#zg)W9fJW3Y&HvA`g`Gh|^KP3B&Gy8%ob1-w zdulHpwT#wvddGie?&JO4+^rwD`4OW3=287?eD3?h-#Bz}2);ctr%#FWybe|7k`0Tc=y=qvwKWWAp5vUaF!$PT0&eqiIk>q;i6qWUW^GH$ z%cMi@od6=6Ss*`p|2h{?j!b^DRB3Q2%f#$vrMXrJ2b=kj3!h`>U^Ri6r(zT)qoAR(L<)Rm z7L#D)A)J4L5A-(lwdDN~>R4xFc;`)SFE=hsA_@vS zvFXN+OkuBq2ea%TMM0D+=6S<2-O42YYxKhg$VG(PXEjjtmH&~!_E>%1?yu6Q)`Q(Y z!M~QWwQ*fwda7G%zxTB%XI~erzqw-2Y0`FXyA%#+Q_XIf+rnOdvK@;rQy8jTEgo|9 z0(WZ=B9l2$ZKKh0Ic<4wfr7wkLXhPA8471mU9`Av9ceqvg3P_uGmf5;>+MUkcW&=O* z&;roT81|y+^TYaf0PyVsO37;Gcwx9F!rN+UjQJjjFyf(WWGi55Y77)DDI9u@e&?=~ zBQ1a^Is0`rph@M*>Z6%#m^LF=&(1dlbdCry>h0G(ugi2b7eB+%O8AD6J zAV_cx!6jI52<|Sy-7UBVcL)~Tf&~ul!QCZ5(1QebcekJi`8Ln{gnPSx_2?cwy1(D& z?Aj%(R;|6(oU;m<(8H&{hQ?<(4+OtRBPV<*e^>4;F#c7Z;lW$Q>42+o|BS|JV-o&| zR@Hhn+pB!a2gFvJP&|zV|6}W-XqE_{qt4k{7bU(5v*@t1 z-D4xs7y7wr=X1R~my}J_^366lIox6#CrkAf-i1BYdXjk6`8s&f-=REpE;p=TwURp9 zQcLaSGUUUv>D~t3)K8F_DjWZG5wi+!6}!}0ZS8q#G-rN0>)y-&@0;e`OIeaamQgfC zGZ?a>k6^9KjW720#SKpGKB8-be9be6%QCJvCMUdC{iH`8BR@1p!mR4iA5$?QHlv-V zT>&uVON9ILvd2-?hs_hHZ5$Qe0d<>xspp2l49N7?eeD@A86}=W+BaDfB6Sv+ftOpF zt(kMqCJ)~dFqFz_t5#46@m&BEEPfij9*rG*+_Dd32m#|4cA6vI6PTFq z@$TGZOp0n2$Ubflko*yWi%1IN7{-JWHdlOj%69`F#WCX1M29UJKsw*?M&mB^b&n4M z;VDYKA2b}EUdhhoY7+@O05$Y`y!(JSPrf%-WAVw&NDoMas<#7a=>}fXCL%FYZ2Icvzd&E)7mH-*`qYCtSJvEvt&p1vQfr%>Wx`lyHPHl3uYFIn6 zH5O07PFKv13bl=&2@0lNO`zgA`|@RK zi2E7K{*T_N>L|)qrxBwKNlo(>8kGYF#utZ>R zI)qeGP5;vy;)U@b_0{e9%l%l^!ygZypf1x&j>YM%=Ta$LEJ%sHs@wL5UgW$-Sj;^u9bL6`vz-1KlqDPZ(8oP&Fxo$cgEzE9$4}hBCQ+6De zEVLz6D(*}oET%&4`UEHoJUu>wfPGe^@c@LJ&9^b2AY77N!l!b=)cn$M#)!}TZKIU1 z;_7m(h{9n-7mAz$65fZtx>-|vsv`K8Yqgz)9Yf|KHEPb>NsXT7SRoEl& z5=|r!4X6z2Q4PbRlE*W*_yM@_O%Izsu9I1_46ZC0++?p7ps-b~fKRD~TB_~>g99T! z9-deVE{LezZW01AdgEST{h5TrQXE%kzo&>@_z2 zsdZ8Gd#^}aQ(Tl*y2-I1Kh3(MFV9UL`Q6j$OSMMM_EakCgJtT&FU!qzMFb+x2Ud2~ zihVxeb67QAAhYBBnO|0Gwf%&D@7fk(t5tBq^-Y}|jj_h&cTc4#7G#kU7xxR5`Ch6? zoyCg#AJXYo54JiuNI9mGSei0KV>nQ{jpe+*pgFz1?j+tTE=?W$dAo%rKCf~Fhi*=K&zd6az`P%M zZ*R=>qjYs_$-=lLC51@EzOD$0{>~!f%oc}dNPP1eRdiZO!#u!v()Cex zg8-0Ve#1_f^{azxsl-B3N_IfNYTA%Is_)|V&$v8dAeK>-_ldUKlMS6`uXJZA_l`RQ zR9h(j;US%NTE@>aY^BdpR(Txkxq~eb~far|fc!7s#G6 z8}j4kyleh6#jlWFGn9(Pm)HtwHO}s4Tgz?n?Rk>sIV{`4K7L{u^}#6Ffp8QeDW`P{ zxV@jT*wQu~JpLmQ->uzh+jS`?pV2Xm-AV>!394PytUj3y>CCLchWq>@`u?79NQF2F z#v9EW#2+I*X%<%Mv>_4;nrn(B3IjJDu-ZVG#2tudME)$_O@x6mT` zcrezQrR@wx#(yOo!#+^YAY`jQ6UJSzM9e*oH#tx#%fs%%CAYzK&-^KL@lC|_-EA~i zoefFuEsSB`o*%C7K7__)cuCiv$ynD4J=IkpBH88wdNr&gfZ)BIo8TJ;U8=iBl$M6o zn0ozkI)Ul-u>pr;%{s;w*+A6_K*J4(fRHE;Na2)HNyA&Mb{J?*^yLY2qMvof1r{JO zqNis7DUe0@S@b0l$<-euC079DWwu1KT1NwUbVK6qrV3qLg3vks+{Iq;YyH*!wY)8m z#J%g@--gH!h|?+ga@+74hj4N`Mx{k%*qnxtqDiI~*dD|=yzSTt!s61{c#BR3R$13L zP1ki@bbbOba2xLL4^%!W+&uv#u=~N~;BDZYHa7Eh$Rky6^xU$3J2|aARW>@~KNDt0 z#SjUzEqsM%N+gv=Kln_gAAK&8=nm5}qGLXSO7JOe3-z_$!a z`ucVM7}Ohvr8EDNIF9tAoVqsSu(DVIKC%;B>#;IF!dh8G&poikE85W{@@%vh?QleJ z)l)*(1?kAR$;A1cAH@hw8Fh$S3R%z;{A{Kw4biqh`)yYX2D&>z&fFi0ZR%UOIw@Zf zA(W^$f34H}ZShD*ckVwzPhB9e!Fus%V@rba=tO?>RbDr?A|!iY<2a`Na(_ciZUTl2 zQpW_}7?r}k%#JL6G^? zJ~Xx#Yyj>4+VXO&sse+ zRG9%F2)hm-_(-4zG@(DUBii2~>=dgX(AyXN`p-UTu95q|-PsA1yF%c1N&l!erg$bC zcg+yu2qsD+fA@`#qxP&-sEmm00Bg__D1a}zJ~#oyvKWc4yD~V66UpHT>+crC&5wa( zkP8`(eaM&WNX((*Y~6lhMEo0bvA3O=qk*KU7N~)a6YvP+fAripv9D}dL^(Rah{rp( zDGbC=yt3^NL!1!Pi31V>MJo8xUq?vmlPJfiX&>bO%#^$W`~OzdO>*Pqg6}=RaTA%U z|J5{6EPqhF@rmkxP;4 zV#t^JZfE-~yXUFpyf0Xd!QQR^jLpxCl&X(WYkLUydY*23m(%h1=<+~)Je8WH{3}6< zdhjnIvAT!TTwgi8NG*2r30@9=#(J%`ZK~1M2cni>%W6TUd$TL#gZnJs#?N{9dARW6 zQE>{UWQSkfZk?qc?*C*i-84$LIX{J1dsJE(Cm!$0SjnO{j?8U|a?UxG`QPHWZ4{Mz zqCwoQt*LTP)Z9t27YMt$alDRk-fbM?mL$cg)1!Su3~*#y_UySCG4GS|WpvrzU-nxm z!&{gZm(w-iv6)x^5QNnM9UL9p;{iJ5Q;}BlhThE$v7y0si=CT=vNsOW6&WiqgJ^Lv z(uN`=rY-0_a~1_@I>KzE85QVdh8y@j%^}%ePe@>bll+I1uOwobWX$16#U7L=;7duf z-7Y%C@ohGQWdgr+TJqiTHF-OwEKqS92#W<1x@cr|`)E86c^<58kCi|1&0kKJJorzha2J`ygu7inOkmu-Yb3X@bpG0whv*tv|plFE~$ z;}Ci-$xQ91&iex@ui;6_05UJ<>iW0U6Cbnv0}ZUFt$;MhoU%c#1#*-%zYYh z{_Nx7eD~!Vjlhh7!*4~K-!(4A`_3jepQ9g)OeEL__J28QPgialw0>`Z%SZo7!7 zW+7c$Foed=o+~+{E~9>Ysb(F~Cwq@z6?_4nhBvUJzW@T>c+`TS2#-&&mmiz=kx~-V z2!AhJ+9s7q!Av@+zB2sMLjJU)uVm6?1mboMuD4oq#sa~R|4QNbLxZ<3tX-DwA1+BJLf@)PHt}N;|@~2x##Fxlg0`-`P{m+PI$4AkCmX@C);idDz|)&-)hPDxIT9_HoHar<%iZ@-Rs zj9}hC$mK)j66$}vFFRz`jCg+p@`mrXNcM8exY(>}ADs9(q=mztm)LBVJ^hLD};yqXu|B}?KJ z7*s3OF|FMrO`n>xK=rmF^uKy7o80;Ym@(M4DAFZo z#GPfE#+}^0I_)M7KbVhQew8!7H{U=aQ)A$+s@-U0@xUK2e0G>seGsrBmasU_4$nQK z>ZfiLJ6WhpjI366*~77JlcZN#YE`^8k5G(1UESw`h&y`8)i_z|QVXzz?NnA8`;q5fke*I#V;+n~`yje)g-4M3 z4p%bj?;K3O_rv5~%xy*1ZNVmdtMVTo4)HrDcTfXp}sKuNO0DG6er&?Y=MD?F$ zEz!m2g@@ago7pjA@NzMa*{|)aHMHNJI1sZs_#@F)P4EAk0m}VX?v41jW&sY^SI70Y z8@;CtvuW;e+htD9L&bVLobu; zo0EYTzxseEW>arvXM4SyR1ri#h$N3H9G=AW?M;`}D@mqaG@*^j79jq8B{irxBJbw; zZLCZD2s3mogs4U5?y3&R&^9TUGz0)PwU`zoOlM!RG|atE$DtIv;chpPbVA;B%ghLM zepu&VjtCK$}#=3jyk|_#giqNck&PX8WfxymK=S=TsVATd$s$T&c)lNQB4%rS9b z>_Im|Njd?pxTToH2cKkruSS1Iw@#$m(ho1yw$-k>2jD*9)f9g@_{&cqu}u^ z-=b!qW9bJ9zdpoOTwAWiPSHI{VAsH5x--#w0JG&Fq4Rdvg6wish2DPJE4F$TrfW4G)FMh~2a4c?r?ne+w}M7~yjOKdMZ;$%e{4W^ zHAE~2dkyD()?I8_(aC0DO)RO~?Qqwnpu%m$MKV6wdX<~OP(JcrC|f<&>~)qO4!zbp zT@H+#{&9GLT2|wX3V1>r3uHUQ8%d`#Lpym3u&l9Xr?%PnEgk#xlV~Ff4xm7`%E(HBnae` zlB2RuZ=(G4ZjUM@E?%;+yVlCdbIDpRDxSk(T#_W(Yy)I3`;#igwWsi-+Zcbun25Eb z&q*3PyngVzBl0ryU~~LVTt<9D*_-r3N(aS}VESb$dPruc7k z({+J#0%418XIps0vyvSrE;(Kg#b^w&Z{qm`59dWt`eph_Oj-IwCfG~-O%WD#B21%( zkl3J!fE&e6zCLLuEL#u{(jrbquSIIwDd1dZJq4(risA4=Fr-pDp}1>53fGa_O`)<5 zx*e{YmxF!#-;a;7Z!8G^a)s}VOckk|8%W9-`GfJ3M`41;W31UM0YKm>B*G}Y&vj!& zCwqM=Y8!b>u;Ye{F%m-N^AV|1F`g=)4wZirLe|o44rE7QV+DiZpdC_ooOUJS4i{Eb z2xu9l?zBk*=x zTR{T1BpK|N5^~DzP_@L-)KKSXtu51Aj2JYF?3Ufa`p&pH_rx*H6ZNp{^W=x z9=`w7(=-D2UDlgVL|~?G=BIi$Ny031uV41^V*I{|J}sm0sd4=kg8r%;@iU_IO>hg< z-q%&qIJ2U7!MGWMXoNJrJ8yH&(4lbM+W!C?a1i;zamGl`_0>+{3lHDxflRa_l__)G zvZ+Xuvu4HK5|TiLxz00*;;+*$7@mt<^d}qbkm3+-5fRmHfZsA1d^r`cbQ@5z2*h=t ze9@?pBM~iUGWwnt<)tWXI`wt%vLUuvbs$dge1f_f3B&vFyf`Zp!@_j{l~mXQqePi2 z*!SC3*kyxS%r`z^bAx$Q>f&gznS=)9iWSu5=N8M*x?HhHwU_Zf z1D-@l5@3|}V#$(r0V(s(Imd3NlT!6+^Jp`iq_dWL`HJ z0JX4^C$+Hu!i)T5$YKg(q%Er4DPJn+{%c5!06uF>R%0{#FRII!9spXE*k}=cf5wnC zJt5CP6$5YnHrJ;GV7N5rYq*)7+jj!^UD6ze&x%fDF||O;Rdp852G1~~T}lAwW!w2) z|GB4DK+7gJ<9k1!do(5sAlMcIRokB5KJ_FUHkUu?_l&5@1p&abCWK+b z7DGSZ%pl1D48sXi?dJLInje9dtw3sU|ASM1nr>TQ7`v1uXV2zTl?b#v#a@c?dhXxl<4lg7=q^j_&TO@ACAaLA|BQ1ukl^xxc-re$sn?st`V@A^%V))%;ZVn3Z$T zCBKF9ep|npqe4A>-x_((jKE*`Vj}R_2(~c#s51ut?2aEkDcz90HV!Eo!{d@i#^0JV zBmPtIinYFKcfG!bRnEfpo*!h!*v{Tu({1$+c}-e|BqwkJkb1HKkZUf06pdB}zqisD z9T%7C!3`D9tZnl|eUr0eiP$-B^M?Oa*@0ACQh_n?d(pq9n(2ViSJ;5FW)aPI$s|6C zOvKA+Lw~mW#l7qx4Knp{o?oTnxwDs;eE{+bU>en6K6-B`@!TKS*WK4S+a#s!6?ps3 z`Tg@ppv~WUP0S;AR(bbUkISX9`<9n;h`Mwfu8WYIu~xf(Z6|LSmI3kvxt8;QY2pL# z6-fY^Q|ir^Sx{@KNuSoNJo{|t310;B(3Aj*SZz+!V^^!h-n<`Qsbc|NLZ*dt$n(E; zDW7z{bgyKqqaj0%<$^o2IciPGMgMFmXd zi`GXSJu#TfYBQEAbyNYc@9eJRevpcMZP*1aq0j^-;G6ha?~I8ph~nea40QA$b&Yp* zzPrn>tVqP6@#n?;0{>3>X>|aV`UBu*6FQZvCq|v@F#u%w59LQIvG3PFg4Lqsov6$o zGo!lMqNqR>GPGJi+3PBbCU6p-o0{v>5Oya$Lig}(1W6l}!~ z=qk^ae|8siR9|2LbcOOl&fqPM8TKvBeV_kI2B0#<5qpzsvju{@1%0dtu8N#;${WkZlz-D(CNE?(>rG2~#qEmll9H!a%*H$i-pMF$ai( z$u@PnrY-=@R09C9viQaB{b#=ZXNH*(jN@ZGAZhiBRw!tS03Vc@wj}wV$p=`}EPH%P zK-lD>^A&~A5-4QR{xS!y$f(~Ms(yrvM+QT&{V%g0e( z_`Qti^8aCy5Pkq!eXT9-=8yk+5tCoUK&*7bl$PQ@)4}ouIEY|B_!m8QSb-ThD^&+w zluMsm{J;Mv+T~5$->jQS>vE0xnbX!@KzR!h8mb8`dzox_QX@03=p6(;kUCcsWET7K$g*zp8gs4 zkLFiJ1?Xe#ff?JKRJ7`CbuXQ2B>uAj{2uL&c~|zShgj!>Wdk39f%M6`<+ro;|FkCo~(@qRrcc%6;ZK5z5#z8t3iXS(#w`Sx?au>VTo0t>rG z?&t6CFF7k$EGZ@}=J(H1lb`}z$vLnHQIp9(4lG1QQ2tO%Jnzf%jF6$p3x=uiq8>i? zKSaaL_Gi1cJddjAAN;b ze~&ES@ihd=*s$#^zJIkX19ZSx#(EQwh zUaq;D%m1^J$(}m!HUa8+lY|KJ0=+!tt)cp-?bxRdL`c7U_H?2o=|C^th~}Q-do;g3 zpacK!O_MoVJAS*mmKL6`;J+p-h6?8P@(^?>r~F>&SJXGjxnk;sYo9Gjb) z`4ttv0oV;!zBl9&p(~$TlKY9s=YG_(lxenZ2|2f*hYb^4f{taNMvm%p$+sXPkAf7Vt|FaWMZvu3xRF-Z5=qcQ7{ z#$_7nsH$Qq5Y=@W6*C1AS6jf1S}fG=PvBkUsWH)Mi+}Ai^qF!T8sCH`@!`h>GtVay z?&K zUZ&fu#v?2C%q{AZhk{I!C4@^V_O)Pao&x~5<7D7$Nd~lVAtH0nrWzus1?65K85GJ3 zXb6}Bd5x_nl6IT=>iOKi!Z~A@Gp$%W9+#3vph_|bRQ@dhnTi_Y2q&F?wmAkypSIX* zs&fFZG{@DrZqD$pvvm)n@1xirz`6YZXafA;0`5KawRdlg6a(L@f&jiDCpsmVNk}zcjiwr78bh? z4Y^~_UWii=VPaq(^;>0eVIdPxez85s`yL2cMfKhBe-c~H2S6wN*`{7z!>L@A4K3eU zsR2R+{Lx6tVW4SP2n~1nd@vM(=a~)qzG^7aVZ0-0>jiIFHK#)031aWQxe5 z&SIQ$y$0)FfuNC#LxgPDEXT*?_W#vYk4{>j*Gv%c=ja5(tz8@-@tnI)g;G&7aoATCwcL_50ak_ zQW~!AXD{663AmrLw%72_UQjR}aAYRZa;N_vbq9G2otmCr_KoQFvk{4sFd|^7X!V16 zre*)N8i_D9HAxhLV$YV{r~fpV`UB_f=K*Uz4fub%|Cpmu_}t80n9lF9R9o${fBF-- zPThwk{&#m~@}pBzGa;w2Gt17^vv%8Pi4D60Hl~tQPp4};FkQ;=D(F>yg&&f}4~^Z_ zzq>&2oj7JBp0@)$`s!^J4-YGbH=B>kN5{Szz-1Nr{r7Jq1lNqL62kBSVV?wr@x#)^iF{ z5T+A&EGX3cIUE&)Ap;B(8$I6tUlHMV;4voj!+%-0C@@+wf2HRn?bCPv|D{Pn?u5I% zdPuPtAG}l7W6_P`?{VT={dMo%2u*e$UDiBR!{rvUdsS+;eQp6g@&X_Yqjr<3VFCZ&|ps!v&c?q_Lt-h6$&TW-gaR+>F?JZMLWs8+f@uU5*U ztyNp3mO6*h;t9Dda?q{p)a7o;pca4xsAlg^OjK?jx0~_%9x-mMIF7I$N<>6ZXjfV# zCEo{A$?p3>%dEQ`%0xe2qn@8$w6g>j5KYur*Bu6PllBur_3w%2b1k$OhbBAC)8`v) z4>!EyCg7e!;y(8J^!oK$6xIA}n@gnXUhy2DV?gWD>yLwl)9nKq7-(&`TA3~@onO0! z{+4=jPAk)8IyAVW|4;;5rr;=LOVOP{By3dq1E6;1HCZBE{)v~DuZgw9_ z4Fr=bCR0S}h)^jku`_Dvi2Rt;WvMNboLnrO8cv{fJ$6Z3E~D4B-1fh?et!{(jROT+ z7e>glR>ym0nLLiCw4^YE zMp46c9!21n^|e=-=AV&AwqpMKUo{8w(@uO3fs#_J$%t}0YCeCmlJ7S#^Vb3PeL%af zZSDsago%U9fD*Awt?@#WaE(8@+VxsuWn^_3v;s{;zS$NWUV#>KSt4b|G{s&A3oWx3 z$_dy%64pPvzk6R@!h%7{k~o#UCXOz=6pPbwlCOGsA_eBY|cr)>D! zR(|C99>GK8<7Rzh($(T#CXzdDGh3fw!DjG<#u`l@W-zSpoZYI@YP7lDV$xUeWQ@kQ zxq;QJY4gig9k~^oL8iq*oy`De>PDNo&!5;xAX>6s8muAa8u+k3N6)ZiTg+RM!PCoZ zy^tRXR+p-`bul8aF(pX)z9`ilc2|=AVf1J5n3iYC)s6)>k^L|Jl1>W@yGbfyP-NX* z%+_Kv2ql=XO}@oB!+Yx}XqGuKPj|j|ED_z{+LPMKe2W4(doaVIJXiN*SKe`p4ZX^vRE;-G_(IPOB9e*f zSZy&+`=@nre7pQ&kPR$qRJR|T@T{BA4{bX*jXmd_tsz-_P{!xRyy9`E{004UBDh(h z@`UPaHi5$6Vu(&Kh1T5oWfjYKifUFeL&~ZG_;MhZ=2q+4QfM(3ve94mWed876)NM5WA!*xl~w9m`!o5BIN#Y= z^P7#e=Y2}V%9rwioOlDc;k2VE?6;aO?!10k25avm_HueiOqqTza*X*n_$mf5M@pnR!bxw>{P zV`A7JJ-lb4OOnMGKm96cf6G-Aj!qwIw0BrnI}>Z(oJy2spw!VsK|F{VqRDu+Zdh_` zJyB|M2=Up!SM|poT>oCUNRHI1bFv9zF*2Ivm`&tj! zB(qyAchuhRF_^)X>ua~KOPtO%SqAPaycqX!EvNls@Yz)TT8+(o?AM^@s(ZB~*mPcj z9b`+DzTrRQ@dusfdZ-pc)dZDszp-vQomeU-l*!_j{5ZBeNSdU3gh$=W$uX%H{d(7= zgO(A5l7r&>k*h71&huE8^$O4qAu+wVK0UfxLG(4JJ@q*67SMBsfZi~&WZa)halFWh z_rB~E&@kET(%|)BDvW5jd(HEu)~VNmI_|h-(K(x+Y~uH8Nj=C9bpiPVYF^hubnwuagqYBL ztmfQ=Smf~(@laS+fkGL)K z2(O}cA4WozDVM1`VsfRTsz=LM$=hs3(b{v)&r`f29!y-%guFcecuJU zX&KNOqTYeQix=b?QldgCYZs8MdZX99EpPNcn+&{=dqBrhI7)-iTdJgf4 z;o8y|f8;UiE(>-z+>2DD^9L{RzX)J*uG_cooqxE6d6Vc!M~@Nxz@o?t>+U5S;LUcA z4vxtwYwMxu%&i`Mucdt4$ivR>Wq6MPayi)15%)i&k#27|TSwK+;MkCO!?kHT`AdF* z-#LhT%_Nq7`>4Qc_7B&glW(6)BZOi4A-ZLgeLDJ=9E^{_%iQR!=uU^jAQ%2ybqd8c}8hnCC9KNl2dmmTQ3u1 z-PRcJprnKImbrF+NPjw$o)D^ABRU0X4~1mtu6|k1=|n;c$HXR1zUoADQLC!6{8BBA zH6W2OUTo@+T2Q5OG-os3Fh<}!Pgusxw4Rg23K1$iphg;mLal{njB26xxTs3lRi5s^ z{Hn8OR)l-*2W=fjpcM+|zR+(VzXo$K7?xB4X~X)HBFSedAtoz-oY z`z$#4qY~k9pDXQY_??+XvodsRHwFWL-xD%@rVhhDGZ3QB(k}yu5|K&>`g14FL_-~J zYSTLUGW0j`sOfMI(Bc&XXHQ36m#w0%FYHP>N4uEyS)Idp)E9mGHL_-YoAYRW9Q)pU zWbgYyi*wLxV>LHPNGk7K{FEZvNg-FLRl~&0v`^ZqR9I2SuUmc{xDu)cCp}g!R63qX z8Y_(kdMnd$B$`H)8ak(}X--VYP@H-`a|I^KfAAvi)NZo9z4)WXZ3`1btS|#qx``?D zL!=WLb$~c+(7brih5JnMjp=Fo!*1POxKe?)K|D2gO?rXPq}E!L!)r2Wtlowm#rkCn z*z|MnBt}?N)1A=3V9A)1+>T?{8Z~}8_?kC++oVbNg9KMAT`nn`@=?`AvNj5{$0yG9s7-FQcIVuupQNHXW)3rH8pZJ6<%5PQuLR6t(|rB*m3IUx zT;3JqE6(vvBO&@Y|3OcS!ozg6+yx3y-gGq(W&exxU6Kc$3uB`RgbIWLJrq(8hpdQ#7)>{JA~gu}GQdSB`)wi=>A}0U9M|4Qo%hpIOFw z6#eTze2V-0m?D(&eG6uf2X(pbgLcP`he)Q>XKh1GFSxE5sjz8G(N*YP7u8QbjLvAj z#=mYo<_QEZ-Pg~r(S8vYCI>zI^fn#NNO$NB)^j@L#|8~YpQw14Vtq&avA~NSFt(Ct zb&$#YYdD_ik(CK$p4E{;06vX(Y~M=X5$w)+mgT41=#q5Iq=6Eqt*V(67KYzDScmxV z5EkAW;fDf)>9EY&whQZ&h~5uJsb6A6{h_t(md$m9?k&po(JX^)k#7arVw+j&E*1hT zH685|lI(3L3@Z!ayHz;d&siWgLt)~RMMs?ii6u0dvClpF0c>41_@nVkCcC3<74Hwqt7K^ zy631RyvscIUE#hm(0br_O)!Os$74~26;g2_m&z0&+^m7;Cdk5Y1arg^`6f)jDqf+WtxlM)!nBGkyxV}uMTkC zd6h6^9Z!Q&Vd`4ev&%A8(!4r;ImKdX!_I64ehNBKzEOR^aTKT@t~(8@4j5Xfy^)UQcJc5+}ixXJ*HIGuv}l5V~M^v6>gaiQ+kTB)BgR^JW04{>6yj zW-0KUA>AzRq>V)M2i<2j4 z0i?Kyhpd7W*26c8Vh8QRE=Dr?NYII^74S;Y{@z51q?Fr%QdHz+4P%XAOz9WAwbG<> zE#|S|IL?DP0y(YY%d+=}e70{RHtT#`ssdtAj~xTC=^CuJpk{n0X;9ZP>8@_)rgiay z;QWOnRAWz}D5C7K8K@g2Cz|9Tz53B^ZFU8AdB_cLv9R{mw4vm){_r2-A0fPgMG1_`2*XKCbn)h)*1*Gs(Op%QRalI0TUyqGhjk>Z zs(w8y6jf^;*oit&5&lS3_c1-dpYcr^M*z6WwfB`g*CR1X1;fE{BlLsbODT3oL>T)> zxQ}nDcVCsa8P&}Tz1^4jO@NPd?7JQul*tLp!wy@!PUO`G=g5k3oh@lw%@3`uUgn_I zq45Rpb`@XgJe`k@aefyTrX7zvnjV~V`01V$myq!0lehQbpk`ehp>1>|lN0a5Qiyy& zZ_`T9e$CDzm|<1+D%faVPJ)mSl6C+Gu9S=uOyRedN-7)HY5gSP5Kc0HF3s!PY$GA7 z+i@1G>2p}}b~v!Dhk<6yKI{5bP^jMQvH5uI&-ew`g1~~|xFb;|v_~~OOVq>Y4 z#~4YaLIyJy`N6nOcTQ76q3Z_oylUoi3^+5H3dEX4xoH&N3J=pe7!njR1nOCxvEEWL zX^5W?&SqegU8Nlg4p`xyGzuka@L1(5Q*#r!%Q9*?`noobK>qDdv^90wHr{766vA52>;YFFNvOcyCKu zz_%Dx^EkJ1R#L)D$c%&>!}phu4LH7ZCfCW5nDd$0(lh+3Dq<5J5b-Q~t(fSMTeFS^C z!%D6V{82IHmCT0Hlg$ncR^Xr+!?32rj_eXtVX5J;=mI^3Q=IpY>K^nByNWt~I-c}) zY16gu23loFme!*tm4Nuh6Ht*9a}`e93ZW1flVG_FNX1fW?99%I%V!FTqZQ{{n+5%S zml#oA;k(f78gw9TlM79hpa$*pfj8;8mC`l%)qFHI_ZtQD4l6y*-m&hFN~C4?ME(x0 zMCuVi(DM=?APC{w-j7C2vS{B~_gp4yw-a_IapCQZ{;WkUDs+0tHt>DA!G@=xufxwL z7-9Y)Ui2Qxr@jB(c+-gVOCR%w_?5OUVz38qJln#pPrf$4Y+}Y7XL?*1mVB^*xTz)> z;^T(vWGj>xLgm+jA%;6Oj9PECCA8x-NXH7Ze)BTMgm{^TrHgy!wiBk?bxuXYuFY6{ zNSRD;=X-LaE7v8bFLm7oSXu+IXTe_mfK$M$O#|Wg&b=kc}MPZKw+NMFvjUIIZXQSQ3IxAVPj8cOduMB}GI{)zHw9LjKI zNkD)VS@&qKjSUa~aP?gPH2p3UF3+n$STa7?mx;WZRaa|Cf?Wi(w8e>tUJb@mOup%* zb|M-U79uhgJD3U$n*7eF#513lw#jj)A1@8vwh6BY50oXLrHn&2+$LP)XDTvZ0N(?0 z)A_5cGq-$WR}%>ei4zi7%0~c%T2a6sXEbF1ADSvo?(VduyV$GimQ(EY=IgACpZ6W% z`cs#XB4DJ!!#V&@)s))V~ z^t=?$ZUpZHXNpYzjiO)j)ei5HO8UtkV(l{M2N%{E+b8HfKS!W=Z;`8hRJg9LKg&jMLltr2;=-u1 z@fKCUxcugur-{G7Q9TkK5(ynDtiEs7Mh9C4%;=XLh`t0~<5>QDB_dHfr6rA)S{2at&Tw0qm;*!~ok>oO)^VpW zOh=zR_=?yHMK8QSwGf}v#gmo!xSw=*Sazz($n1;L&+n z%S}f5CWFKuO;69Je@#Q1Lgf=MY8Axgy$!eXzVi21dTjvA{+$@injaCue z9q~YxAS{{I!GVe(Z$5S8{z2f*14Ex9yAJZXZPv^|60|YLtC=btYLp!e)z+Q|u2kLR z0MzYt8eQ-(kKy$(r+bz9&v>NqFwM&w${zlVH*9zlqaBPRR9r{;_5__@6gS-)>5y5t z?11i!d1Qab1NSbvU*oeh8p&`S+~P0bfO))o`7v<8mt6J4sK!cH?g?o30}BF={hJZo z(ZohG0=5FCJjJ<=V`M=lW~)}^jl(G>6nE&j4eqX(V}~1Yi!Nn{6K}tqIDVE_cfQWj z=GhAMza$rLoGm~!e>`pUPoFA>wn;7!@ZwGMz|#PSXV?RwOzUtgUU<*h)w1KLr%&xb z2Z}E!Z~ei>jm$4hCde=|=hGGqCY_!S7hJGp=W4`o946heOzV~HIAs5EGm$;9mPQJ>iAT){XY0QqQcfxL5E3hBLJ zuLzwXXDnl-JoG8frtMTR0Y$*o1PRhtpU_~)s=eD0=={4bIxM^0q9a_^T#pQS94Kz*hE`LR z6e0$OlKBW~8)3_G8knpuno_^GwowT7uNu1jM(Q<3o=}0b37(}@pvGT-cJFc}F_)E1 zNE8TWW0p?4TZ)!}A-!CJX<;}E$Yc=O&RV>U1V>I6j zUjO$5UO+~1cdy{ z|2#YH+jFi@ZnLtUWUetY$DHH+##gXWh1*XPsr$ahGT#nng%fs&UDtLy$JL?mq_&lJ z$5C(lsl*D#g1tW{?A6;G9)0&;$`sv)UHW8=f-4d}?ZgmX~>hlazU-lcj zpZ4*qy?Q^Y<{H@s-doe!J%A(i0n z{M-Diqwk&CzmwWqcKh1jNoFVc1V;_K)Y;CpByr{*^ytzMqZ?Ay#?{M!y{JO)be^w= zZStN+cj(K7E!S!wpYGB}lhr18k2Rh$XO=XtYP%{8T%(&I&)|H^cJ9dJy1qKMd+%&+ z@om@a7vE1(&tG}HyE~Se5jlJdHk1t~3bgc!Inq>R(EMVmvuFG5>68fl$;T5lQ%v4=>$*~Am% zc+}bO$9MS*ge7oB6K~SmIzeKyFghWl z5%Hi_p0x0k@Qlo&cILJG07mi}cr&$;h$j9xtgq2zZ2u;s$m7Epq_62pW-O=G^QESa zuADF0x9UvJ3)1D4)zN_ITqktqa}YTkQ;uSjxOEvABes+N^gaxL>6j`CHH75dw!`X~ zv!UWh3=HO%f!^&t<6Kx~x(w)jM_yPR{^rul*z_pKne0Dp|VKk5Y(ed(on76>GFL?m&td5PcrVdM!W*43x z?D4+p-mZ@=)^d4Yt43rnr@r{0fJV3h@H+`s701EPZij zd`!uH>7-=2nc$;dU;F-yL3Rw=_R>Pdh`s$8x_hNcNpw8dN4+-S@vkMhDyn^k88J)g zyy?VITBKe*_^5UK7EjXIoK~OqvdJ2iC)?ao+Wz(x>N-X{Z1%xacm09g7^Y863rGgh zg{T6%OZ}mu54Dh;sV-uH@D{=3R9L}xyAD4`dkEFBqdpoj|J~rqz;5e{K;g9#LJXIffRnx<$BSl-P@9b$BC~qWyL&7}& z9ACT2?ZTV=xW*qZQWAeiSh(7WX9oWq9ISlL7d*4+3zi_=E*vapUorhT1GI6t7%(Y~?_RN8TgXT|ZhZ~jpI^qad(|v~Lw?)jFG*`57QVyj302RpLs~Z>i!VvKGqHg+7 z)DwIqyqq9pt*%nZ(B%kDGA)TzngcrYvRDa2D|>~jmO`TNv3Xv${9LyeP2J= zB2s`V>pHtOK|YLbK{Rqtfn1c4-OoqkQhpomCxVNDeuD>W%1a}6=IQ3kv%U4#k55AV z@ISnCvI$1zs`5p0j+khIRr#`I{ycg~bm-1Abs@dACcXOAQN_{1w|q$@ZYL}KZ}_a( zn0e08v3ak~B>l8Iaw2*$OfS=~GNc^9)Hg~L$=(MhT3@n36$><2ZSht#`aC$Z@e-^$ z#OSPtz9-%3>x*n3-2}ur|7#EPKYY{}b$F?oamei6hEQA!MF0a3pWXmsu?{7P6Dj;g z;4kWguWO=+!jG&g0vyN>^v|^4awKH{7J0+GxG_Tc*pHV0zH#&*6!ZQc4(2u=z<8J; zcLFJVVEB=NpeRRKQdK)>i>e_oT#kG1`Fh*LaE^mo^V)?vpdwoq@h@UW@?vSAmiqyF zQYJKP&w;q88uq|9Uc5Y%`i)B znHG@gl5?Mu+4@4;O-yf`timdKP0WWr2(8|=rA}htnJz0Yy>VUr=wq?!f6Lm4&g|{* zXklZ11#cM-m!SaCQiyxS1;%<@!Su1#KEP?bNZ99o(fAj#g~bgdpgs?91h6{Kad)Eh zOd?Xi>sPgU^u=PEJ1%JT2GK0=U}7-S;SwsXDal)5bL?ie>Tf7!i8cJ5S)ns|k*>n-oz^wKftq3xbjVMsvBJhvt#$d$qy_$F(*hx5V0EKXvA4n+edF@-pe#_U^uEty zxD!5E3CzN-t4Dp@YriUi_M{H)UcutA_2@wvV()@~}Tf6EeW-#z#Ke)*4bDhPwH2Dw1_Jrs$7Rti{Y;oVlj&Gasrb)d7_2t=NCMu;z0Yh2% z>8!1O-PW7|w(aEM8nCoZ?~6e;pRy>Pw2?eb+Cpd{9T2EdfvjAkk0o6{nfwo>6YGskX@3cF&YbP z!dauXA2i10)T=jE++Pc-vE)>k3_%Zue817BdT4Z`v>~c!h>CRi2p=~%ZB;iAVPPvh zv+1{la>6pADl|ue2R_FzzeK%1?nyij5mC2k9d{!pz6rrMS>(6gA#}S_ydR99=MUVu z0yAi!gz5+*JNmDF^>ZLRfH7ITBbw{jF(Pe%0lWPEDyH-!IXD6g;Xzeuc`*p$)h)l4g@cG{o!hS z#;RbbzDmryogXz{&Y3%66d&Etn}0m|ZMt!g+SUOV&I!LL4Y`!bkLSBDD}d+;P&?w0 zT`1pn&Wiu~d!z-!?^zTo^983N#==reA}-0fedQ>KBI0ow^)7KNpPyZ?yfF$$@+A9f zB#_T9fVbo~C&le$u6)~1kj<8l<#refUQYsDIW>qJI)n1d`@@95vMd^qF_H#XgCSefRuNiQPq?M6GM?p$9sa zG?Xati|7~{oH0NruOUh|G?OuOcQF4}m;S+OmK^Jh(eY#B`Cp+rpe&C$!tsd)O&|_ z5+=z$zv1EA2;nEf^f@$h$aXPfBRAm(9eWKMgN@@(&J$leKNPa$f^OzefN$3DSs^<_ zfsflmdFH>2O=AEy*Fb-f5>=Xld4Lb&G**-qXwy$YjY~m5kc% mean(na.rm = TRUE) %>% log(base = 10)) + func <- quote(log(1:10 %>% mean(na.rm = TRUE), base = 10)) + func1 <- quote(log(mean(1:10, na.rm = TRUE), base = 10)) + pipe3 <- quote(iris %>% lm(Sepal.Length ~ Sepal.Width, data = .)) + func3 <- quote(lm(Sepal.Length ~ Sepal.Width, data = iris)) + + expect_correct(check_code(user = func, solution = pipe)) + expect_correct(check_code(user = func1, solution = pipe)) + expect_correct(check_code(user = pipe, solution = func)) + expect_correct(check_code(user = pipe, solution = func1)) + expect_correct(check_code(user = pipe, solution = pipe)) + expect_correct(check_code(user = func, solution = func1)) + expect_correct(check_code(user = func1, solution = func1)) + expect_correct(check_code(user = func3, solution = pipe3)) + expect_correct(check_code(user = pipe3, solution = func3)) + expect_correct(check_code(user = pipe3, solution = pipe3)) + +}) diff --git a/tests/testthat/test_check_result.R b/tests/testthat/test_check_result.R new file mode 100644 index 00000000..3cabd4bd --- /dev/null +++ b/tests/testthat/test_check_result.R @@ -0,0 +1,107 @@ +context("Check Result") + +# these tests are largely redundant exercises that have been tested against detect_mistakes() +expect_correct <- function(x) { + expect_s3_class(x, "grader_result") + expect_true(x$correct) +} +expect_message <- function(x, message, correct = FALSE) { + expect_s3_class(x, "grader_result") + expect_equal(x$correct, correct) + expect_true(grepl(message, paste0(x$message, collapse = ""), fixed = TRUE)) +} + +test_that("Spots differences in atomics", { + + user <- quote(1) + + expect_correct( + check_result( + user = user, + results( + result(1, correct = TRUE) + ) + ) + ) + + user <- quote(2/2) + expect_correct( + check_result( + user = user, + results( + result(1, correct = TRUE) + ) + ) + ) + + user <- quote(3/2) + expect_message( + check_result( + user = user, + results( + result(1, correct = TRUE) + ), + incorrect = "check failed!" + ), + "check failed!" + ) +}) + + +test_that("Gives correct message", { + + # empty + expect_message( + check_result( + user = rlang::quo(NULL), + empty_msg = "NOT FOUND", + results( + result(0, correct = TRUE, message = "Result 0"), + result(1, correct = FALSE, message = "Result 1") + ) + ), + "NOT FOUND", FALSE + ) + + + user <- quote(1) + + # correct + expect_message( + check_result( + user = user, + correct = "{correct} {message}", + results( + result(1, correct = TRUE, message = "Result 1") + ) + ), + "Result 1", TRUE + ) + + # incorrect + expect_message( + check_result( + user = user, + incorrect = "{correct} {message}", + results( + result(0, correct = TRUE, message = "Result 0"), + result(1, correct = FALSE, message = "Result 1") + ) + ), + "FALSE Result 1", FALSE + ) + + # not found + expect_message( + check_result( + user = user, + incorrect = "found: {rlang::`%||%`(message, '')}", + results( + result(0, correct = TRUE, message = "Result 0"), + result(2, correct = FALSE, message = "Result 1") + ) + ), + "found: ", FALSE + ) + +}) diff --git a/tests/testthat/test_detect_mistakes.R b/tests/testthat/test_detect_mistakes.R index c9920a40..c19fbca4 100644 --- a/tests/testthat/test_detect_mistakes.R +++ b/tests/testthat/test_detect_mistakes.R @@ -1,405 +1,404 @@ context("Check code with calls") -library(tidyverse) a <- function(x) x b <- function(x) x test_that("detect_mistakes detects surplus code", { - + # function user <- quote(a(b(1))) solution <- quote(b(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "a(b(1))", that = quote(b())) ) - + user <- quote(b(b(1))) solution <- quote(b(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = quote(1)) ) - + user <- quote(a(b(1))) solution <- quote(a(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = quote(1)) ) - + # non-function user <- quote(1(a(1))) solution <- quote(a(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1(a(1))", that = "a()") ) - + # internal atomic # arguments user <- quote(b(1)) solution <- quote(b()) expect_equal( detect_mistakes(user, solution) - , + , surplus_argument(quote(b()), quote(1)) ) - + # internal non-function user <- quote(a(1(1))) solution <- quote(a(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1(1)", that = quote(1)) ) }) test_that("detect_mistakes detects missing code", { - + # function user <- quote(b(1)) solution <- quote(a(b(1))) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = quote(a())) ) - + # non-function user <- quote(1(1)) solution <- quote(a(b(1))) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1(1)", that = quote(a())) ) - + # internal atomic user <- quote(a()) solution <- quote(a(1)) expect_equal( detect_mistakes(user, solution) - , + , missing_argument(this_call = quote(a()), that = quote(1)) ) - + # internal function user <- quote(a(1)) solution <- quote(a(b(1))) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = quote(b())) ) - + # internal non-function would not appear in a solution }) test_that("detect_mistakes detects mis-matched code", { - + # function user <- quote(b(1)) solution <- quote(a(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = quote(a())) ) - + # non-function user <- quote(1(1)) solution <- quote(a(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1(1)", that = quote(a())) ) - + # internal atomic user <- quote(a(1)) solution <- quote(a(2)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = quote(2)) ) - + # internal function user <- quote(a(b(1))) solution <- quote(a(c(1))) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = quote(c())) ) - + # internal non-function user <- quote(a(1(1))) solution <- quote(a(b(1))) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1(1)", that = quote(b())) ) - + }) test_that("detect_mistakes works with atomic solutions", { - + user <- quote(2) solution <- quote(1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "2", that = quote(1)) ) - + # function user <- quote(a(1)) solution <- quote(1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "a(1)", that = quote(1)) ) - + user <- quote(a()) solution <- quote(1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "a()", that = quote(1)) ) - + user <- quote(a(1)) solution <- quote(pi) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "a(1)", that = quote(pi)) ) - + # non-function user <- quote(pi(1)) solution <- quote(pi) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "pi(1)", that = quote(pi)) ) - - # internal atomics, functions, non-functions, infixes, - # and pipes will not matter if the above tests pass. - # Why? Because checking will stop at the initial call + + # internal atomics, functions, non-functions, infixes, + # and pipes will not matter if the above tests pass. + # Why? Because checking will stop at the initial call # because it is not an atomic. - + }) test_that("detect_mistakes works with infix operators", { - + # surplus user <- quote(b(1 + 2)) solution <- quote(b(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value("1 + 2", quote(1)) ) - + # missing user <- quote(sqrt(1)) solution <- quote(sqrt(1 + 2)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = "1 + 2") ) - + user <- quote(sqrt(1)) solution <- quote(sqrt(1 + 2 + 3)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = "1 + 2 + 3") ) - + user <- quote(sqrt(1 + 2)) solution <- quote(sqrt(1 + 2 + 3)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ 3") ) - + user <- quote(sqrt(1 + 3)) solution <- quote(sqrt(1 + 2 + 3)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1", that = "1 + 2") ) - + # internal infix user <- quote(a(1 + 2)) solution <- quote(a(1 + 3)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ 3") ) - + user <- quote(a(1 + 2 + 4)) solution <- quote(a(1 + 3 + 4)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ 3") ) - + user <- quote(a(1 + 2 + 4)) solution <- quote(a(1 + 3 + 5)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 4", that = "+ 5") ) - + user <- quote(a(2 + 1)) solution <- quote(a(3 + 1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "2", that = "3") ) - + user <- quote(a(1 + 1)) solution <- quote(a(1 - 1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1 + 1", that = "1 - 1") ) - + user <- quote(a(1 + 1 + 1)) solution <- quote(a(1 - 1 + 1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1 + 1", that = "1 - 1") ) - + # surplus user <- quote(1 + 2) solution <- quote(1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value("1 + 2", quote(1)) ) - + # missing user <- quote(1) solution <- quote(1 + 2) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = "1 + 2") ) - + user <- quote(1) solution <- quote(1 + 2 + 3) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = "1 + 2 + 3") ) - + user <- quote(1 + 2) solution <- quote(1 + 2 + 3) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ 3") ) - + user <- quote(1 + 3) solution <- quote(1 + 2 + 3) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1", that = "1 + 2") ) - + # internal infix user <- quote(1 + 2) solution <- quote(1 + 3) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ 3") ) - + user <- quote(1 + 2 + 4) solution <- quote(1 + 3 + 4) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ 3") ) - + user <- quote(1 + 2 + 4) solution <- quote(1 + 3 + 5) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 4", that = "+ 5") ) - + user <- quote(2 + 1) solution <- quote(3 + 1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "2", that = "3") ) - + user <- quote(1 + 1) solution <- quote(1 - 1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1 + 1", that = "1 - 1") ) - + user <- quote(1 + 1 + 1) solution <- quote(1 - 1 + 1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1 + 1", that = "1 - 1") ) - + # function user <- quote(a(1)) solution <- quote(1 + pi) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "a(1)", that = "1 + pi") ) - + user <- quote(b(1)) solution <- quote(b(1) + 2) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = "b(1) + 2") ) @@ -407,7 +406,7 @@ test_that("detect_mistakes works with infix operators", { solution <- quote(b(1) + a(2)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = "b(1) + a(2)") ) @@ -416,7 +415,7 @@ test_that("detect_mistakes works with infix operators", { solution <- quote(1 + pi) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "pi(1)", that = "1 + pi") ) @@ -424,217 +423,217 @@ test_that("detect_mistakes works with infix operators", { solution <- quote(b(1) + 2) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "1(1)", that = "b(1) + 2") ) - - # internal atomics, functions, non-functions, infixes, - # and pipes will not matter if the above tests pass. - # Why? Because checking will stop at the initial call + + # internal atomics, functions, non-functions, infixes, + # and pipes will not matter if the above tests pass. + # Why? Because checking will stop at the initial call # because it is not an infix. - + }) test_that("detect_mistakes works with pipes", { - + # internal pipe user <- quote(b(1 %>% abs())) solution <- quote(b(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value("abs(1)", quote(1)) ) - + user <- quote(sqrt(1)) solution <- quote(sqrt(1 %>% log())) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = quote(log())) ) - + user <- quote(sqrt(1)) solution <- quote(sqrt(1 %>% log() %>% abs())) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = quote(abs())) ) - + user <- quote(sqrt(1 %>% log())) solution <- quote(sqrt(1 %>% log() %>% abs())) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "log(1)", that = quote(abs())) ) - + user <- quote(sqrt(1 + 2)) solution <- quote(sqrt(1 + 2 %>% log())) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ log()") ) - + # internal pipe user <- quote(a(2 %>% abs())) solution <- quote(a(2 %>% log())) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "abs(2)", that = "log()") ) - + user <- quote(a(2 %>% abs() %>% sqrt())) solution <- quote(a(2 %>% log() %>% sqrt())) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "abs(2)", that = "log()") ) - + user <- quote(a(2 %>% abs())) solution <- quote(a(2 + log(1))) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "abs(2)", that = "2 + log(1)") ) - + # exernal pipe user <- quote(1 %>% abs()) solution <- quote(1) expect_equal( detect_mistakes(user, solution) - , + , wrong_value("abs(1)", quote(1)) ) - + user <- quote(1) solution <- quote(1 %>% log()) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = quote(log())) ) - + user <- quote(1) solution <- quote(1 %>% log() %>% abs()) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = quote(1), that = quote(abs())) ) - + user <- quote(1 %>% log()) solution <- quote(1 %>% log() %>% abs()) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "log(1)", that = quote(abs())) ) - + user <- quote(1 + 2) solution <- quote(1 + 2 %>% log()) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "+ 2", that = "+ log()") ) - + # internal pipe user <- quote(2 %>% abs()) solution <- quote(2 %>% log()) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "abs(2)", that = "log()") ) - + user <- quote(2 %>% abs() %>% sqrt()) solution <- quote(2 %>% log() %>% sqrt()) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "abs(2)", that = "log()") ) - + user <- quote(2 %>% abs()) solution <- quote(2 + log(1)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "abs(2)", that = "2 + log(1)") ) - + user <- quote(b(1)) solution <- quote(b(1) %>% a()) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "b(1)", that = "a()") ) }) test_that("detect_mistakes handles argument names correctly", { - + user <- quote(c(x = a(b(1)))) solution <- quote(c(x = b(1))) expect_equal( detect_mistakes(user, solution) - , - wrong_value(this = "a(b(1))", + , + wrong_value(this = "a(b(1))", that = quote(b()), this_name = "x", that_name = "x") ) - + user <- quote(b(x = 1)) solution <- quote(b()) expect_equal( detect_mistakes(user, solution) - , - surplus_argument(this_call = quote(b()), - this = quote(1), + , + surplus_argument(this_call = quote(b()), + this = quote(1), this_name = "x") ) - + user <- quote(b(x = a(1))) solution <- quote(b()) expect_equal( detect_mistakes(user, solution) - , - surplus_argument(this_call = quote(b()), - this = "a(1)", + , + surplus_argument(this_call = quote(b()), + this = "a(1)", this_name = "x") ) - + user <- quote(mean(1:10, cut = 1, na.rm = TRUE)) solution <- quote(mean(1:10, trim = 1, na.rm = TRUE)) expect_equal( detect_mistakes(user, solution) - , - wrong_value(this = quote(1), + , + wrong_value(this = quote(1), this_name = "cut", that = quote(1), that_name = "trim") ) - + user <- quote(mean(1:10, cut = 1, na.rm = TRUE)) solution <- quote(mean(1:10, 1, na.rm = TRUE)) expect_null( detect_mistakes(user, solution) ) - + user <- quote(mean(1:10, cut = 1, na.rm = TRUE)) solution <- quote(mean(1:10, TRUE, cut = 1)) expect_equal( detect_mistakes(user, solution) - , - wrong_value(this = quote(1), + , + wrong_value(this = quote(1), this_name = "cut", that = quote(TRUE)) ) @@ -642,22 +641,22 @@ test_that("detect_mistakes handles argument names correctly", { }) test_that("detect_mistakes handles weird cases", { - + user <- quote(sum(sum(1, 2), 3)) solution <- quote(sum(1, 2, 3)) expect_equal( detect_mistakes(user, solution) - , + , wrong_value(this = "sum(1, 2)", that = quote(1)) ) - + user <- quote(sum(1, 2)) solution <- quote(sum(1, 2, 3)) expect_equal( detect_mistakes(user, solution) - , - missing_argument(this_call = quote(sum()), + , + missing_argument(this_call = quote(sum()), that = quote(3)) ) - -}) \ No newline at end of file + +}) diff --git a/tests/testthat/test_strict_check.R b/tests/testthat/test_strict_check.R deleted file mode 100644 index 24d80df3..00000000 --- a/tests/testthat/test_strict_check.R +++ /dev/null @@ -1,197 +0,0 @@ -library(tidyverse) -context("Strict check") - -# these tests are largely redundant exercises that have been tested against detect_mistakes() - -test_that("Spots differences in atomics", { - - user <- quote(1) - solution <- quote(1) - expect_null( - strict_check(user = user, solution = solution) - ) - - user <- quote(1) - solution <- quote(2) - expect_equal( - strict_check(user = user, solution = solution) - , - wrong_value(this = quote(1), that = quote(2)) - ) -}) - -test_that("Spots differences in names", { - - user <- quote(x) - solution <- quote(y) - expect_equal( - strict_check(user = user, solution = solution) - , - wrong_value(this = quote(x), that = quote(y)) - ) - - user <- quote(x) - solution <- quote(x) - expect_null( - strict_check(user = user, solution = solution) - ) - - user <- quote(5) - solution <- quote(y) - expect_equal( - strict_check(user = user, solution = solution) - , - wrong_value(this = quote(5), that = quote(y)) - ) -}) - -test_that("Spots differences in calls", { - a <- quote(map(lists, mean, na.rm = TRUE)) - b <- quote(map(vecs, mean, na.rm = TRUE)) - c <- quote(map(lists, mean)) - d <- quote(map(vecs, mean)) - - expect_null( - strict_check(user = a, solution = a) - ) - - expect_equal( - strict_check(user = a, solution = b) - , - wrong_value(this = quote(lists), that = quote(vecs)) - ) - - expect_equal( - strict_check(user = a, solution = c) - , - surplus_argument(this_call = "map()", - this_name = "na.rm", - this = quote(TRUE)) - ) - - expect_equal( - strict_check(user = c, solution = a) - , - missing_argument(this_call = "map()", - that_name = "na.rm", - that = quote(TRUE)) - ) -}) - -test_that("Mentions only first non-matching element", { - w <- quote(1) - x <- quote(log(1)) - y <- quote(sqrt(log(2))) - z <- quote(sqrt(log(1))) - - expect_null( - strict_check(user = w, solution = w) - ) - - expect_equal( - strict_check(user = w, solution = z) - , - wrong_value(this = quote(1), that = quote(sqrt())) - ) - - expect_equal( - strict_check(user = x, solution = z) - , - wrong_value(this = "log(1)", that = quote(sqrt())) - ) - - expect_equal( - strict_check(user = y, solution = z) - , - wrong_value(this = "2", that = quote(1)) - ) - -}) - -test_that("Spots differences in argument names", { - a <- quote(mean(1:10, trim = 1, na.rm = TRUE)) - b <- quote(mean(1:10, 1, TRUE)) - c <- quote(mean(1:10, cut = 1, na.rm = TRUE)) - - expect_null( - strict_check(user = a, solution = a) - ) - - expect_null( - strict_check(user = b, solution = a) - ) - - expect_equal( - strict_check(user = c, solution = a) - , - wrong_value(this = quote(1), this_name = "cut", - that = quote(1), that_name = "trim") - ) - -}) - -test_that("Ignore differences in argument positions (for non ... arguments)", { - a <- quote(round(x = pi, digits = 2)) - b <- quote(round(pi, digits = 2)) - c <- quote(round(2, x = pi)) - d <- quote(round(digits = 2, x = pi)) - - expect_null( - strict_check(user = b, solution = a) - ) - - expect_null( - strict_check(user = c, solution = a) - ) - - expect_null( - strict_check(user = d, solution = a) - ) - - expect_null( - strict_check(user = a, solution = d) - ) - -}) - -test_that("Returns intelligent error when no solution code", { - - expect_error( - strict_check() - , - "No solution is provided for this exercise." - ) - -}) - -test_that("Returns intelligent error when no user code", { - expect_error( - strict_check(solution = quote(5)) - , - "I didn't receive your code. Did you write any?" - ) -}) - -test_that("Spot differences when pipes are involved", { - pipe <- quote(iris %>% filter(Species == "Virginica") %>% select(Sepal.Length)) - func <- quote(select(iris %>% filter(Species == "Virginica"), Sepal.Length)) - func1 <- quote(select(filter(iris, Species == "Virginica"), Sepal.Length)) - pipe1 <- quote(iris %>% filter(Species == "Virginica") %>% select(Petal.Length)) - pipe2 <- quote(iris %>% arrange(Species) %>% select(Sepal.Length)) - pipe3 <- quote(iris %>% lm(Sepal.Length ~ Sepal.Width, data = .)) - func3 <- quote(lm(Sepal.Length ~ Sepal.Width, data = iris)) - - expect_null(strict_check(user = func, solution = pipe)) - expect_null(strict_check(user = func1, solution = pipe)) - expect_null(strict_check(user = pipe, solution = func)) - expect_null(strict_check(user = pipe, solution = func1)) - expect_null(strict_check(user = pipe, solution = pipe)) - expect_null(strict_check(user = func, solution = func1)) - expect_null(strict_check(user = func1, solution = func1)) - expect_null(strict_check(user = func3, solution = pipe3)) - expect_null(strict_check(user = pipe3, solution = func3)) - expect_null(strict_check(user = pipe3, solution = pipe3)) - -}) - - diff --git a/tests/testthat/test_test_result.R b/tests/testthat/test_test_result.R new file mode 100644 index 00000000..7648662e --- /dev/null +++ b/tests/testthat/test_test_result.R @@ -0,0 +1,94 @@ +context("Test Result") + +# these tests are largely redundant exercises that have been tested against detect_mistakes() +expect_correct <- function(x) { + expect_s3_class(x, "grader_result") + expect_true(x$correct) +} +expect_message <- function(x, message, correct = FALSE) { + expect_s3_class(x, "grader_result") + expect_equal(x$correct, correct) + expect_true(grepl(message, paste0(x$message, collapse = ""), fixed = TRUE)) +} + +test_that("Spots differences in atomics", { + + user <- quote(1) + + expect_message( + test_result( + user = user, + tests( + checkmate::expect_numeric + ) + ), + "1/1", TRUE + ) + + expect_message( + test_result( + user = user, + tests( + checkmate::expect_numeric, + checkmate::expect_character + ) + ), + "1/2", FALSE + ) + + expect_message( + test_result( + user = user, + tests( + checkmate::expect_numeric, + test("test: is character", checkmate::expect_character) + ) + ), + "test: is character", FALSE + ) +}) + + +test_that("Gives correct message", { + + # empty + expect_message( + test_result( + user = rlang::quo(NULL), + empty_msg = "NOT FOUND", + tests( + checkmate::expect_numeric + ) + ), + "NOT FOUND", FALSE + ) + + + user <- quote(1) + + # correct + expect_message( + test_result( + user = user, + correct = "{num_correct}-{num_total}", + tests( + checkmate::expect_numeric + ) + ), + "1-1", TRUE + ) + + # incorrect + expect_message( + test_result( + user = user, + incorrect = "{num_correct}-{num_total}", + tests( + checkmate::expect_numeric, + checkmate::expect_character + ) + ), + "1-2", FALSE + ) + +}) diff --git a/tests/testthat/test_unpipe.R b/tests/testthat/test_unpipe.R index 38bba53d..ed67f696 100644 --- a/tests/testthat/test_unpipe.R +++ b/tests/testthat/test_unpipe.R @@ -1,23 +1,22 @@ -library(tidyverse) context("Unpipe pipes") test_that("unpipe() strips off the top level of piping", { pipe <- quote(iris %>% filter(Species == "Virginica") %>% select(Sepal.Length)) func <- quote(select(iris %>% filter(Species == "Virginica"), Sepal.Length)) - + expect_equal(unpipe(pipe), func) }) test_that("unpipe() recognizes . syntax", { pipe2 <- quote(iris %>% lm(Sepal.Length ~ Sepal.Width, data = .)) func2 <- quote(lm(Sepal.Length ~ Sepal.Width, data = iris)) - + expect_equal(unpipe(pipe2), func2) }) test_that("unpipe() does not alter unpiped code", { func2 <- quote(lm(Sepal.Length ~ Sepal.Width, data = iris)) - + expect_equal(unpipe(func2), func2) -}) \ No newline at end of file +})