feat: throw warning when prediction and measure type do not match (#1188

)

* feat: throw warning when prediction and measure type do not match

* ...

NEWS.md

@@ -1,5 +1,6 @@
 # mlr3 (development version)
 
+* feat: Throw warning when prediction and measure type do not match.
 * fix: The `mlr_reflections` were broken when an extension package was not loaded on the workers.
   Extension packages must now register themselves in the `mlr_reflections$loaded_packages` field.
 

R/Measure.R

@@ -194,7 +194,7 @@ Measure = R6Class("Measure",
     #'
     #' @return `numeric(1)`.
     score = function(prediction, task = NULL, learner = NULL, train_set = NULL) {
-      assert_measure(self, task = task, learner = learner)
+      assert_measure(self, task = task, learner = learner, prediction = prediction)
       assert_prediction(prediction, null.ok = "requires_no_prediction" %nin% self$properties)
 
       if ("requires_task" %in% self$properties && is.null(task)) {

R/assertions.R

@@ -194,8 +194,9 @@ assert_predictable = function(task, learner) {
 
 #' @export
 #' @param measure ([Measure]).
+#' @param prediction ([Prediction]).
 #' @rdname mlr_assertions
-assert_measure = function(measure, task = NULL, learner = NULL, .var.name = vname(measure)) {
+assert_measure = function(measure, task = NULL, learner = NULL, prediction = NULL, .var.name = vname(measure)) {
   assert_class(measure, "Measure", .var.name = .var.name)
 
   if (!is.null(task)) {
@@ -236,6 +237,13 @@ assert_measure = function(measure, task = NULL, learner = NULL, .var.name = vnam
     }
   }
 
+  if (!is.null(prediction)) {
+    # same as above but works without learner e.g. measure$score(prediction)
+    if (measure$check_prerequisites != "ignore" && measure$predict_type %nin% prediction$predict_types) {
+      warningf("Measure '%s' is missing predict type '%s' of prediction", measure$id, measure$predict_type)
+    }
+  }
+
   invisible(measure)
 }
 

tests/testthat/test_Measure.R

@@ -188,3 +188,13 @@ test_that("checks on predict_sets", {
   expect_error({m$predict_sets = "imaginary"}, "Must be a subset")
 })
 
+test_that("measure and prediction type is checked", {
+  learner = lrn("classif.rpart")
+  task = tsk("pima")
+  learner$train(task)
+  pred = learner$predict(task)
+
+  measure = msr("classif.logloss")
+  expect_warning(measure$score(pred), "is missing predict type")
+})
+