## ----setup, include=FALSE-----------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  fig.width = 6,
  fig.height = 4,
  dpi = 110
)

## ----data---------------------------------------------------------------------
library(uniml)

data <- iris
data$Species <- factor(data$Species)
table(data$Species)

spec <- uniml_spec(
  task_type = "multiclass",
  outcome = "Species"
)
spec <- set_learners(spec, c("cart", "rf"), tune = TRUE)
spec

## ----task---------------------------------------------------------------------
prepared <- as_uniml_data(data, spec)
levels(prepared$outcome)

task <- make_task(data, spec)
task$class_names
task$positive

## ----train--------------------------------------------------------------------
learner <- train_model(
  task,
  mlr3::lrn("classif.rpart", predict_type = "prob")
)
prediction <- learner$predict(task)
head(prediction$prob, 3)

## ----metrics------------------------------------------------------------------
metrics <- calc_metrics(prediction)
metrics[setdiff(names(metrics), c("confusion", "per_class"))]

## ----confusion----------------------------------------------------------------
metrics$confusion
metrics$per_class

## ----macro--------------------------------------------------------------------
data.frame(
  sensitivity = metrics$sensitivity,
  balanced_accuracy = metrics$balanced_accuracy,
  accuracy = metrics$accuracy,
  kappa = metrics$kappa
)

## ----cv-----------------------------------------------------------------------
resampling <- mlr3::rsmp("cv", folds = 5L)
resampling$instantiate(task)
cv <- mlr3::resample(task, learner, resampling, store_models = FALSE)

pooled <- as_uniml_multiclass(cv$prediction(), truth = cv$prediction()$truth)
cv_metrics <- calc_metrics(pooled)
round(cv_metrics$auc, 3)
round(cv_metrics$accuracy, 3)

## ----tune---------------------------------------------------------------------
learners <- resolve_learners(spec, task = task)
tuned <- tune_model(
  task,
  learners[["cart"]],
  resampling = mlr3::rsmp("cv", folds = 3L),
  tuner = "random_search",
  terminator = mlr3tuning::trm("evals", n_evals = 4L),
  seed = 123
)
tuned$status
tuned$best_params

## ----rank---------------------------------------------------------------------
score_row <- function(model, prediction) {
  values <- calc_metrics(prediction)
  data.frame(
    model = model,
    set = "validation",
    auc = values$auc,
    accuracy = values$accuracy,
    balanced_accuracy = values$balanced_accuracy,
    f1 = values$f1,
    stringsAsFactors = FALSE
  )
}

forest <- train_model(task, mlr3::lrn("classif.ranger", predict_type = "prob"))
comparison <- rbind(
  score_row("cart", prediction),
  score_row("rf", forest$predict(task))
)
comparison
rank_models(comparison, task_type = "multiclass")

## ----roc, fig.cap = "One-vs-rest ROC curves."---------------------------------
binary_prediction <- as_uniml_multiclass(prediction)
plot_roc(binary_prediction)

## ----confusion-plot, fig.cap = "Confusion matrix: reference classes in rows, predicted in columns."----
plot_confusion(binary_prediction)

## ----calibration--------------------------------------------------------------
calibration <- multiclass_calibration(prediction$prob, prediction$truth)
calibration$summary
calibration$per_class

## ----calibration-plot, fig.cap = "Reliability of the multiclass predictions: top-label confidence (left) and one-vs-rest per class (right)."----
plot_calibration(binary_prediction, scope = "both")

## ----calibration-metrics------------------------------------------------------
metrics <- calc_metrics(prediction)
round(metrics$ece, 3)
round(metrics$ece_classwise, 3)
metrics$per_class[, c("class", "ece", "mce")]

## ----recalibrate--------------------------------------------------------------
recalibrated <- calibrate_probabilities(binary_prediction, folds = 5L)
recalibrated$parameters$fitted
recalibrated$metrics_before[, c("accuracy", "ece", "logloss")]
recalibrated$metrics_after[, c("accuracy", "ece", "logloss")]

## ----recalibrate-plot, fig.cap = "The same predictions after temperature scaling."----
plot_calibration(
  as_uniml_multiclass(recalibrated$calibrated, truth = binary_prediction$truth),
  scope = "top_label"
)

