## ----setup, include=FALSE-----------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  fig.width = 6,
  fig.height = 4,
  dpi = 110
)

## ----specification------------------------------------------------------------
library(uniml)

development <- read.csv(
  system.file("extdata", "uniml-cvd-mortality.csv", package = "uniml")
)
new_admissions <- read.csv(
  system.file("extdata", "uniml-cvd-mortality-new.csv", package = "uniml")
)

set.seed(123)
development <- development[1:800, ]
new_admissions <- new_admissions[1:400, ]

spec <- uniml_spec(
  task_type = "binary",
  outcome = "died",
  positive = "yes",
  id = "id"
)
spec <- set_preprocess(
  spec,
  impute = "median_mode",
  scale = "zscore",
  encode = "onehot"
)
spec <- set_feature_selection(
  spec,
  methods = c("univariate", "lasso"),
  combine = "intersect"
)
spec <- set_learners(spec, c("lr", "cart"), tune = FALSE)
spec <- set_evaluation(
  spec,
  ci = "bootstrap",
  threshold_method = "youden",
  calibration = "platt",
  select_on = "validation"
)
# Kernel SHAP is the most expensive step of a workflow, so its two size knobs are
# part of the specification. They are small here to keep the vignette quick; a
# paper-quality explanation wants a few hundred rows on both sides.
spec <- set_explain(spec, n_explain = 20, bg_n = 20)
spec

## ----split--------------------------------------------------------------------
split <- make_split(
  as_uniml_data(development, spec),
  prop = c(0.8, 0.2),
  stratify = TRUE,
  seed = 123
)

data.frame(
  set = c("train", "validation"),
  n = c(nrow(split$train$data), nrow(split$validation$data)),
  events = c(
    sum(split$train$data$died == "yes"),
    sum(split$validation$data$died == "yes")
  )
)

## ----workflow-----------------------------------------------------------------
result <- run_workflow(
  spec,
  train = split$train$data,
  validation = split$validation$data,
  test = new_admissions[1:200, ],
  external = new_admissions[201:400, ],
  ci_R = 25
)

result

## ----metrics------------------------------------------------------------------
metrics <- result$metrics
metrics[, c("model", "set", "probability", "auc", "brier", "accuracy")]

## ----ci-----------------------------------------------------------------------
head(result$ci[result$ci$set == "validation", ], 4)

## ----ranking------------------------------------------------------------------
result$ranking[, c("model", "score", "rank", "metric", "set")]

## ----cv-----------------------------------------------------------------------
cv_auc <- subset(result$cv$summary, metric == "auc")
cv_auc[, c("model", "mean", "sd", "n")]

## ----roc, fig.cap = "ROC curves on the held-out validation set."--------------
plot_roc(result, set = "validation")

## ----calibration, fig.cap = "Calibration on the validation set, raw and calibrated probabilities."----
plot_calibration(result, set = "validation")

## ----dca, fig.cap = "Decision curve analysis."--------------------------------
plot_dca(result)

## ----confusion, fig.cap = "Confusion matrix at the Youden threshold."---------
plot_confusion(result, models = "best")

## ----role-table---------------------------------------------------------------
role_table <- subset(
  result$metrics,
  set %in% c("validation", "test", "external"),
  select = c("model", "set", "probability", "auc", "brier", "sensitivity",
             "specificity")
)
role_table[role_table$probability == "calibrated", ]

## ----external-roc, fig.cap = "ROC curves on the external cohort."-------------
plot_roc(result, set = "external")

## ----external-calibration, fig.cap = "Calibration on the external cohort."----
plot_calibration(result, set = "external")

## ----thresholds---------------------------------------------------------------
as.data.frame(result$meta$thresholds)

## ----nri----------------------------------------------------------------------
if (!is.null(result$tests$nri_idi)) {
  result$tests$nri_idi$summary
}

## ----shap, fig.cap = "Global SHAP importance of the best model."--------------
explained <- explain_shap(result, model = "best", n_explain = 30, bg_n = 20)
plot_shap_importance(explained)

## ----predict------------------------------------------------------------------
predicted <- predict_uniml(
  result,
  newdata = new_admissions[1:5, ],
  model = "best"
)
predicted[, c("model", "prob", "calibrated_prob", "probability", "predicted")]

## ----tripod-------------------------------------------------------------------
checklist <- tripod_ai_check(result)
table(checklist$status)
subset(checklist, status != "reported", select = c("domain", "status"))

## ----export-------------------------------------------------------------------
out <- tempfile("uniml-vignette-")
exported <- export_results(result, dir = out, include_plots = FALSE)
basename(exported$files)

## ----leakage------------------------------------------------------------------
spec_in <- set_leakage_policy(
  spec,
  mode = "strict_in",
  in_cv = c("selection", "balance", "tuning")
)

spec_in$leakage_policy
spec_in$in_cv

