KNN: Binary Classification

# nolint start
library(mlexperiments)

See https://github.com/kapsner/mlexperiments/blob/main/R/learner_knn.R for implementation details.

Preprocessing

Import and Prepare Data

library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
  data.table::as.data.table() |>
  na.omit()

seed <- 123
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"
to_num <- c(
  "Cl.thickness",
  "Cell.size",
  "Cell.shape",
  "Marg.adhesion",
  "Epith.c.size"
)
dataset[, (to_num) := lapply(.SD, as.numeric), .SDcols = to_num]

General Configurations

seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
  # on cran
  ncores <- 2L
} else {
  ncores <- ifelse(
    test = parallel::detectCores() > 4,
    yes = 4L,
    no = ifelse(
      test = parallel::detectCores() < 2L,
      yes = 1L,
      no = parallel::detectCores()
    )
  )
}
options("mlexperiments.bayesian.max_init" = 4L)

Generate Training- and Test Data

data_split <- splitTools::partition(
  y = dataset[, get(target_col)],
  p = c(train = 0.7, test = 0.3),
  type = "stratified",
  seed = seed
)

train_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L


test_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1L

Generate Training Data Folds

fold_list <- splitTools::create_folds(
  y = train_y,
  k = 3,
  type = "stratified",
  seed = seed
)

Experiments

Prepare Experiments

# required learner arguments, not optimized
learner_args <- list(
  l = 2,
  test = parse(text = "fold_test$x"),
  use.all = FALSE
)

# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- list(type = "response")
performance_metric <- metric("ACC")
performance_metric_args <- NULL
return_models <- FALSE

# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
  k = seq(4, 68, 6)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
  set.seed(123)
  sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
  parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}

# required for bayesian optimization
parameter_bounds <- list(k = c(2L, 80L))
optim_args <- list(
  n_iter = ncores,
  kappa = 3.5,
  acq = "ucb"
)

Hyperparameter Tuning

tuner <- mlexperiments::MLTuneParameters$new(
  learner = LearnerKnn$new(),
  strategy = "grid",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_grid <- tuner$execute(k = 3)

head(tuner_results_grid)
#>    setting_id metric_optim_mean     k     l use.all
#>         <int>             <num> <num> <num>  <lgcl>
#> 1:          1        0.03979364    16     2   FALSE
#> 2:          2        0.06281546    64     2   FALSE
#> 3:          3        0.03771031    10     2   FALSE
#> 4:          4        0.04393394    34     2   FALSE
#> 5:          5        0.05862242    58     2   FALSE
#> 6:          6        0.04393394    28     2   FALSE

Bayesian Optimization

tuner <- mlexperiments::MLTuneParameters$new(
  learner = LearnerKnn$new(),
  strategy = "bayesian",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds

tuner$learner_args <- learner_args
tuner$optim_args <- optim_args

tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.016  Round = 1   k = 10.0000 Value = -0.03771031
#> elapsed = 0.02   Round = 2   k = 4.0000  Value = -0.04607001
#> elapsed = 0.014  Round = 3   k = 64.0000 Value = -0.06281546
#> elapsed = 0.013  Round = 4   k = 52.0000 Value = -0.06070576
#> elapsed = 0.012  Round = 5   k = 25.0000 Value = -0.0418506
#> elapsed = 0.015  Round = 6   k = 80.0000 Value = -0.05860932
#> elapsed = 0.011  Round = 7   k = 17.0000 Value = -0.03976727
#> elapsed = 0.013  Round = 8   k = 37.0000 Value = -0.04811371
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.03771031

head(tuner_results_bayesian)
#>    setting_id     k       Value     l use.all metric_optim_mean
#>         <int> <num>       <num> <num>  <lgcl>             <num>
#> 1:          1    10 -0.03771031     2   FALSE        0.03771031
#> 2:          2     4 -0.04607001     2   FALSE        0.04607001
#> 3:          3    64 -0.06281546     2   FALSE        0.06281546
#> 4:          4    52 -0.06070576     2   FALSE        0.06070576
#> 5:          5    25 -0.04185060     2   FALSE        0.04185060
#> 6:          6    80 -0.05860932     2   FALSE        0.05860932

k-Fold Cross Validation

validator <- mlexperiments::MLCrossValidation$new(
  learner = LearnerKnn$new(),
  fold_list = fold_list,
  ncores = ncores,
  seed = seed
)

validator$learner_args <- tuner$results$best.setting

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3

head(validator_results)
#>      fold performance     k     l use.all
#>    <char>       <num> <num> <num>  <lgcl>
#> 1:  Fold1   0.9746835    10     2   FALSE
#> 2:  Fold2   0.9496855    10     2   FALSE
#> 3:  Fold3   0.9625000    10     2   FALSE

Nested Cross Validation

validator <- mlexperiments::MLNestedCV$new(
  learner = LearnerKnn$new(),
  strategy = "grid",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>

head(validator_results)
#>      fold performance     k     l use.all
#>    <char>       <num> <num> <num>  <lgcl>
#> 1:  Fold1   0.9746835    10     2   FALSE
#> 2:  Fold2   0.9496855    10     2   FALSE
#> 3:  Fold3   0.9625000    10     2   FALSE

Inner Bayesian Optimization

validator <- mlexperiments::MLNestedCV$new(
  learner = LearnerKnn$new(),
  strategy = "bayesian",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"


validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.009  Round = 1   k = 10.0000 Value = -0.04699348
#> elapsed = 0.008  Round = 2   k = 4.0000  Value = -0.05639805
#> elapsed = 0.01   Round = 3   k = 64.0000 Value = -0.07523658
#> elapsed = 0.01   Round = 4   k = 52.0000 Value = -0.07209193
#> elapsed = 0.009  Round = 5   k = 25.0000 Value = -0.05331217
#> elapsed = 0.011  Round = 6   k = 80.0000 Value = -0.08152589
#> elapsed = 0.009  Round = 7   k = 17.0000 Value = -0.05016752
#> elapsed = 0.011  Round = 8   k = 37.0000 Value = -0.06271675
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.04699348
#>
#> CV fold: Fold2
#> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.009  Round = 1   k = 10.0000 Value = -0.02830189
#> elapsed = 0.009  Round = 2   k = 4.0000  Value = -0.0408805
#> elapsed = 0.01   Round = 3   k = 64.0000 Value = -0.07232704
#> elapsed = 0.01   Round = 4   k = 52.0000 Value = -0.05974843
#> elapsed = 0.01   Round = 5   k = 25.0000 Value = -0.03773585
#> elapsed = 0.01   Round = 6   k = 80.0000 Value = -0.08490566
#> elapsed = 0.01   Round = 7   k = 17.0000 Value = -0.02830189
#> elapsed = 0.01   Round = 8   k = 39.0000 Value = -0.05031447
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.02830189
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.01   Round = 1   k = 10.0000 Value = -0.04117999
#> elapsed = 0.008  Round = 2   k = 4.0000  Value = -0.0442947
#> elapsed = 0.01   Round = 3   k = 64.0000 Value = -0.06004792
#> elapsed = 0.01   Round = 4   k = 52.0000 Value = -0.05687332
#> elapsed = 0.009  Round = 5   k = 27.0000 Value = -0.04432465
#> elapsed = 0.009  Round = 6   k = 17.0000 Value = -0.04432465
#> elapsed = 0.011  Round = 7   k = 80.0000 Value = -0.06951183
#> elapsed = 0.01   Round = 8   k = 38.0000 Value = -0.05372866
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.04117999

head(validator_results)
#>      fold performance     k     l use.all
#>    <char>       <num> <num> <num>  <lgcl>
#> 1:  Fold1   0.9746835    10     2   FALSE
#> 2:  Fold2   0.9496855    10     2   FALSE
#> 3:  Fold3   0.9625000    10     2   FALSE