In the new version of unifiedml available on CRAN, you can benchmark different models using k-fold cross-validation (section 1 of this blog post), and there’s a unified interface for predicting model probabilities (section 2 of this blog post).

install.packages("unifiedml")install.packages(c("e1071", "randomForest", "caret"))install.packages("glmnet")library(unifiedml)

1 – Benchmarking models

set.seed(123)X <- iris[, 1:4]y <- iris$Speciesmodels <- list( # `Model` is exported from package 'unifiedml'  glm  = Model$new(caret::train), # caret can be used (see https://topepo.github.io/caret/available-models.html)  rf   = Model$new(randomForest::randomForest), # or a native pkg  svm  = Model$new(e1071::svm) # or another pkg)params <- list(  glm = list(method = "glmnet",             tuneGrid = data.frame(alpha = 0, lambda = 0.01), # for caret model, all hyperparameters must be provided             trControl = trainControl(method = "none")),  rf  = list(ntree = 150), # Not necessarily needing to specify all hyperparameters  svm = list(kernel = "radial",             cost = 1,             gamma = 0.1))results <- unifiedml::benchmark(models, X, y, cv = 5, params = params)[1/3] Fitting model: glmMean CV score for glm: 0.9533[2/3] Fitting model: rfMean CV score for rf: 0.9600[3/3] Fitting model: svmMean CV score for svm: 0.9733print(results) # 5-fold cross-validation results$glm$glm$avg_score[1] 0.9533333$glm$scores    fold1     fold2     fold3     fold4     fold5 0.9333333 0.9666667 0.9333333 0.9333333 1.0000000 $rf$rf$avg_score[1] 0.96$rf$scores    fold1     fold2     fold3     fold4     fold5 0.9333333 1.0000000 0.9333333 0.9333333 1.0000000 $svm$svm$avg_score[1] 0.9733333$svm$scores    fold1     fold2     fold3     fold4     fold5 0.9666667 1.0000000 0.9666667 0.9333333 1.0000000 # initialize empty vectorsmodel_vec <- c()fold_vec  <- c()score_vec <- c()for (model in names(results)) {  scores <- results[[model]]$scores  model_vec <- c(model_vec, rep(model, length(scores)))  fold_vec  <- c(fold_vec, names(scores))  score_vec <- c(score_vec, as.numeric(scores))}df <- data.frame(  model = model_vec,  fold  = fold_vec,  score = score_vec)library(ggplot2)ggplot(df, aes(x = model, y = score, fill = model)) +  geom_violin(trim = FALSE, alpha = 0.6) +  geom_jitter(width = 0.08, size = 2) +  theme_minimal() +  labs(    title = "Cross-validation score distribution",    x = "Model",    y = "Score"  ) +  theme(legend.position = "none")

2 - Unified interface for predicting probabilities

# Load required packageslibrary(unifiedml)library(randomForest)library(nnet)library(e1071)# Load iris datasetdata(iris)# Setup reproducible dataset.seed(42)# Create feature matrix (all 4 numeric features)X <- as.matrix(iris[, 1:4])colnames(X) <- c("Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width")# Target: Species (multi-class with 3 levels)y_multiclass <- iris$Species# Create binary classification target (Versicolor vs others)y_binary <- factor(  ifelse(iris$Species == "versicolor", "versicolor", "other"),  levels = c("other", "versicolor"))# Split into train/test (75% train, 25% test)set.seed(42)train_idx <- sample(1:nrow(X), size = floor(0.75 * nrow(X)), replace = FALSE)test_idx <- setdiff(1:nrow(X), train_idx)X_train <- X[train_idx, ]X_test <- X[test_idx, ]y_train_multiclass <- y_multiclass[train_idx]y_test_multiclass <- y_multiclass[test_idx]y_train_binary <- y_binary[train_idx]y_test_binary <- y_binary[test_idx]cat("\n")cat("============================================================================\n")cat("IRIS DATASET - Summary\n")cat("============================================================================\n")cat(sprintf("Training samples: %d\n", nrow(X_train)))cat(sprintf("Test samples: %d\n", nrow(X_test)))cat(sprintf("Features: %d\n", ncol(X_train)))cat(sprintf("Classes: %s\n", paste(levels(y_multiclass), collapse = ", ")))# ============================================================================# EXAMPLE 1: randomForest - Multi-class Classification on IRIS# ============================================================================cat("\n")cat("============================================================================\n")cat("EXAMPLE 1: randomForest - Multi-class Classification\n")cat("============================================================================\n")mod_rf <- Model$new(randomForest::randomForest)mod_rf$fit(X_train, y_train_multiclass, ntree = 100)cat("\nPredicting probabilities for first 5 test samples:\n")probs_rf <- mod_rf$predict_proba(X_test[1:5, ])cat("\nProbability matrix:\n")print(round(probs_rf, 3))cat("\nInterpretation:\n")for(i in 1:5) {  cat(sprintf("\nSample %d (Actual: %s):\n", i, as.character(y_test_multiclass[i])))  cat(sprintf("  setosa:     %.1f%%\n", probs_rf[i, "setosa"] * 100))  cat(sprintf("  versicolor: %.1f%%\n", probs_rf[i, "versicolor"] * 100))  cat(sprintf("  virginica:  %.1f%%\n", probs_rf[i, "virginica"] * 100))  cat(sprintf("  Predicted:  %s\n", colnames(probs_rf)[which.max(probs_rf[i, ])]))}# Get class predictionspred_classes_rf <- mod_rf$predict(X_test[1:5, ], type = "class")cat("\nPredicted classes (first 5):", as.character(pred_classes_rf), "\n")cat("Actual classes (first 5):   ", as.character(y_test_multiclass[1:5]), "\n")# Calculate accuracy on full test setprobs_all_rf <- mod_rf$predict_proba(X_test)pred_all_rf <- colnames(probs_all_rf)[apply(probs_all_rf, 1, which.max)]accuracy_rf <- mean(pred_all_rf == as.character(y_test_multiclass))cat(sprintf("\nTest set accuracy: %.1f%%\n", accuracy_rf * 100))# ============================================================================# EXAMPLE 2: nnet - Multi-class Classification on IRIS# ============================================================================cat("\n")cat("============================================================================\n")cat("EXAMPLE 2: nnet - Multi-class Classification\n")cat("============================================================================\n")mod_nnet <- Model$new(nnet::nnet)mod_nnet$fit(X_train, y_train_multiclass, size = 10, maxit = 200, trace = FALSE)cat("\nPredicting probabilities for first 5 test samples:\n")probs_nnet <- mod_nnet$predict_proba(X_test[1:5, ])cat("\nProbability matrix (all 3 classes):\n")print(round(probs_nnet, 3))cat("\nDetailed predictions:\n")for(i in 1:5) {  cat(sprintf("\nSample %d (Actual: %s):\n", i, as.character(y_test_multiclass[i])))  cat(sprintf("  setosa:     %.1f%%\n", probs_nnet[i, "setosa"] * 100))  cat(sprintf("  versicolor: %.1f%%\n", probs_nnet[i, "versicolor"] * 100))  cat(sprintf("  virginica:  %.1f%%\n", probs_nnet[i, "virginica"] * 100))  cat(sprintf("  Predicted:  %s\n", colnames(probs_nnet)[which.max(probs_nnet[i, ])]))}# Get class predictionspred_classes_nnet <- mod_nnet$predict(X_test[1:5, ], type = "class")cat("\nPredicted classes (first 5):", as.character(pred_classes_nnet), "\n")cat("Actual classes (first 5):   ", as.character(y_test_multiclass[1:5]), "\n")# Calculate accuracyprobs_all_nnet <- mod_nnet$predict_proba(X_test)pred_all_nnet <- colnames(probs_all_nnet)[apply(probs_all_nnet, 1, which.max)]accuracy_nnet <- mean(pred_all_nnet == as.character(y_test_multiclass))cat(sprintf("\nTest set accuracy: %.1f%%\n", accuracy_nnet * 100))# ============================================================================# EXAMPLE 3: SVM - Multi-class Classification on IRIS# ============================================================================cat("\n")cat("============================================================================\n")cat("EXAMPLE 3: SVM - Multi-class Classification\n")cat("============================================================================\n")mod_svm <- Model$new(e1071::svm)mod_svm$fit(X_train, y_train_multiclass, probability = TRUE, kernel = "radial")cat("\nPredicting probabilities for first 5 test samples:\n")probs_svm <- mod_svm$predict_proba(X_test[1:5, ])cat("\nProbability matrix:\n")print(round(probs_svm, 4))cat("\nDetailed predictions:\n")for(i in 1:5) {  cat(sprintf("\nSample %d (Actual: %s):\n", i, as.character(y_test_multiclass[i])))  cat(sprintf("  setosa:     %.1f%%\n", probs_svm[i, "setosa"] * 100))  cat(sprintf("  versicolor: %.1f%%\n", probs_svm[i, "versicolor"] * 100))  cat(sprintf("  virginica:  %.1f%%\n", probs_svm[i, "virginica"] * 100))  cat(sprintf("  Predicted:  %s\n", colnames(probs_svm)[which.max(probs_svm[i, ])]))}# Calculate accuracyprobs_all_svm <- mod_svm$predict_proba(X_test)pred_all_svm <- colnames(probs_all_svm)[apply(probs_all_svm, 1, which.max)]accuracy_svm <- mean(pred_all_svm == as.character(y_test_multiclass))cat(sprintf("\nTest set accuracy: %.1f%%\n", accuracy_svm * 100))============================================================================IRIS DATASET - Summary============================================================================Training samples: 112Test samples: 38Features: 4Classes: setosa, versicolor, virginica============================================================================EXAMPLE 1: randomForest - Multi-class Classification============================================================================Predicting probabilities for first 5 test samples:Probability matrix:  setosa versicolor virginica1      1          0         02      1          0         03      1          0         04      1          0         05      1          0         0attr(,"assign")[1] 1 1 1attr(,"contrasts")attr(,"contrasts")$pred[1] "contr.treatment"attr(,"extraction_method")[1] "fallback::1"attr(,"model_class")[1] "randomForest.formula"Interpretation:Sample 1 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 2 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 3 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 4 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 5 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaPredicted classes (first 5): setosa setosa setosa setosa setosa Actual classes (first 5):    setosa setosa setosa setosa setosa Test set accuracy: 94.7%============================================================================EXAMPLE 2: nnet - Multi-class Classification============================================================================Predicting probabilities for first 5 test samples:Probability matrix (all 3 classes):  setosa versicolor virginica1      1          0         02      1          0         03      1          0         04      1          0         05      1          0         0attr(,"extraction_method")[1] "fallback::5"attr(,"model_class")[1] "nnet.formula"Detailed predictions:Sample 1 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 2 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 3 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 4 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 5 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaPredicted classes (first 5): setosa setosa setosa setosa setosa Actual classes (first 5):    setosa setosa setosa setosa setosa Test set accuracy: 97.4%============================================================================EXAMPLE 3: SVM - Multi-class Classification============================================================================Predicting probabilities for first 5 test samples:Probability matrix:  setosa versicolor virginica1      1          0         02      1          0         03      1          0         04      1          0         05      1          0         0attr(,"assign")[1] 1 1 1attr(,"contrasts")attr(,"contrasts")$pred[1] "contr.treatment"attr(,"extraction_method")[1] "fallback::1"attr(,"model_class")[1] "svm.formula"Detailed predictions:Sample 1 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 2 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 3 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 4 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaSample 5 (Actual: setosa):  setosa:     100.0%  versicolor: 0.0%  virginica:  0.0%  Predicted:  setosaTest set accuracy: 94.7%