@@ -3,8 +3,8 @@ Type: Package

 Title: A framework for cross-validated classification problems, with

        applications to differential variability and differential

        distribution testing

-Version: 3.1.19

-Date: 2022-09-28

+Version: 3.1.23

+Date: 2022-10-14

 Author: Dario Strbenac, Ellis Patrick, John Ormerod, Graham Mann, Jean Yang

 Maintainer: Dario Strbenac <dario.strbenac@sydney.edu.au>

 VignetteBuilder: knitr

@@ -14,7 +14,8 @@ Depends: R (>= 4.1.0), generics, methods, S4Vectors, MultiAssayExperiment, BiocP

 Imports: grid, genefilter, utils, dplyr, tidyr, rlang, randomForest

 Suggests: limma, edgeR, car, Rmixmod, ggplot2 (>= 3.0.0), gridExtra (>= 2.0.0), cowplot,

         BiocStyle, pamr, PoiClaClu, parathyroidSE, knitr, htmltools, gtable,

-        scales, e1071, rmarkdown, IRanges, robustbase, glmnet, class, randomForestSRC

+        scales, e1071, rmarkdown, IRanges, robustbase, glmnet, class, randomForestSRC,

+        MatrixModels, xgboost

 Description: The software formalises a framework for classification in R.

              There are four stages; Data transformation, feature selection, classifier training,

              and prediction. The requirements of variable types and names are

@@ -59,6 +60,7 @@ Collate:

     'interfaceRandomForest.R'

     'interfaceRandomForestSurvival.R'

     'interfaceSVM.R'

+    'interfaceXGB.R'

     'performancePlot.R'

     'plotFeatureClasses.R'

     'prepareData.R'

@@ -485,7 +485,7 @@ setClassUnion("SelectParamsOrNULL", c("SelectParams", "NULL"))

 #' @section Constructor:

 #' \describe{

 #' \item{}{\preformatted{SelectParams(featureRanking, characteristics = DataFrame(), minPresence = 1, intermediate = character(0),

-#' subsetToSelections = TRUE, tuneParams = list(nFeatures = seq(10, 100, 10), performanceType = "Balanced Error"), ...)} Creates a \code{SelectParams}

+#' subsetToSelections = TRUE, tuneParams = list(nFeatures = seq(10, 100, 10), performanceType = "Balanced Accuracy"), ...)} Creates a \code{SelectParams}

 #' object which stores the function(s) which will do the selection and parameters that the

 #' function will use.

 #' \describe{\item{\code{featureRanking}}{A character keyword referring to a registered feature ranking function. See \code{\link{available}}

@@ -537,7 +537,7 @@ standardGeneric("SelectParams"))

 #' @export

 setMethod("SelectParams", c("characterOrList"),

           function(featureRanking, characteristics = DataFrame(), minPresence = 1, 

-                   intermediate = character(0), subsetToSelections = TRUE, tuneParams = list(nFeatures = seq(10, 100, 10), performanceType = "Balanced Error"), ...)

+                   intermediate = character(0), subsetToSelections = TRUE, tuneParams = list(nFeatures = seq(10, 100, 10), performanceType = "Balanced Accuracy"), ...)

           {

             if(is.character(featureRanking)) featureRanking <- .selectionKeywordToFunction(featureRanking) else featureRanking <- lapply(featureRanking, .selectionKeywordToFunction)

             if(!is.list(featureRanking) && (ncol(characteristics) == 0 || !"Selection Name" %in% characteristics[, "characteristic"]))

@@ -655,15 +655,23 @@ setClassUnion("characterOrFunction", c("character", "function"))

 setMethod("TrainParams", c("characterOrFunction"),

           function(classifier, balancing = c("downsample", "upsample", "none"), characteristics = DataFrame(), intermediate = character(0), tuneParams = NULL, getFeatures = NULL, ...)

           {

-            if(is.character(classifier))              

-              classifier <- .classifierKeywordToParams(classifier)[[1]]@classifier # Training function.              

+            extras <- list(...)              

+            if(is.character(classifier))

+            {

+              trainParams <- .classifierKeywordToParams(classifier)[[1]] # Get a default params object.

+              if(is.null(getFeatures) && !is.null(trainParams@getFeatures))

+                getFeatures <- trainParams@getFeatures

+              classifier <- trainParams@classifier # Training function.

+            }

             if(ncol(characteristics) == 0 || !"Classifier Name" %in% characteristics[, "characteristic"])

             {

               characteristics <- rbind(characteristics, S4Vectors::DataFrame(characteristic = "Classifier Name", value = .ClassifyRenvir[["functionsTable"]][.ClassifyRenvir[["functionsTable"]][, "character"] == attr(classifier, "name"), "name"]))

             }

+            

+            if(length(extras) == 0) extras <- NULL

             new("TrainParams", classifier = classifier, characteristics = characteristics,

                 intermediate = intermediate, getFeatures = getFeatures, tuneParams = tuneParams,

-                otherParams = list(...))

+                otherParams = extras)

           })

 #' @usage NULL

@@ -17,6 +17,8 @@

                                                     ncol = 2, byrow = TRUE, dimnames = list(NULL, c("type", "better"))

 ) |> as.data.frame()

+.ClassifyRenvir[["performanceTypes"]] <- .ClassifyRenvir[["performanceInfoTable"]][, "type"]

+

 # Nice-looking names for feature selection and classification functions, to automatically use

 # in a variety of performance plots.

 .ClassifyRenvir[["functionsTable"]] <- matrix(

@@ -49,7 +51,8 @@

     "coxphRanking", "Cox Proportional Hazards",

     "coxnetTrainInterface", "Penalised Cox Proportional Hazards",

     #"NEMOEtrainInterface", "Nutrition-Ecotype Mixture of Experts",

-    "rfsrcTrainInterface", "Random Survival Forest"),

+    "rfsrcTrainInterface", "Random Survival Forest",

+    "extremeGradientBoostingTrainInterface", "Extreme Gradient Boosting"),

   ncol = 2, byrow = TRUE, dimnames = list(NULL, c("character", "name"))

 ) |> as.data.frame()

@@ -81,7 +84,8 @@

     "mixturesNormals", "Mixture of normals feature voting classifier.",

     "CoxPH", "Cox proportional hazards.",

     "CoxNet", "Penalised Cox proportional hazards.",

-    "randomSurvivalForest", "Random survival forest."

+    "randomSurvivalForest", "Random survival forest.",

+    "XGB", "Extreme gradient booster."

     ),

   ncol = 2, byrow = TRUE, dimnames = list(NULL, c("classifier Keyword", "Description"))

 ) |> as.data.frame()

@@ -22,7 +22,9 @@

 #' Set to NULL or "all" if all features should be used.

 #' @param selectionMethod A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

 #' and performing multiview classification, the respective classification methods will be used on each assay.

-#' @param selectionOptimisation A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise nFeatures. 

+#' @param selectionOptimisation A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.

+#' @param performanceType Default: \code{"auto"}. If \code{"auto"}, then balanced accuracy for classification or C-index for survival. Any one of the

+#' options described in \code{\link{calcPerformance}} may otherwise be specified.

 #' @param classifier A character vector of classification methods to compare. If a named character vector with names corresponding to different assays, 

 #' and performing multiview classification, the respective classification methods will be used on each assay.

 #' @param multiViewMethod A character vector specifying the multiview method or data integration approach to use.

@@ -87,6 +89,7 @@ setMethod("crossValidate", "DataFrame",

                    nFeatures = 20,

                    selectionMethod = "t-test",

                    selectionOptimisation = "Resubstitution",

+                   performanceType = "auto",

                    classifier = "randomForest",

                    multiViewMethod = "none",

                    assayCombinations = "all",

@@ -101,6 +104,17 @@ setMethod("crossValidate", "DataFrame",

               measurements <- measurementsAndOutcome[["measurements"]]

               outcome <- measurementsAndOutcome[["outcome"]]

+              # Ensure performance type is one of the ones that can be calculated by the package.

+              if(!performanceType %in% c("auto", .ClassifyRenvir[["performanceTypes"]]))

+                stop(paste("performanceType must be one of", paste(c("auto", .ClassifyRenvir[["performanceTypes"]]), collapse = ", "), "but is", performanceType))

+              

+              if(performanceType == "auto")

+              {

+                if(is.character(outcome) && (length(outcome) == 1 || length(outcome) == nrow(measurements)) || is.factor(outcome))

+                  performanceType <- "Balanced Accuracy"

+                else performanceType <- "C-index"

+              }

+              

               # Which data-types or data-views are present?

               assayIDs <- unique(mcols(measurements)$assay)

               if(is.null(assayIDs)) assayIDs <- 1

@@ -151,7 +165,7 @@ Using an ordinary GLM instead.")

                           sapply(classifier[[assayIndex]], function(classifierForAssay) {

                               # Loop over classifiers

                               sapply(selectionMethod[[assayIndex]], function(selectionForAssay) {

-                                  # Loop over classifiers

+                                  # Loop over selectors

                                   set.seed(seed)

                                   measurementsUse <- measurements

                                   if(assayIndex != 1) measurementsUse <- measurements[, mcols(measurements)[, "assay"] == assayIndex, drop = FALSE]

@@ -161,6 +175,7 @@ Using an ordinary GLM instead.")

                                       nFeatures = nFeatures[assayIndex],

                                       selectionMethod = selectionForAssay,

                                       selectionOptimisation = selectionOptimisation,

+                                      performanceType = performanceType,

                                       classifier = classifierForAssay,

                                       multiViewMethod = multiViewMethod,

                                       nFolds = nFolds,

@@ -202,6 +217,7 @@ Using an ordinary GLM instead.")

                          nFeatures = nFeatures[assayIndex],

                          selectionMethod = selectionMethod[assayIndex],

                          selectionOptimisation = selectionOptimisation,

+                         performanceType = performanceType, 

                          classifier = classifier[assayIndex],

                          multiViewMethod = ifelse(length(assayIndex) == 1, "none", multiViewMethod),

                          nFolds = nFolds,

@@ -235,6 +251,7 @@ Using an ordinary GLM instead.")

                          nFeatures = nFeatures[assayIndex],

                          selectionMethod = selectionMethod[assayIndex],

                          selectionOptimisation = selectionOptimisation,

+                         performanceType = performanceType,

                          classifier = classifier[assayIndex],

                          multiViewMethod = ifelse(length(assayIndex) == 1, "none", multiViewMethod),

                          nFolds = nFolds,

@@ -268,6 +285,7 @@ Using an ordinary GLM instead.")

                          nFeatures = nFeatures[assayIndex],

                          selectionMethod = selectionMethod[assayIndex],

                          selectionOptimisation = selectionOptimisation,

+                         performanceType = performanceType,

                          classifier = classifier[assayIndex],

                          multiViewMethod = ifelse(length(assayIndex) == 1, "none", multiViewMethod),

                          nFolds = nFolds,

@@ -292,6 +310,7 @@ setMethod("crossValidate", "MultiAssayExperiment",

                    nFeatures = 20,

                    selectionMethod = "t-test",

                    selectionOptimisation = "Resubstitution",

+                   performanceType = "auto",

                    classifier = "randomForest",

                    multiViewMethod = "none",

                    assayCombinations = "all",

@@ -307,6 +326,7 @@ setMethod("crossValidate", "MultiAssayExperiment",

                             nFeatures = nFeatures,

                             selectionMethod = selectionMethod,

                             selectionOptimisation = selectionOptimisation,

+                            performanceType = performanceType,

                             classifier = classifier,

                             multiViewMethod = multiViewMethod,

                             assayCombinations = assayCombinations,

@@ -324,6 +344,7 @@ setMethod("crossValidate", "data.frame", # data.frame of numeric measurements.

                    nFeatures = 20,

                    selectionMethod = "t-test",

                    selectionOptimisation = "Resubstitution",

+                   performanceType = "auto",

                    classifier = "randomForest",

                    multiViewMethod = "none",

                    assayCombinations = "all",

@@ -338,6 +359,7 @@ setMethod("crossValidate", "data.frame", # data.frame of numeric measurements.

                             nFeatures = nFeatures,

                             selectionMethod = selectionMethod,

                             selectionOptimisation = selectionOptimisation,

+                            performanceType = performanceType,

                             classifier = classifier,

                             multiViewMethod = multiViewMethod,

                             assayCombinations = assayCombinations,

@@ -355,6 +377,7 @@ setMethod("crossValidate", "matrix", # Matrix of numeric measurements.

                    nFeatures = 20,

                    selectionMethod = "t-test",

                    selectionOptimisation = "Resubstitution",

+                   performanceType = "auto",

                    classifier = "randomForest",

                    multiViewMethod = "none",

                    assayCombinations = "all",

@@ -369,6 +392,7 @@ setMethod("crossValidate", "matrix", # Matrix of numeric measurements.

                             nFeatures = nFeatures,

                             selectionMethod = selectionMethod,

                             selectionOptimisation = selectionOptimisation,

+                            performanceType = performanceType,

                             classifier = classifier,

                             multiViewMethod = multiViewMethod,

                             assayCombinations = assayCombinations,

@@ -388,6 +412,7 @@ setMethod("crossValidate", "list",

                    nFeatures = 20,

                    selectionMethod = "t-test",

                    selectionOptimisation = "Resubstitution",

+                   performanceType = "auto",

                    classifier = "randomForest",

                    multiViewMethod = "none",

                    assayCombinations = "all",

@@ -435,6 +460,7 @@ setMethod("crossValidate", "list",

                             nFeatures = nFeatures,

                             selectionMethod = selectionMethod,

                             selectionOptimisation = selectionOptimisation,

+                            performanceType = performanceType,

                             classifier = classifier,

                             multiViewMethod = multiViewMethod,

                             assayCombinations = assayCombinations,

@@ -527,7 +553,7 @@ generateCrossValParams <- function(nRepeats, nFolds, nCores, selectionOptimisati

     }

     tuneMode <- selectionOptimisation

     if(tuneMode == "CV") tuneMode <- "Nested CV"

-    if(!any(tuneMode %in% c("Resubstitution", "Nested CV", "none"))) stop("selectionOptimisation must be CV or Resubstitution or none")

+    if(!any(tuneMode %in% c("Resubstitution", "Nested CV", "none"))) stop("selectionOptimisation must be Nested CV or Resubstitution or none")

     CrossValParams(permutations = nRepeats, folds = nFolds, parallelParams = BPparam, tuneMode = tuneMode)

 }

 ######################################

@@ -579,6 +605,7 @@ generateModellingParams <- function(assayIDs,

                                     nFeatures,

                                     selectionMethod,

                                     selectionOptimisation,

+                                    performanceType = "auto",

                                     classifier,

                                     multiViewMethod = "none"

 ){

@@ -588,6 +615,7 @@ generateModellingParams <- function(assayIDs,

                                           nFeatures,

                                           selectionMethod,

                                           selectionOptimisation,

+                                          performanceType = performanceType,

                                           classifier,

                                           multiViewMethod)

         return(params)

@@ -610,8 +638,6 @@ generateModellingParams <- function(assayIDs,

     }

     classifier <- unlist(classifier)

-

-    performanceType <- ifelse(classifier %in% c("CoxPH", "CoxNet", "randomSurvivalForest"), "C-index", "Balanced Accuracy")

     # Check classifier

     knownClassifiers <- .ClassifyRenvir[["classifyKeywords"]][, "classifier Keyword"]

@@ -619,6 +645,7 @@ generateModellingParams <- function(assayIDs,

         stop(paste("Classifier must exactly match of these (be careful of case):", paste(knownClassifiers, collapse = ", ")))

     classifierParams <- .classifierKeywordToParams(classifier)

+    classifierParams$trainParams@tuneParams <- c(classifierParams$trainParams@tuneParams, performanceType = performanceType)

     selectionMethod <- unlist(selectionMethod)

@@ -658,6 +685,7 @@ generateMultiviewParams <- function(assayIDs,

                                     nFeatures,

                                     selectionMethod,

                                     selectionOptimisation,

+                                    performanceType,

                                     classifier,

                                     multiViewMethod){

@@ -676,6 +704,7 @@ generateMultiviewParams <- function(assayIDs,

                                  measurements = assayTrain[assayIDs],

                                  MoreArgs = list(

                                      selectionOptimisation = selectionOptimisation,

+                                     performanceType = performanceType,

                                      classifier = classifier,

                                      multiViewMethod = "none"),

                                  SIMPLIFY = FALSE)

@@ -689,7 +718,6 @@ generateMultiviewParams <- function(assayIDs,

                                           classifier = classifier,

                                           multiViewMethod = "none")

-        performanceType <- ifelse(classifier %in% c("CoxPH", "CoxNet", "randomSurvivalForest"), "C-index", "Balanced Accuracy")

         # Update selectParams to use

         params@selectParams <- SelectParams("selectMulti",

                                             params = paramsAssays,

@@ -715,6 +743,7 @@ generateMultiviewParams <- function(assayIDs,

                                  classifier = classifier[assayIDs],

                                  MoreArgs = list(

                                      selectionOptimisation = selectionOptimisation,

+                                     performanceType = performanceType,

                                      multiViewMethod = "none"),

                                  SIMPLIFY = FALSE)

@@ -743,6 +772,7 @@ generateMultiviewParams <- function(assayIDs,

                                  classifier = classifier[assayIDs],

                                  MoreArgs = list(

                                      selectionOptimisation = selectionOptimisation,

+                                     performanceType = performanceType,

                                      multiViewMethod = "none"),

                                  SIMPLIFY = FALSE)

@@ -771,6 +801,7 @@ generateMultiviewParams <- function(assayIDs,

                                  measurements = assayTrain[["clinical"]],

                                  classifier = classifier["clinical"],

                                  selectionOptimisation = selectionOptimisation,

+                                 performanceType = performanceType,

                                  multiViewMethod = "none"))

@@ -793,6 +824,7 @@ CV <- function(measurements = NULL,

                nFeatures = NULL,

                selectionMethod = "t-test",

                selectionOptimisation = "Resubstitution",

+               performanceType,

                classifier = "elasticNetGLM",

                multiViewMethod = "none",

                nFolds = 5,

@@ -831,6 +863,7 @@ CV <- function(measurements = NULL,

                                                nFeatures = nFeatures,

                                                selectionMethod = selectionMethod,

                                                selectionOptimisation = selectionOptimisation,

+                                               performanceType = performanceType,

                                                classifier = classifier,

                                                multiViewMethod = multiViewMethod)

@@ -879,18 +912,32 @@ train.data.frame <- function(x, outcomeTrain, ...)

 #' @rdname crossValidate

 #' @param assayIDs A character vector for assays to train with. Special value \code{"all"}

 #' uses all assays in the input object.

+#' @param performanceType Performance metric to optimise if classifier has any tuning parameters.

 #' @method train DataFrame

 #' @export

-train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", multiViewMethod = "none", assayIDs = "all", ...) # ... for prepareData.

+train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", performanceType = "auto",

+                            multiViewMethod = "none", assayIDs = "all", ...) # ... for prepareData.

                    {

               prepArgs <- list(x, outcomeTrain)

               extraInputs <- list(...)

               prepExtras <- numeric()

               if(length(extraInputs) > 0)

-                prepExtras <- which(names(extrasInputs) %in% .ClassifyRenvir[["prepareDataFormals"]])

+                prepExtras <- which(names(extraInputs) %in% .ClassifyRenvir[["prepareDataFormals"]])

               if(length(prepExtras) > 0)

                 prepArgs <- append(prepArgs, extraInputs[prepExtras])

               measurementsAndOutcome <- do.call(prepareData, prepArgs)

+              

+              # Ensure performance type is one of the ones that can be calculated by the package.

+              if(!performanceType %in% c("auto", .ClassifyRenvir[["performanceTypes"]]))

+                stop(paste("performanceType must be one of", paste(c("auto", .ClassifyRenvir[["performanceTypes"]]), collapse = ", "), "but is", performanceType))

+              

+              if(performanceType == "auto")

+              {

+                if(is.character(outcomeTrain) && (length(outcomeTrain) == 1 || length(outcomeTrain) == nrow(x)) || is.factor(outcomeTrain))

+                  performanceType <- "Balanced Accuracy"

+                else performanceType <- "C-index"

+              }

+              

               measurements <- measurementsAndOutcome[["measurements"]]

               outcomeTrain <- measurementsAndOutcome[["outcome"]]

@@ -898,7 +945,7 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", multiV

               if(assayIDs == "all") assayIDs <- unique(mcols(measurements)[, "assay"])

               if(is.null(assayIDs)) assayIDs <- 1

               names(assayIDs) <- assayIDs

-              names(classifier) <- classifier

+              names(classifier) <- assayIDs

               if(multiViewMethod == "none"){

                   resClassifier <-

@@ -906,10 +953,13 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", multiV

                           # Loop over assays

                           sapply(classifier[[assayIndex]], function(classifierForAssay) {

                               # Loop over classifiers

+                              

                                   measurementsUse <- measurements

                                   if(assayIndex != 1) measurementsUse <- measurements[, mcols(measurements)[, "assay"] == assayIndex, drop = FALSE]

                                   classifierParams <- .classifierKeywordToParams(classifierForAssay)

+                                  if(!is.null(classifierParams$trainParams@tuneParams))

+                                    classifierParams$trainParams@tuneParams <- c(classifierParams$trainParams@tuneParams, performanceType = performanceType)

                                   modellingParams <- ModellingParams(balancing = "none", selectParams = NULL,

                                                                trainParams = classifierParams$trainParams, predictParams = classifierParams$predictParams)

@@ -985,7 +1035,6 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", multiV

 #' @rdname crossValidate

 #' @method train list

 #' @export

-# Each of the first four variables are named lists with names of assays.

 train.list <- function(x, outcomeTrain, ...)

               {

                 # Check data type is valid

@@ -1001,7 +1050,7 @@ train.list <- function(x, outcomeTrain, ...)

                   stop("All datasets must have the same samples")

                 # Check the number of outcome is the same

-                if (!all(sapply(x, nrow) == length(x)) && !is.character(x))

+                if (!all(sapply(x, nrow) == length(outcomeTrain)) && !is.character(outcomeTrain))

                   stop("outcome must have same number of samples as measurements")

               df_list <- sapply(x, S4Vectors::DataFrame)

@@ -1016,6 +1065,7 @@ train.list <- function(x, outcomeTrain, ...)

               # Each list of tabular data has been collapsed into a DataFrame.

               # Will be subset to relevant assayIDs inside the DataFrame method.

+              

               train(combined_df, outcomeTrain, ...)

 }

@@ -1028,13 +1078,13 @@ train.MultiAssayExperiment <- function(x, outcomeColumns, ...)

               extraInputs <- list(...)

               prepExtras <- trainExtras <- numeric()

               if(length(extraInputs) > 0)

-                prepExtras <- which(names(extrasInputs) %in% .ClassifyRenvir[["prepareDataFormals"]])

+                prepExtras <- which(names(extraInputs) %in% .ClassifyRenvir[["prepareDataFormals"]])

               if(length(prepExtras) > 0)

                 prepArgs <- append(prepArgs, extraInputs[prepExtras])

               measurementsAndOutcome <- do.call(prepareData, prepArgs)

               trainArgs <- list(measurementsAndOutcome[["measurements"]], measurementsAndOutcome[["outcome"]])

               if(length(extraInputs) > 0)

-                trainExtras <- which(!names(extrasInputs) %in% .ClassifyRenvir[["prepareDataFormals"]])

+                trainExtras <- which(!names(extraInputs) %in% .ClassifyRenvir[["prepareDataFormals"]])

               if(length(trainExtras) > 0)

                 trainArgs <- append(trainArgs, extraInputs[trainExtras])

               do.call(train, trainArgs)

@@ -1066,11 +1116,9 @@ predict.trainedByClassifyR <- function(object, newData, ...)

               newData <- prepareData(newData, useFeatures = allFeatureNames(object))

               # Some classifiers dangerously use positional matching rather than column name matching.

               # newData columns are sorted so that the right column ordering is guaranteed.

-            } else {stop("'newData' is not one of the valid data types. It is of type ", class(newData), '.')}

-  if(is(object, "ClassifyResult"))

-  {

-    object@modellingParams@predictParams@predictor(object@finalModel[[1]], newData)

-  } else if (is(object, "listOfModels")) { # Object is itself a trained model and it is assumed that a predict method is defined for it.

-    mapply(function(model, assay) predict(model, assay), object, newData, SIMPLIFY = FALSE)

-  } else predict(object, newData)

+            }

+

+    if (is(object, "listOfModels")) 

+         mapply(function(model, assay) predict(model, assay), object, newData, SIMPLIFY = FALSE)

+    else predict(object, newData) # Object is itself a trained model and it is assumed that a predict method is defined for it.

 }

@@ -29,7 +29,7 @@ DLDApredictInterface <- function(model, measurementsTest, returnType = c("both",

     message("Predicting classes using trained DLDA classifier.")

   #predict(model, as.matrix(test))

-  predictions <- .predict(model, as.matrix(measurementsTest)) # Copy located in utilities.R.

+  predictions <- predict(model, as.matrix(measurementsTest)) # Copy located in utilities.R.

   switch(returnType, class = predictions[["class"]], # Factor vector.

          score = predictions[["posterior"]][, model[["groups"]]], # Numeric matrix.

@@ -1,27 +1,28 @@

-# An Interface for randomForest Package's randomForest Function

-randomForestTrainInterface <- function(measurementsTrain, classesTrain, mTryProportion = 0.5, ..., verbose = 3)

+# An Interface for ranger Package's randomForest Function

+randomForestTrainInterface <- function(measurementsTrain, outcomeTrain, mTryProportion = 0.5, ..., verbose = 3)

 {

-  if(!requireNamespace("randomForest", quietly = TRUE))

-    stop("The package 'randomForest' could not be found. Please install it.")

+  if(!requireNamespace("ranger", quietly = TRUE))

+    stop("The package 'ranger' could not be found. Please install it.")

   if(verbose == 3)

-    message("Fitting random forest classifier to training data and making predictions on test

-            data.")

+    message("Fitting random forest classifier to training data.")

   mtry <- round(mTryProportion * ncol(measurementsTrain)) # Number of features to try.

   # Convert to base data.frame as randomForest doesn't understand DataFrame.

-  randomForest::randomForest(as(measurementsTrain, "data.frame"), classesTrain, mtry = mtry, keep.forest = TRUE, ...)

+  ranger::ranger(x = as(measurementsTrain, "data.frame"), y = outcomeTrain, mtry = mtry, importance = "impurity_corrected", ...)

 }

 attr(randomForestTrainInterface, "name") <- "randomForestTrainInterface"

-# forest is of class randomForest

+# forest is of class ranger

 randomForestPredictInterface <- function(forest, measurementsTest, ..., returnType = c("both", "class", "score"), verbose = 3)

 {

   returnType <- match.arg(returnType)

+  classes <- forest$forest$levels

   if(verbose == 3)

     message("Predicting using random forest.")  

   measurementsTest <- as.data.frame(measurementsTest)

-  classPredictions <- predict(forest, measurementsTest)

-  classScores <- predict(forest, measurementsTest, type = "vote")[, forest[["classes"]], drop = FALSE]

+  classPredictions <- predict(forest, measurementsTest)$predictions

+  classScores <- predict(forest, measurementsTest, predict.all = TRUE)[[1]]

+  classScores <- t(apply(classScores, 1, function(sampleRow) table(factor(classes[sampleRow], levels = classes)) / forest$forest$num.trees))

   switch(returnType, class = classPredictions,

          score = classScores,

          both = data.frame(class = classPredictions, classScores, check.names = FALSE))

@@ -35,7 +36,7 @@ randomForestPredictInterface <- function(forest, measurementsTest, ..., returnTy

 forestFeatures <- function(forest)

                   {

-                    rankedFeaturesIndices <- order(randomForest::importance(forest), decreasing = TRUE)

-                    selectedFeaturesIndices <- randomForest::varUsed(forest, count = FALSE)

+                    rankedFeaturesIndices <- order(ranger::importance(forest), decreasing = TRUE)

+                    selectedFeaturesIndices <- which(ranger::importance(forest) > 0)

                     list(rankedFeaturesIndices, selectedFeaturesIndices)

                   }

\ No newline at end of file

@@ -6,9 +6,10 @@ rfsrcTrainInterface <- function(measurementsTrain, survivalTrain, mTryProportion

   if(verbose == 3)

     message("Fitting rfsrc classifier to training data and making predictions on test data.")

-  bindedMeasurements <- cbind(measurementsTrain, event = survivalTrain[,1], time = survivalTrain[,2])

+  bindedMeasurements <- cbind(measurementsTrain, event = survivalTrain[, 1], time = survivalTrain[, 2])

   mtry <- round(mTryProportion * ncol(measurementsTrain)) # Number of features to try.

-  randomForestSRC::rfsrc(Surv(event = event, time = time) ~ ., as.data.frame(bindedMeasurements), mtry = mtry, ...)

+  randomForestSRC::rfsrc(Surv(time, event) ~ ., data = as.data.frame(bindedMeasurements), mtry = mtry,

+                          var.used = "all.trees", importance = TRUE, ...)

 }

 attr(rfsrcTrainInterface, "name") <- "rfsrcTrainInterface"

@@ -18,4 +19,11 @@ rfsrcPredictInterface <- function(model, measurementsTest, ..., verbose = 3)

   predictedOutcome = predict(model, as.data.frame(measurementsTest), ...)$predicted

   names(predictedOutcome) = rownames(measurementsTest)

   predictedOutcome

-}

\ No newline at end of file

+}

+

+rfsrcFeatures <- function(forest)

+                  {

+                    rankedFeaturesIndices <- order(forest[["importance"]], decreasing = TRUE)

+                    selectedFeaturesIndices <- which(forest[["var.used"]] > 0)

+                    list(rankedFeaturesIndices, selectedFeaturesIndices)

+                  }

\ No newline at end of file

new file mode 100644

@@ -0,0 +1,82 @@

+# An Interface for xgboost Package's xgboost Function

+extremeGradientBoostingTrainInterface <- function(measurementsTrain, outcomeTrain, mTryProportion = 0.5, nrounds = 10, ..., verbose = 3)

+{

+  if(!requireNamespace("xgboost", quietly = TRUE))

+    stop("The package 'xgboost' could not be found. Please install it.")

+  if(verbose == 3)

+    message("Fitting extreme gradient boosting classifier to training data and making predictions on test

+            data.")

+  measurementsTrain <- as(measurementsTrain, "data.frame")

+  # Convert to one-hot encoding as xgboost doesn't understand factors. Need to get rid of intercept afterwards.

+  measurementsTrain <- MatrixModels::model.Matrix(~ 0 + ., data = measurementsTrain, sparse = TRUE)

+  

+  isClassification <- FALSE

+  numClasses <- NULL

+  if(is(outcomeTrain, "Surv")) # xgboost only knows about numeric vectors.

+  {

+    time <- outcomeTrain[, "time"]

+    event <- as.numeric(outcomeTrain[, "status"])

+    if(max(event) == 2) event <- event - 1

+    outcomeTrain <- time * ifelse(event == 1, 1, -1) # Negative for censoring.

+    objective <- "survival:cox"

+  } else { # Classification task.

+    isClassification <- TRUE

+    classes <- levels(outcomeTrain)

+    numClasses <- length(classes)

+    objective <- "multi:softprob"

+    outcomeTrain <- as.numeric(outcomeTrain) - 1 # Classes are represented as 0, 1, 2, ...

+  }

+  

+  trained <- xgboost::xgboost(measurementsTrain, outcomeTrain, objective = objective, nrounds = nrounds,

+                              num_class = numClasses, colsample_bynode = mTryProportion, verbose = 0, ...)

+  if(isClassification)

+  {

+    attr(trained, "classes") <- classes # Useful for factor predictions in predict method.

+    attr(trained, "featureNames") <- colnames(measurementsTrain)

+    attr(trained, "featureGroups") <- measurementsTrain@assign

+  }

+  trained

+}

+attr(extremeGradientBoostingTrainInterface, "name") <- "extremeGradientBoostingTrainInterface"

+    

+# booster is of class xgb.Booster

+extremeGradientBoostingPredictInterface <- function(booster, measurementsTest, ..., returnType = c("both", "class", "score"), verbose = 3)

+{

+  returnType <- match.arg(returnType)

+  if(verbose == 3)

+    message("Predicting using boosted random forest.")  

+  measurementsTest <- as(measurementsTest, "data.frame")

+  # Convert to one-hot encoding as xgboost doesn't understand factors. Need to get rid of intercept afterwards.

+  measurementsTest <- MatrixModels::model.Matrix(~ 0 + ., data = measurementsTest, sparse = TRUE)

+  scores <- predict(booster, measurementsTest, reshape = TRUE)

+  if(!is.null(attr(booster, "classes"))) # It is a classification task.

+  {

+    classPredictions <- attr(booster, "classes")[apply(scores, 1, function(sampleRow) which.max(sampleRow)[1])]

+    classPredictions <- factor(classPredictions, levels = attr(booster, "classes"))

+    result <- switch(returnType, class = classPredictions,

+                     score = scores,

+                     both = data.frame(class = classPredictions, scores, check.names = FALSE))

+  } else { # A survival task.

+     result <- scores

+  }

+  result

+}

+

+################################################################################

+#

+# Get selected features

+#

+################################################################################

+

+XGBfeatures <- function(booster)

+                  {

+                    importanceGains <- xgboost::xgb.importance(model = booster)[["Gain"]]

+                    gains <- rep(0, length(unique(attr(booster, "featureGroups"))))

+                    featureGroups <- attr(booster, "featureGroups")[match(xgboost::xgb.importance(model = booster)[["Feature"]], attr(booster, "featureNames"))]

+                    maxGains <- by(importanceGains, featureGroups, max)

+                    indicesUsed <- as.numeric(names(maxGains))

+                    gains[indicesUsed]  <- maxGains # Put into particular indexes.

+                    rankedFeaturesIndices <- order(gains, decreasing = TRUE)

+                    selectedFeaturesIndices <- indicesUsed

+                    list(rankedFeaturesIndices, selectedFeaturesIndices)

+                  }

@@ -1,6 +1,6 @@

 # Random Forest

 RFparams <- function() {

-    trainParams <- TrainParams(randomForestTrainInterface, tuneParams = list(mTryProportion = c(0.25, 0.33, 0.50, 0.66, 0.75, 1.00), ntree = seq(100, 500, 100), performanceType = "Balanced Error"),

+    trainParams <- TrainParams(randomForestTrainInterface, tuneParams = list(mTryProportion = c(0.25, 0.33, 0.50, 0.66, 0.75, 1.00), num.trees = c(10, seq(100, 500, 100))),

                                getFeatures = forestFeatures)

     predictParams <- PredictParams(randomForestPredictInterface)

@@ -9,15 +9,24 @@ RFparams <- function() {

 # Random Survival Forest

 RSFparams <- function() {

-    trainParams <- TrainParams(rfsrcTrainInterface, tuneParams = list(mTryProportion = c(0.25, 0.33, 0.50, 0.66, 0.75, 1.00), ntree = seq(100, 500, 100), performanceType = "Balanced Error"))

+    trainParams <- TrainParams(rfsrcTrainInterface, tuneParams = list(mTryProportion = c(0.25, 0.33, 0.50, 0.66, 0.75, 1.00), ntree = c(10, seq(100, 500, 100))),

+                               getFeatures = rfsrcFeatures)

     predictParams <- PredictParams(rfsrcPredictInterface)

     return(list(trainParams = trainParams, predictParams = predictParams))

 }

+XGBparams <- function() {

+    trainParams <- TrainParams(extremeGradientBoostingTrainInterface, tuneParams = list(mTryProportion = c(0.25, 0.33, 0.50, 0.66, 0.75, 1.00), nrounds = c(5, 10, 15)),

+                               getFeatures = XGBfeatures)

+    predictParams <- PredictParams(extremeGradientBoostingPredictInterface)

+    

+    return(list(trainParams = trainParams, predictParams = predictParams))  

+}

+

 # k Nearest Neighbours

 kNNparams <- function() {

-    trainParams <- TrainParams(kNNinterface, tuneParams = list(k = 1:5, performanceType = "Balanced Error"))

+    trainParams <- TrainParams(kNNinterface, tuneParams = list(k = 1:5))

     predictParams <- NULL

     return(list(trainParams = trainParams, predictParams = predictParams))

 }

@@ -42,7 +51,7 @@ elasticNetGLMparams <- function() {

 SVMparams = function() {

     trainParams <- TrainParams(SVMtrainInterface,

                    tuneParams = list(kernel = c("linear", "polynomial", "radial", "sigmoid"),

-                                     cost = 10^(-3:3), performanceType = "Balanced Error"))

+                                     cost = 10^(-3:3)))

     predictParams <- PredictParams(SVMpredictInterface)

     return(list(trainParams = trainParams, predictParams = predictParams))

@@ -66,7 +75,7 @@ DLDAparams = function() {

 # naive Bayes Kernel

 naiveBayesParams <- function() {

-    trainParams <- TrainParams(naiveBayesKernel, tuneParams = list(difference = c("unweighted", "weighted"), performanceType = "Balanced Error"))

+    trainParams <- TrainParams(naiveBayesKernel, tuneParams = list(difference = c("unweighted", "weighted")))

     predictParams <- NULL

     return(list(trainParams = trainParams, predictParams = predictParams))

 }

@@ -142,6 +142,7 @@

     tuneParamsTrain <- append(tuneParamsTrain, modellingParams@trainParams@tuneParams)

     tuneCombosTrain <- expand.grid(tuneParamsTrain, stringsAsFactors = FALSE)  

     modellingParams@trainParams@tuneParams <- NULL

+    

     allPerformanceTables <- lapply(rankings, function(rankingsVariety)

     {

       # Creates a matrix. Columns are top n features, rows are varieties (one row if None).

@@ -166,7 +167,7 @@

           result <- runTest(measurementsTrain, outcomeTrain, measurementsTrain, outcomeTrain,

                             crossValParams = NULL, modellingParams = modellingParams,

                             verbose = verbose, .iteration = "internal")

-          

+

           predictions <- result[["predictions"]]

           # Classifiers will use a column "class" and survival models will use a column "risk".

           if(class(predictions) == "data.frame")

@@ -221,6 +222,7 @@

                             measurementsTrain, outcomeTrain,

                             crossValParams = NULL, modellingParams,

                             verbose = verbose, .iteration = "internal")

+

           predictions <- result[["predictions"]]

           if(class(predictions) == "data.frame")

             predictedOutcome <- predictions[, "class"]

@@ -274,7 +276,7 @@

         result <- runTest(measurementsTrain, outcomeTrain, measurementsTrain, outcomeTrain,

                           crossValParams = NULL, modellingParams,

                           verbose = verbose, .iteration = "internal")

-        

+

         predictions <- result[["predictions"]]

         if(class(predictions) == "data.frame")

           predictedOutcome <- predictions[, colnames(predictions) %in% c("class", "risk")]

@@ -511,6 +513,7 @@

         keyword,

         "randomForest" = RFparams(),

         "randomSurvivalForest" = RSFparams(),

+        "XGB" = XGBparams(),

         "GLM" = GLMparams(),

         "elasticNetGLM" = elasticNetGLMparams(),

         "SVM" = SVMparams(),

@@ -578,7 +581,8 @@

   obj

 }

-.predict <- function(object, newdata, ...) { # Remove once sparsediscrim is reinstated to CRAN.

+#' @method predict dlda

+predict.dlda <- function(object, newdata, ...) { # Remove once sparsediscrim is reinstated to CRAN.

   if (!inherits(object, "dlda"))  {

     stop("object not of class 'dlda'")

   }

@@ -2,6 +2,29 @@

 \title{ClassifyR News}

 \encoding{UTF-8}

+\section{Version 3.2.0}{\itemize{

+  \item

+  Fast Cox survival analysis.

+  \item

+  Simple parameter sets, as used by crossVaildate, now come with tuning parameter grid as standard.

+  \item

+  Wrappers are greatly simplified. Now, there is only one method for a data frame and they are not exported because they are not used directly by the end-user anyway.

+  \item

+  \code{prepareData} function to filter and subset input data using common ways, such as missingness and variability.

+  \item

+  Invalid column names of data (e.g. spaces, hyphens) are converted into safe names before modelling but converted back into original names for tracking ranked and selected features.

+  \item

+  \code{available} function shows the keywords corresponding to transformation, selection, classifier functions.

+  \item

+  More functions have automatically-selected parameters based on input data, reducing required user-specified parameters.

+  \item

+  New classifiers added for random survival forests and extreme gradient boosting.

+  \item

+  Adaptive sampling for modelling with uncertainty of class labels can be enabled with \code{adaptiveResamplingDelta}.

+  \item

+  Parameter tuning fixed to only use samples from the training set.

+}}

+

 \section{Version 3.0.0}{\itemize{

   \item

   Now supports survival models and their evaluation, in addition to existing classification functionality.

@@ -17,7 +17,7 @@ feature selection. The empty constructor is provided for convenience.

 \describe{

 \item{}{\preformatted{SelectParams(featureRanking, characteristics = DataFrame(), minPresence = 1, intermediate = character(0),

-subsetToSelections = TRUE, tuneParams = list(nFeatures = seq(10, 100, 10), performanceType = "Balanced Error"), ...)} Creates a \code{SelectParams}

+subsetToSelections = TRUE, tuneParams = list(nFeatures = seq(10, 100, 10), performanceType = "Balanced Accuracy"), ...)} Creates a \code{SelectParams}

 object which stores the function(s) which will do the selection and parameters that the

 function will use.

 \describe{\item{\code{featureRanking}}{A character keyword referring to a registered feature ranking function. See \code{\link{available}}

@@ -24,6 +24,7 @@ crossValidate(measurements, outcome, ...)

   nFeatures = 20,

   selectionMethod = "t-test",

   selectionOptimisation = "Resubstitution",

+  performanceType = "auto",

   classifier = "randomForest",

   multiViewMethod = "none",

   assayCombinations = "all",

@@ -40,6 +41,7 @@ crossValidate(measurements, outcome, ...)

   nFeatures = 20,

   selectionMethod = "t-test",

   selectionOptimisation = "Resubstitution",

+  performanceType = "auto",

   classifier = "randomForest",

   multiViewMethod = "none",

   assayCombinations = "all",

@@ -56,6 +58,7 @@ crossValidate(measurements, outcome, ...)

   nFeatures = 20,

   selectionMethod = "t-test",

   selectionOptimisation = "Resubstitution",

+  performanceType = "auto",

   classifier = "randomForest",

   multiViewMethod = "none",

   assayCombinations = "all",

@@ -72,6 +75,7 @@ crossValidate(measurements, outcome, ...)

   nFeatures = 20,

   selectionMethod = "t-test",

   selectionOptimisation = "Resubstitution",

+  performanceType = "auto",

   classifier = "randomForest",

   multiViewMethod = "none",

   assayCombinations = "all",

@@ -88,6 +92,7 @@ crossValidate(measurements, outcome, ...)

   nFeatures = 20,

   selectionMethod = "t-test",

   selectionOptimisation = "Resubstitution",

+  performanceType = "auto",

   classifier = "randomForest",

   multiViewMethod = "none",

   assayCombinations = "all",

@@ -106,6 +111,7 @@ crossValidate(measurements, outcome, ...)

   x,

   outcomeTrain,

   classifier = "randomForest",

+  performanceType = "auto",

   multiViewMethod = "none",

   assayIDs = "all",

   ...

@@ -136,7 +142,9 @@ Set to NULL or "all" if all features should be used.}

 \item{selectionMethod}{A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

 and performing multiview classification, the respective classification methods will be used on each assay.}

-\item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise nFeatures.}

+\item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.}

+

+\item{performanceType}{Performance metric to optimise if classifier has any tuning parameters.}

 \item{classifier}{A character vector of classification methods to compare. If a named character vector with names corresponding to different assays, 

 and performing multiview classification, the respective classification methods will be used on each assay.}

@@ -13,7 +13,7 @@ generateCrossValParams(nRepeats, nFolds, nCores, selectionOptimisation)

 \item{nCores}{A numeric specifying the number of cores used if the user wants to use parallelisation.}

-\item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise nFeatures.}

+\item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.}

 }

 \value{

 CrossValParams object

@@ -10,6 +10,7 @@ generateModellingParams(

   nFeatures,

   selectionMethod,

   selectionOptimisation,

+  performanceType = "auto",

   classifier,

   multiViewMethod = "none"

 )

@@ -28,7 +29,9 @@ Set to NULL or "all" if all features should be used.}

 \item{selectionMethod}{A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

 and performing multiview classification, the respective classification methods will be used on each assay.}

-\item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise nFeatures.}

+\item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.}

+

+\item{performanceType}{Performance metric to optimise if classifier has any tuning parameters.}

 \item{classifier}{A character vector of classification methods to compare. If a named character vector with names corresponding to different assays, 

 and performing multiview classification, the respective classification methods will be used on each assay.}

@@ -164,7 +164,7 @@ Although being a cross-validation framework, a number of popular feature selecti

 In the following tables, a function that is used when no function is explicitly specified by the user is shown as <span style="padding:4px; border:2px dashed #e64626;">functionName</span>.

-The functions below produce a ranking, of which different size subsets are tried and the classifier performance evaluated, to select a best subset of features, based on a criterion such as balanced error rate, for example.

+The functions below produce a ranking, of which different size subsets are tried and the classifier performance evaluated, to select a best subset of features, based on a criterion such as balanced accuracy rate, for example.

 Function | Description | DM | DV | DD

 ----|---------------------------|----|----|----

@@ -189,6 +189,8 @@ Function(s) | Description | DM | DV | DD

 <span style="font-family: 'Courier New', monospace;">mixModelsTrain</span>, <span style="font-family: 'Courier New', monospace;">mixModelsPredict</span> | Feature-wise mixtures of normals and voting | ✔ | ✔ | ✔ 

 <span style="font-family: 'Courier New', monospace;">naiveBayesKernel</span> | Feature-wise kernel density estimation and voting | ✔ | ✔ | ✔ 

 <span style="font-family: 'Courier New', monospace;">randomForestTrainInterface</span>, <span style="font-family: 'Courier New', monospace;">randomForestPredictInterface</span> | Wrapper for randomForest's functions <span style="font-family: 'Courier New', monospace;">randomForest</span> and <span style="font-family: 'Courier New', monospace;">predict</span> | ✔ | ✔ | ✔ 

+<span style="font-family: 'Courier New', monospace;">extremeGradientBoostingTrainInterface</span>, <span style="font-family: 'Courier New', monospace;">extremeGradientBoostingPredictInterface</span> | Wrapper for xgboost's functions <span style="font-family: 'Courier New', monospace;">xgboost</span> and <span style="font-family: 'Courier New', monospace;">predict</span> | ✔ | ✔ | ✔ 

+<span style="font-family: 'Courier New', monospace;">kNNinterface</span> | Wrapper for class's function <span style="font-family: 'Courier New', monospace;">knn</span> | ✔ | ✔ | ✔ 

 <span style="font-family: 'Courier New', monospace;">SVMtrainInterface</span>, <span style="font-family: 'Courier New', monospace;">SVMpredictInterface</span> | Wrapper for e1071's functions <span style="font-family: 'Courier New', monospace;">svm</span> and <span style="font-family: 'Courier New', monospace;">predict.svm</span> | ✔ | ✔ †| ✔ †

 \* If ordinary numeric measurements have been transformed to absolute deviations using <span style="font-family: 'Courier New', monospace;">subtractFromLocation</span>.<br>

@@ -249,7 +251,7 @@ bestGenePlot <- plotFeatureClasses(measurements, classes, names(mostChosen), dot

 The means of the abundance levels of `r names(sortedPercentages)[1]` are substantially different between the people with and without asthma. *plotFeatureClasses* can also plot categorical data, such as may be found in a clinical data table, as a bar chart.