@@ -3,8 +3,8 @@ Type: Package

 Title: A framework for cross-validated classification problems, with

        applications to differential variability and differential

        distribution testing

-Version: 3.2.5

-Date: 2022-11-28

+Version: 3.2.6

+Date: 2022-12-08

 Author: Dario Strbenac, Ellis Patrick, Sourish Iyengar, Harry Robertson, Andy Tran, John Ormerod, Graham Mann, Jean Yang

 Maintainer: Dario Strbenac <dario.strbenac@sydney.edu.au>

 VignetteBuilder: knitr

@@ -22,8 +22,8 @@

 #' or assays. If a numeric vector these will be optimised over using \code{selectionOptimisation}. If a named vector with the same names of multiple assays, 

 #' a different number of features will be used for each assay. If a named list of vectors, the respective number of features will be optimised over. 

 #' Set to NULL or "all" if all features should be used.

-#' @param selectionMethod Default: "auto". A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

-#' and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"} t-test (two categories) / F-test (three or more categories) ranking

+#' @param selectionMethod Default: \code{"auto"}. A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

+#' and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"}, t-test (two categories) / F-test (three or more categories) ranking

 #' and top \code{nFeatures} optimisation is done. Otherwise, the ranking method is per-feature Cox proportional hazards p-value.

 #' @param selectionOptimisation A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.

 #' @param performanceType Default: \code{"auto"}. If \code{"auto"}, then balanced accuracy for classification or C-index for survival. Otherwise, any one of the

@@ -91,10 +91,10 @@ setMethod("crossValidate", "DataFrame",

           function(measurements,

                    outcome,

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -297,10 +297,10 @@ setMethod("crossValidate", "MultiAssayExperiment",

           function(measurements,

                    outcome, 

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -331,10 +331,10 @@ setMethod("crossValidate", "data.frame", # data.frame of numeric measurements.

           function(measurements,

                    outcome, 

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -364,10 +364,10 @@ setMethod("crossValidate", "matrix", # Matrix of numeric measurements.

           function(measurements,

                    outcome,

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -399,10 +399,10 @@ setMethod("crossValidate", "list",

           function(measurements,

                    outcome, 

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -764,19 +764,18 @@ generateMultiviewParams <- function(assayIDs,

 }

 # measurements, outcome are mutually exclusive with x, outcomeTrain, measurementsTest, outcomeTest.

-CV <- function(measurements = NULL,

-               outcome = NULL, x = NULL, outcomeTrain = NULL, measurementsTest = NULL, outcomeTest = NULL,

+CV <- function(measurements, outcome, x, outcomeTrain, measurementsTest, outcomeTest,

                assayIDs,

-               nFeatures = NULL,

-               selectionMethod = "t-test",

-               selectionOptimisation = "Resubstitution",

+               nFeatures,

+               selectionMethod,

+               selectionOptimisation,

                performanceType,

-               classifier = "elasticNetGLM",

-               multiViewMethod = "none",

-               nFolds = 5,

-               nRepeats = 100,

-               nCores = 1,

-               characteristicsLabel = NULL)

+               classifier,

+               multiViewMethod,

+               nFolds,

+               nRepeats,

+               nCores,

+               characteristicsLabel)

 {

     # Which data-types or data-views are present?

@@ -848,7 +847,7 @@ train.data.frame <- function(x, outcomeTrain, ...)

 #' @param performanceType Performance metric to optimise if classifier has any tuning parameters.

 #' @method train DataFrame

 #' @export

-train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", performanceType = "auto",

+train.DataFrame <- function(x, outcomeTrain, selectionMethod = "auto", nFeatures = 20, classifier = "auto", performanceType = "auto",

                             multiViewMethod = "none", assayIDs = "all", ...) # ... for prepareData.

                    {

               prepArgs <- list(x, outcomeTrain)

@@ -863,18 +862,20 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", perfor

               # Ensure performance type is one of the ones that can be calculated by the package.

               if(!performanceType %in% c("auto", .ClassifyRenvir[["performanceTypes"]]))

                 stop(paste("performanceType must be one of", paste(c("auto", .ClassifyRenvir[["performanceTypes"]]), collapse = ", "), "but is", performanceType))

-              

+

+              isCategorical <- is.character(outcomeTrain) && (length(outcomeTrain) == 1 || length(outcomeTrain) == nrow(measurements)) || is.factor(outcomeTrain)

               if(performanceType == "auto")

-              {

-                if(is.character(outcomeTrain) && (length(outcomeTrain) == 1 || length(outcomeTrain) == nrow(x)) || is.factor(outcomeTrain))

-                  performanceType <- "Balanced Accuracy"

-                else performanceType <- "C-index"

-              }

+                if(isCategorical) performanceType <- "Balanced Accuracy" else performanceType <- "C-index"

+              if(length(selectionMethod) == 1 && selectionMethod == "auto")

+                if(isCategorical) selectionMethod <- "t-test" else selectionMethod <- "CoxPH"

+              if(length(classifier) == 1 && classifier == "auto")

+                if(isCategorical) classifier <- "randomForest" else classifier <- "CoxPH"

               measurements <- measurementsAndOutcome[["measurements"]]

               outcomeTrain <- measurementsAndOutcome[["outcome"]]

               classifier <- cleanClassifier(classifier = classifier, measurements = measurements)

+              selectionMethod <- cleanSelectionMethod(selectionMethod = selectionMethod, measurements = measurements)

               if(assayIDs == "all") assayIDs <- unique(S4Vectors::mcols(measurements)[, "assay"])

               if(is.null(assayIDs)) assayIDs <- 1

               names(assayIDs) <- assayIDs

@@ -886,30 +887,52 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", perfor

                           # Loop over assays

                           sapply(classifier[[assayIndex]], function(classifierForAssay) {

                               # Loop over classifiers

+                                sapply(selectionMethod[[assayIndex]], function(selectionForAssay) {

+                                  # Loop over selectors

                                   measurementsUse <- measurements

                                   if(assayIndex != 1) measurementsUse <- measurements[, S4Vectors::mcols(measurements)[, "assay"] == assayIndex, drop = FALSE]

+                                  modellingParams <- generateModellingParams(assayIDs = assayIDs, measurements = measurements, nFeatures = nFeatures,

+                                                     selectionMethod = selectionMethod, selectionOptimisation = "Resubstitution", performanceType = performanceType,

+                                                     classifier = classifier, multiViewMethod = "none")

+                                  topFeatures <- .doSelection(measurementsUse, outcomeTrain, CrossValParams(), modellingParams, verbose = 0)

+                                  selectedFeaturesIndices <- topFeatures[[2]] # Extract for subsetting.

+                                  tuneDetailsSelect <- topFeatures[[3]]

+                                  measurementsUse <- measurementsUse[, selectedFeaturesIndices]

+

                                   classifierParams <- .classifierKeywordToParams(classifierForAssay)

-                                  if(!is.null(classifierParams$trainParams@tuneParams))

-                                    classifierParams$trainParams@tuneParams <- c(classifierParams$trainParams@tuneParams, performanceType = performanceType)

                                   modellingParams <- ModellingParams(balancing = "none", selectParams = NULL,

-                                                               trainParams = classifierParams$trainParams, predictParams = classifierParams$predictParams)

+                                                                     trainParams = classifierParams$trainParams, predictParams = classifierParams$predictParams)

+                                  if(!is.null(tuneDetailsSelect))

+                                  {

+                                    tuneDetailsSelectUse <- tuneDetailsSelect[["tuneCombinations"]][tuneDetailsSelect[["bestIndex"]], , drop = FALSE]

+                                    avoidTune <- match(colnames(tuneDetailsSelectUse), names(modellingParams@trainParams@tuneParams))

+                                    if(any(!is.na(avoidTune)))

+                                    {

+                                      modellingParams@trainParams@otherParams <- c(modellingParams@trainParams@otherParams, tuneDetailsSelectUse[!is.na(avoidTune)])

+                                      modellingParams@trainParams@tuneParams <- modellingParams@trainParams@tuneParams[-na.omit(avoidTune)]

+                                      if(length(modellingParams@trainParams@tuneParams) == 0) modellingParams@trainParams@tuneParams <- NULL

+                                    }

+                                  }

+                                  if(!is.null(modellingParams@trainParams@tuneParams))

+                                    modellingParams$trainParams@tuneParams <- c(modellingParams$trainParams@tuneParams, performanceType = performanceType)

-                                  .doTrain(measurementsUse, outcomeTrain, NULL, NULL, CrossValParams(), modellingParams, verbose = 0)[["model"]]

+                                  trained <- .doTrain(measurementsUse, outcomeTrain, NULL, NULL, CrossValParams(), modellingParams, verbose = 0)[["model"]]

+                                  attr(trained, "predictFunction") <- classifierParams$predictParams@predictor

+                                  trained

                                   ## train model

-                          },

-                          simplify = FALSE)

-                      },

-                      simplify = FALSE)

+                                }, simplify = FALSE)

+                          }, simplify = FALSE)

+                      }, simplify = FALSE)

-                  models <- unlist(resClassifier, recursive = FALSE)

+                  models <- unlist(unlist(resClassifier, recursive = FALSE), recursive = FALSE)

                   if(length(models) == 1) {

                       model <- models[[1]]

-                      class(model) <- c(class(model), "trainedByClassifyR")

+                      class(model) <- c("trainedByClassifyR", class(model))

                       models <- NULL

                   } else {

-                      class(models) <- c(class(models), "listOfModels", "trainedByClassifyR")

+                      class(models) <- c("listOfModels", "trainedByClassifyR", class(models))

                   }

               }

@@ -932,6 +955,8 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", perfor

                # Generate params for each assay. This could be extended to have different selectionMethods for each type

                  paramsAssays <- mapply(generateModellingParams,

+                                        nFeatures = nFeatures[assayIDs],

+                                        selectionMethod = selectionMethod[assayIDs],

                                         assayIDs = assayIDs,

                                         measurements = assayTrain[assayIDs],

                                         classifier = classifier[assayIDs],

@@ -1049,9 +1074,11 @@ predict.trainedByClassifyR <- function(object, newData, ...)

               newData <- prepareData(newData, useFeatures = allFeatureNames(object))

               # Some classifiers dangerously use positional matching rather than column name matching.

               # newData columns are sorted so that the right column ordering is guaranteed.

-            }

-

+    }

+    

+    predictFunctionUse <- attr(object, "predictFunction")

+    class(object) <- rev(class(object)) # Now want the predict method of the specific model to be picked, so put model class first.

     if (is(object, "listOfModels")) 

-         mapply(function(model, assay) predict(model, assay), object, newData, SIMPLIFY = FALSE)

-    else predict(object, newData) # Object is itself a trained model and it is assumed that a predict method is defined for it.

+         mapply(function(model, assay) predictFunctionUse(model, assay), object, newData, SIMPLIFY = FALSE)

+    else predictFunctionUse(object, newData) # Object is itself a trained model and it is assumed that a predict method is defined for it.

 }

@@ -13,7 +13,8 @@

 #' are features.

 #' @param outcome Either a factor vector of classes, a \code{\link{Surv}} object, or

 #' a character string, or vector of such strings, containing column name(s) of column(s)

-#' containing either classes or time and event information about survival.

+#' containing either classes or time and event information about survival. If column names

+#' of survival information, time must be in first column and event status in the second.

 #' @param outcomeColumns If \code{measurements} is a \code{MultiAssayExperiment}, the

 #' names of the column (class) or columns (survival) in the table extracted by \code{colData(data)}

 #' that contain(s) the each individual's outcome to use for prediction.

@@ -46,6 +47,14 @@ setMethod("prepareData", "matrix",

   prepareData(S4Vectors::DataFrame(measurements, check.names = FALSE), outcome, ...)

 })

+#' @rdname prepareData

+#' @export

+setMethod("prepareData", "data.frame",

+  function(measurements, outcome, ...)

+{

+  prepareData(S4Vectors::DataFrame(measurements, check.names = FALSE), outcome, ...)

+})

+

 #' @rdname prepareData

 #' @export

 setMethod("prepareData", "DataFrame",

@@ -22,10 +22,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -39,10 +39,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -56,10 +56,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -73,10 +73,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -90,10 +90,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -110,7 +110,9 @@ crossValidate(measurements, outcome, ...)

 \method{train}{DataFrame}(

   x,

   outcomeTrain,

-  classifier = "randomForest",

+  selectionMethod = "auto",

+  nFeatures = 20,

+  classifier = "auto",

   performanceType = "auto",

   multiViewMethod = "none",

   assayIDs = "all",

@@ -141,8 +143,8 @@ or assays. If a numeric vector these will be optimised over using \code{selectio

 a different number of features will be used for each assay. If a named list of vectors, the respective number of features will be optimised over. 

 Set to NULL or "all" if all features should be used.}

-\item{selectionMethod}{Default: "auto". A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

-and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"} t-test (two categories) / F-test (three or more categories) ranking

+\item{selectionMethod}{Default: \code{"auto"}. A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

+and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"}, t-test (two categories) / F-test (three or more categories) ranking

 and top \code{nFeatures} optimisation is done. Otherwise, the ranking method is per-feature Cox proportional hazards p-value.}

 \item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.}

@@ -5,10 +5,13 @@

 \alias{prepareData,matrix-method}

 \alias{prepareData,DataFrame-method}

 \alias{prepareData,MultiAssayExperiment-method}

+\alias{prepareData,data.frame-method}

 \title{Convert Different Data Classes into DataFrame and Filter Features}

 \usage{

 \S4method{prepareData}{matrix}(measurements, outcome, ...)

+\S4method{prepareData}{data.frame}(measurements, outcome, ...)

+

 \S4method{prepareData}{DataFrame}(

   measurements,

   outcome,

@@ -31,7 +34,8 @@ are features.}

 \item{outcome}{Either a factor vector of classes, a \code{\link{Surv}} object, or

 a character string, or vector of such strings, containing column name(s) of column(s)

-containing either classes or time and event information about survival.}

+containing either classes or time and event information about survival. If column names

+of survival information, time must be in first column and event status in the second.}

 \item{useFeatures}{If \code{measurements} is a \code{MultiAssayExperiment},

 a two-column table of features to use. The first column must have assay names