@@ -3,8 +3,8 @@ Type: Package

 Title: A framework for cross-validated classification problems, with

        applications to differential variability and differential

        distribution testing

-Version: 3.3.8

-Date: 2022-11-28

+Version: 3.3.9

+Date: 2022-12-08

 Authors@R:

     c(

     person(given = "Dario", family = "Strbenac", email = "dario.strbenac@sydney.edu.au", role = c("aut", "cre")),

@@ -20,8 +20,8 @@ VignetteBuilder: knitr

 Encoding: UTF-8

 biocViews: Classification, Survival

 Depends: R (>= 4.1.0), generics, methods, S4Vectors, MultiAssayExperiment, BiocParallel, survival

-Imports: grid, genefilter, utils, dplyr, tidyr, rlang, ranger

-Suggests: limma, edgeR, car, Rmixmod, ggplot2 (>= 3.0.0), gridExtra (>= 2.0.0), cowplot,

+Imports: grid, genefilter, utils, dplyr, plyr, tidyr, rlang, ranger, ggplot2 (>= 3.0.0), ggpubr, reshape2

+Suggests: limma, edgeR, car, Rmixmod, gridExtra (>= 2.0.0), cowplot,

         BiocStyle, pamr, PoiClaClu, parathyroidSE, knitr, htmltools, gtable,

         scales, e1071, rmarkdown, IRanges, robustbase, glmnet, class, randomForestSRC,

         MatrixModels, xgboost

@@ -44,6 +44,7 @@ Collate:

     'classes.R'

     'calcPerformance.R'

     'constants.R'

+    'crissCrossValidate.R'

     'crossValidate.R'

     'data.R'

     'distribution.R'

@@ -75,6 +76,7 @@ Collate:

     'prepareData.R'

     'previousSelection.R'

     'previousTrained.R'

+    'randomSelection.R'

     'rankingBartlett.R'

     'rankingCoxPH.R'

     'rankingDMD.R'

@@ -22,6 +22,8 @@ export(calcCVperformance)

 export(calcExternalPerformance)

 export(chosenFeatureNames)

 export(colCoxTests)

+export(crissCrossPlot)

+export(crissCrossValidate)

 export(crossValidate)

 export(distribution)

 export(edgesToHubNetworks)

@@ -86,8 +88,13 @@ exportMethods(show)

 exportMethods(totalPredictions)

 exportMethods(tunedParameters)

 import(MultiAssayExperiment)

+import(dplyr)

+import(ggplot2)

+import(ggpubr)

 import(grid)

 import(methods)

+import(plyr)

+import(reshape2)

 import(utils)

 importFrom(S4Vectors,as.data.frame)

 importFrom(S4Vectors,do.call)

@@ -14,7 +14,7 @@

 #' 

 #' @aliases ROCplot ROCplot,list-method

 #' @param results A list of \code{\link{ClassifyResult}} objects.

-#' @param mode Default: "merge". Whether to merge all predictions of all

+#' @param mode Default: \code{"merge"}. Whether to merge all predictions of all

 #' iterations of cross-validation into one set or keep them separate. Keeping

 #' them separate will cause separate ROC curves to be computed for each

 #' iteration and confidence intervals to be drawn with the solid line being the

@@ -815,7 +815,7 @@ setClass("ModellingParams", representation(

 #' @rdname ModellingParams-class

 #' @aliases ModellingParams ModellingParams-class

 #' @docType class

-#' @param balancing Default: "downsample". A character value specifying what kind

+#' @param balancing Default: \code{"downsample"}. A character value specifying what kind

 #' of class balancing to do, if any.

 #' @param transformParams Parameters used for feature transformation inside of C.V.

 #' specified by a \code{\link{TransformParams}} instance. Optional, can be \code{NULL}.

@@ -45,6 +45,7 @@

     "pairsDifferencesRanking", "Pairs Differences",

     "previousSelection", "Previous Selection", 

     "previousTrained", "Previous Trained",

+    "randomSelection", "Random Selection",

     "randomForestTrainInterface", "Random Forest",

     "SVMtrainInterface", "Support Vector Machine",

     "coxphTrainInterface", "Cox Proportional Hazards",

@@ -67,7 +68,8 @@

     "likelihoodRatio", "Likelihood ratio test (normal distribution).",

     "KS", "Kolmogorov-Smirnov test for differences in distributions.",

     "KL", "Kullback-Leibler divergence between distributions.",

-    "CoxPH", "Cox proportional hazards Wald test per-feature."

+    "CoxPH", "Cox proportional hazards Wald test per-feature.",

+    "randomSelection", "Randomly selects a specified number of features."

     ),

   ncol = 2, byrow = TRUE, dimnames = list(NULL, c("selectionMethod Keyword", "Description"))

 ) |> as.data.frame()

new file mode 100644

@@ -0,0 +1,234 @@

+#' A function to perform pairwise cross validation

+#'

+#' This function has been designed to perform cross-validation and model prediction on datasets in a pairwise manner.

+#'

+#' @param measurements A \code{list} of either \code{\link{DataFrame}}, \code{\link{data.frame}} or \code{\link{matrix}} class measurements.

+#' @param outcomes A \code{list} of vectors that respectively correspond to outcomes of the samples in \code{measurements} list.

+#' @param nFeatures The number of features to be used for modelling.

+#' @param selectionMethod Default: \code{"auto"}. A character keyword of the feature algorithm to be used. If \code{"auto"}, t-test (two categories) /

+#' F-test (three or more categories) ranking and top \code{nFeatures} optimisation is done. Otherwise, the ranking method is per-feature Cox proportional

+#' hazards p-value.

+#' @param selectionOptimisation A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise nFeatures.

+#' @param trainType Default: \code{"modelTrain"}. A keyword specifying whether a fully trained model is used to make predictions on the test

+#' set or if only the feature identifiers are chosen using the training data set and a number of training-predictions are made by cross-validation

+#' in the test set.

+#' @param classifier Default: \code{"auto"}. A character keyword of the modelling algorithm to be used. If \code{"auto"}, then a random forest is used

+#' for a classification task or Cox proportional hazards model for a survival task.

+#' @param nFolds A numeric specifying the number of folds to use for cross-validation.

+#' @param nRepeats A numeric specifying the the number of repeats or permutations to use for cross-validation.

+#' @param nCores A numeric specifying the number of cores used if the user wants to use parallelisation. 

+#' @param performanceType Default: \code{"auto"}. If \code{"auto"}, then balanced accuracy for classification or C-index for survival. Otherwise, any one of the

+#' options described in \code{\link{calcPerformance}} may otherwise be specified.

+#' @param doRandomFeatures Default: \code{FALSE}. Whether to perform random feature selection to establish a baseline performance. Either \code{FALSE} or \code{TRUE}

+#' are permitted values.

+#' @return A list with elements \code{"real"} for the matrix of pairwise performance metrics using real

+#' feature selection, \code{"random"} if \code{doRandomFeatures} is \code{TRUE} for metrics of random selection and

+#' \code{"params"} for a list of parameters used during the execution of this function.

+#' @author Harry Robertson

+

+#' @import plyr

+#' @import dplyr

+#'

+#' @export

+#' 

+

+crissCrossValidate <- function(measurements, outcomes, 

+                               nFeatures = 20, selectionMethod = "auto",

+                               selectionOptimisation = "Resubstitution",

+                               trainType = c("modelTrain", "modelTest"),

+                               performanceType = "auto",

+                               doRandomFeatures = FALSE,

+                               classifier = "auto",

+                               nFolds = 5, nRepeats = 20, nCores = 1)

+{

+  trainType <- match.arg(trainType)

+  if(!is.list(measurements)) stop("'measurements' is not of type list but is of type", class(measurements))

+  if(!is.list(outcomes)) stop("'outcomes' is not of type list but is of type", class(outcomes))

+  isCategorical <- is.character(outcomes[[1]]) && (length(outcomes[[1]]) == 1 || length(outcomes[[1]]) == nrow(measurements[[1]])) || is.factor(outcomes[[1]])

+  if(performanceType == "auto")

+    if(isCategorical) performanceType <- "Balanced Accuracy" else performanceType <- "C-index"

+  if(length(selectionMethod) == 1 && selectionMethod == "auto")

+    if(isCategorical) selectionMethod <- "t-test" else selectionMethod <- "CoxPH"

+  if(length(classifier) == 1 && classifier == "auto")

+    if(isCategorical) classifier <- "randomForest" else classifier <- "CoxPH"

+  

+  dataCleaned <- mapply(function(measurementsOne, outcomesOne)

+  {

+    prepareData(measurementsOne, outcomesOne)

+  }, measurements, outcomes, SIMPLIFY = FALSE)

+  measurements <- lapply(dataCleaned, "[[", 1)

+  outcomes <- lapply(dataCleaned, "[[", 2)

+  

+  # If trainType is modelTrain, then build a model on a data set and test it on every data set.

+  if(trainType == "modelTrain")

+  {

+    # Build a model for each dataset.

+    trainedModels <- mapply(function(measurementsOne, outcomesOne)

+    {

+      train(measurementsOne, outcomesOne,

+            nFeatures = nFeatures,

+            selectionMethod = selectionMethod, selectionOptimisation = selectionOptimisation,

+            classifier = classifier, multiViewMethod = "none")

+     }, measurements, outcomes, SIMPLIFY = FALSE)

+

+    # Perform pair-wise model assessment.

+    performanceAllPairs <- lapply(trainedModels, function(trainedModel)

+    {

+      mapply(function(testData, testOutcomes)

+      {

+        predictions <- predict(trainedModel, testData)

+        if(is(predictions, "tabular")) predictions <- predictions[, na.omit(match(c("class", "risk"), colnames(predictions)))]

+        calcExternalPerformance(predictions, testOutcomes, performanceType)

+      }, measurements, outcomes)

+    })

+

+    realPerformance <- matrix(unlist(performanceAllPairs), ncol = length(measurements), byrow = TRUE,

+                              dimnames = list(paste("Select and Train", names(measurements)), paste("Predict", names(measurements))))

+    realPerformance <- round(realPerformance, 2)

+  } else { # trainType is "modelTest".

+    trainedModels <- mapply(function(measurementsOne, outcomesOne)

+    {

+      crossValidate(measurementsOne, outcomesOne,

+                    nFeatures = nFeatures,

+                    selectionMethod = selectionMethod,

+                    selectionOptimisation = selectionOptimisation,

+                    classifier = classifier,

+                    multiViewMethod = "none",

+                    nFolds = nFolds,

+                    nCores = nCores,

+                    nRepeats = nRepeats)

+     }, measurements, outcomes, SIMPLIFY = FALSE)

+    

+    # Make it for runTests, which allows existing results to be passed into selection process.

+    crossValParams <- generateCrossValParams(nRepeats, nFolds, nCores, selectionOptimisation)

+

+    performanceAllPairs <- lapply(trainedModels, function(trainedModel)

+    {

+      mapply(function(measurementsOne, outcomesOne)

+      {

+        classifierParams <- .classifierKeywordToParams(classifier)

+        modellingParams <- ModellingParams(selectParams = SelectParams("previousSelection", intermediate = ".iteration", classifyResult = trainedModel),

+                                           trainParams = classifierParams$trainParams,

+                                           predictParams = classifierParams$predictParams)

+        

+        result <- runTests(measurementsOne, outcomesOne, crossValParams, modellingParams)

+        mean(performance(calcCVperformance(result, performanceType))[[performanceType]])

+      }, measurements, outcomes, SIMPLIFY = FALSE)

+     })

+    

+    realPerformance <- matrix(unlist(performanceAllPairs), ncol = length(measurements), byrow = TRUE,

+                              dimnames = list(paste("Select", names(measurements)), paste("Cross-validate", names(measurements))))

+    realPerformance <- round(realPerformance, 2)

+  }

+

+  # Return matrix of pair-wise model accuracy. 

+  # I've made this a list so that I can add things to it later on. 

+  result <- list(real = realPerformance)

+  # We want to include a set of nFeatures to compare between our feature selection method.

+  if(doRandomFeatures == TRUE){

+    message("Starting random feature selection procedure.")

+    # Sample nFeatures randomly from each dataset.

+    randomFeatures <- lapply(measurements, function(dataset) sample(colnames(dataset), nFeatures))

+    performanceAllPairs <- lapply(randomFeatures, function(randomFeaturesSet)

+    {

+      mapply(function(testData, testOutcomes)

+      {

+        result <- crossValidate(testData[, randomFeaturesSet], testOutcomes,

+                    nFeatures = nFeatures,

+                    selectionMethod = "none",

+                    classifier = classifier,

+                    multiViewMethod = "none",

+                    nFolds = nFolds,

+                    nCores = nCores,

+                    nRepeats = nRepeats)

+        mean(performance(calcCVperformance(result, performanceType))[[performanceType]])

+      }, measurements, outcomes)

+    })

+

+    randomPerformance <- matrix(unlist(performanceAllPairs), ncol = length(measurements), byrow = TRUE,

+                              dimnames = list(paste("Random Select", names(measurements)), paste("Cross-validate", names(measurements))))

+    randomPerformance <- round(randomPerformance, 2)

+    

+    result$random <- randomPerformance

+  }

+  

+# Add information about the params to the output.

+  result$params <- list(nFeatures = nFeatures, selectionMethod = selectionMethod,

+                     selectionOptimisation = selectionOptimisation,

+                     classifier = classifier, nFolds = nFolds, nRepeats = nRepeats, nCores = nCores,

+                     trainType = trainType, performanceType = performanceType,

+                     doRandomFeatures = doRandomFeatures)

+  

+  result

+}

+

+#' A function to plot the output of the crissCrossValidate function.

+#'

+#' This function has been designed to give a heatmap output of the crissCrossValidate function.

+#'

+#' @param crissCrossResult The output of the crissCrossValidate function.

+#' @param includeValues If TRUE, then the values of the matrix will be included in the plot.

+#' @author Harry Robertson

+#' 

+#' @import ggplot2

+#' @import reshape2

+#' @import ggpubr

+#' 

+#' @export

+

+crissCrossPlot <- function(crissCrossResult, includeValues = FALSE){

+

+  attach(crissCrossResult)

+  scalebar_title <- params$performanceType

+

+  # If the user does not want to compare features.

+  if(params$trainType == "modelTrain"){

+    melted_cormat <- reshape2::melt(real, na.rm = TRUE)

+  

+    ggheatmap <- ggplot(melted_cormat, aes(Var1, Var2, fill = value)) +

+      geom_tile(color = "white") +

+      scale_fill_gradient2(high = "red", mid = "white", low = "blue", 

+                          midpoint = 0.5, limit = c(0,1), space = "Lab", 

+                          name=as.character(scalebar_title)) +

+      theme_bw() + xlab("Training Dataset") + ylab("Testing Dataset") +

+      theme(axis.text.x = element_text(angle = 90, vjust = 1, size = 8, hjust = 1)) + 

+      theme(axis.text.y = element_text(vjust = 1, size = 8, hjust = 1)) +

+      coord_fixed() 

+    

+    if(includeValues == TRUE) ggheatmap <- ggheatmap + geom_text(aes(label = value), color = "black", size = 3)

+  }

+  

+  else if(params$trainType == "modelTest"){

+    melted_cormat_1 <- melt(real, na.rm = TRUE)

+    ggheatmap_1 <- ggplot(melted_cormat_1, aes(Var1, Var2, fill = value)) +

+      geom_tile(color = "white") +

+      scale_fill_gradient2(high = "red", mid = "white", low = "blue", 

+                          midpoint = 0.5, limit = c(0,1), space = "Lab", 

+                          name=as.character(scalebar_title)) +

+      theme_bw() + xlab("Features Extracted") + ylab("Dataset Tested") +

+      theme(axis.text.x = element_text(angle = 90, vjust = 1, size = 8, hjust = 1)) + 

+      theme(axis.text.y = element_text(vjust = 1, size = 8, hjust = 1)) +

+      coord_fixed()

+    if(includeValues == TRUE) ggheatmap_1 <- ggheatmap_1 + geom_text(aes(label = value), color = "black", size = 3)

+    

+    if(params$doRandomFeatures == TRUE){

+      melted_cormat_2 <- melt(random, na.rm = TRUE)

+      ggheatmap_2 <- ggplot(melted_cormat_2, aes(Var1, Var2, fill = value)) +

+        geom_tile(color = "white") +

+        scale_fill_gradient2(high = "red", mid = "white", low = "blue", 

+                            midpoint = 0.5, limit = c(0,1), space = "Lab", 

+                            name=as.character(scalebar_title)) +

+        theme_bw() + xlab("Features Extracted") + ylab("Dataset Tested") +

+        theme(axis.text.x = element_text(angle = 90, vjust = 1, size = 8, hjust = 1)) + 

+        theme(axis.text.y = element_text(vjust = 1, size = 8, hjust = 1)) +

+        coord_fixed()

+      if(includeValues == TRUE) ggheatmap_2 <- ggheatmap_2 + geom_text(aes(label = value), color = "black", size = 3)

+

+        ggheatmap <- ggarrange(ggheatmap_1, ggheatmap_2, labels = c("A - Feature Selection", "B - Random Features"), 

+                           ncol = 2, common.legend = TRUE, legend = "right")

+    } else {

+      ggheatmap <- ggheatmap_1

+    }

+  }

+  print(ggheatmap)

+}

\ No newline at end of file

@@ -22,8 +22,8 @@

 #' or assays. If a numeric vector these will be optimised over using \code{selectionOptimisation}. If a named vector with the same names of multiple assays, 

 #' a different number of features will be used for each assay. If a named list of vectors, the respective number of features will be optimised over. 

 #' Set to NULL or "all" if all features should be used.

-#' @param selectionMethod Default: "auto". A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

-#' and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"} t-test (two categories) / F-test (three or more categories) ranking

+#' @param selectionMethod Default: \code{"auto"}. A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

+#' and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"}, t-test (two categories) / F-test (three or more categories) ranking

 #' and top \code{nFeatures} optimisation is done. Otherwise, the ranking method is per-feature Cox proportional hazards p-value.

 #' @param selectionOptimisation A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.

 #' @param performanceType Default: \code{"auto"}. If \code{"auto"}, then balanced accuracy for classification or C-index for survival. Otherwise, any one of the

@@ -91,10 +91,10 @@ setMethod("crossValidate", "DataFrame",

           function(measurements,

                    outcome,

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -297,10 +297,10 @@ setMethod("crossValidate", "MultiAssayExperiment",

           function(measurements,

                    outcome, 

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -331,10 +331,10 @@ setMethod("crossValidate", "data.frame", # data.frame of numeric measurements.

           function(measurements,

                    outcome, 

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -364,10 +364,10 @@ setMethod("crossValidate", "matrix", # Matrix of numeric measurements.

           function(measurements,

                    outcome,

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -399,10 +399,10 @@ setMethod("crossValidate", "list",

           function(measurements,

                    outcome, 

                    nFeatures = 20,

-                   selectionMethod = "t-test",

+                   selectionMethod = "auto",

                    selectionOptimisation = "Resubstitution",

                    performanceType = "auto",

-                   classifier = "randomForest",

+                   classifier = "auto",

                    multiViewMethod = "none",

                    assayCombinations = "all",

                    nFolds = 5,

@@ -764,19 +764,18 @@ generateMultiviewParams <- function(assayIDs,

 }

 # measurements, outcome are mutually exclusive with x, outcomeTrain, measurementsTest, outcomeTest.

-CV <- function(measurements = NULL,

-               outcome = NULL, x = NULL, outcomeTrain = NULL, measurementsTest = NULL, outcomeTest = NULL,

+CV <- function(measurements, outcome, x, outcomeTrain, measurementsTest, outcomeTest,

                assayIDs,

-               nFeatures = NULL,

-               selectionMethod = "t-test",

-               selectionOptimisation = "Resubstitution",

+               nFeatures,

+               selectionMethod,

+               selectionOptimisation,

                performanceType,

-               classifier = "elasticNetGLM",

-               multiViewMethod = "none",

-               nFolds = 5,

-               nRepeats = 100,

-               nCores = 1,

-               characteristicsLabel = NULL)

+               classifier,

+               multiViewMethod,

+               nFolds,

+               nRepeats,

+               nCores,

+               characteristicsLabel)

 {

     # Which data-types or data-views are present?

@@ -848,7 +847,7 @@ train.data.frame <- function(x, outcomeTrain, ...)

 #' @param performanceType Performance metric to optimise if classifier has any tuning parameters.

 #' @method train DataFrame

 #' @export

-train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", performanceType = "auto",

+train.DataFrame <- function(x, outcomeTrain, selectionMethod = "auto", nFeatures = 20, classifier = "auto", performanceType = "auto",

                             multiViewMethod = "none", assayIDs = "all", ...) # ... for prepareData.

                    {

               prepArgs <- list(x, outcomeTrain)

@@ -863,18 +862,20 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", perfor

               # Ensure performance type is one of the ones that can be calculated by the package.

               if(!performanceType %in% c("auto", .ClassifyRenvir[["performanceTypes"]]))

                 stop(paste("performanceType must be one of", paste(c("auto", .ClassifyRenvir[["performanceTypes"]]), collapse = ", "), "but is", performanceType))

-              

+

+              isCategorical <- is.character(outcomeTrain) && (length(outcomeTrain) == 1 || length(outcomeTrain) == nrow(measurements)) || is.factor(outcomeTrain)

               if(performanceType == "auto")

-              {

-                if(is.character(outcomeTrain) && (length(outcomeTrain) == 1 || length(outcomeTrain) == nrow(x)) || is.factor(outcomeTrain))

-                  performanceType <- "Balanced Accuracy"

-                else performanceType <- "C-index"

-              }

+                if(isCategorical) performanceType <- "Balanced Accuracy" else performanceType <- "C-index"

+              if(length(selectionMethod) == 1 && selectionMethod == "auto")

+                if(isCategorical) selectionMethod <- "t-test" else selectionMethod <- "CoxPH"

+              if(length(classifier) == 1 && classifier == "auto")

+                if(isCategorical) classifier <- "randomForest" else classifier <- "CoxPH"

               measurements <- measurementsAndOutcome[["measurements"]]

               outcomeTrain <- measurementsAndOutcome[["outcome"]]

               classifier <- cleanClassifier(classifier = classifier, measurements = measurements)

+              selectionMethod <- cleanSelectionMethod(selectionMethod = selectionMethod, measurements = measurements)

               if(assayIDs == "all") assayIDs <- unique(S4Vectors::mcols(measurements)[, "assay"])

               if(is.null(assayIDs)) assayIDs <- 1

               names(assayIDs) <- assayIDs

@@ -886,30 +887,52 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", perfor

                           # Loop over assays

                           sapply(classifier[[assayIndex]], function(classifierForAssay) {

                               # Loop over classifiers

+                                sapply(selectionMethod[[assayIndex]], function(selectionForAssay) {

+                                  # Loop over selectors

                                   measurementsUse <- measurements

                                   if(assayIndex != 1) measurementsUse <- measurements[, S4Vectors::mcols(measurements)[, "assay"] == assayIndex, drop = FALSE]

+                                  modellingParams <- generateModellingParams(assayIDs = assayIDs, measurements = measurements, nFeatures = nFeatures,

+                                                     selectionMethod = selectionMethod, selectionOptimisation = "Resubstitution", performanceType = performanceType,

+                                                     classifier = classifier, multiViewMethod = "none")

+                                  topFeatures <- .doSelection(measurementsUse, outcomeTrain, CrossValParams(), modellingParams, verbose = 0)

+                                  selectedFeaturesIndices <- topFeatures[[2]] # Extract for subsetting.

+                                  tuneDetailsSelect <- topFeatures[[3]]

+                                  measurementsUse <- measurementsUse[, selectedFeaturesIndices]

+

                                   classifierParams <- .classifierKeywordToParams(classifierForAssay)

-                                  if(!is.null(classifierParams$trainParams@tuneParams))

-                                    classifierParams$trainParams@tuneParams <- c(classifierParams$trainParams@tuneParams, performanceType = performanceType)

                                   modellingParams <- ModellingParams(balancing = "none", selectParams = NULL,

-                                                               trainParams = classifierParams$trainParams, predictParams = classifierParams$predictParams)

+                                                                     trainParams = classifierParams$trainParams, predictParams = classifierParams$predictParams)

+                                  if(!is.null(tuneDetailsSelect))

+                                  {

+                                    tuneDetailsSelectUse <- tuneDetailsSelect[["tuneCombinations"]][tuneDetailsSelect[["bestIndex"]], , drop = FALSE]

+                                    avoidTune <- match(colnames(tuneDetailsSelectUse), names(modellingParams@trainParams@tuneParams))

+                                    if(any(!is.na(avoidTune)))

+                                    {

+                                      modellingParams@trainParams@otherParams <- c(modellingParams@trainParams@otherParams, tuneDetailsSelectUse[!is.na(avoidTune)])

+                                      modellingParams@trainParams@tuneParams <- modellingParams@trainParams@tuneParams[-na.omit(avoidTune)]

+                                      if(length(modellingParams@trainParams@tuneParams) == 0) modellingParams@trainParams@tuneParams <- NULL

+                                    }

+                                  }

+                                  if(!is.null(modellingParams@trainParams@tuneParams))

+                                    modellingParams$trainParams@tuneParams <- c(modellingParams$trainParams@tuneParams, performanceType = performanceType)

-                                  .doTrain(measurementsUse, outcomeTrain, NULL, NULL, CrossValParams(), modellingParams, verbose = 0)[["model"]]

+                                  trained <- .doTrain(measurementsUse, outcomeTrain, NULL, NULL, CrossValParams(), modellingParams, verbose = 0)[["model"]]

+                                  attr(trained, "predictFunction") <- classifierParams$predictParams@predictor

+                                  trained

                                   ## train model

-                          },

-                          simplify = FALSE)

-                      },

-                      simplify = FALSE)

+                                }, simplify = FALSE)

+                          }, simplify = FALSE)

+                      }, simplify = FALSE)

-                  models <- unlist(resClassifier, recursive = FALSE)

+                  models <- unlist(unlist(resClassifier, recursive = FALSE), recursive = FALSE)

                   if(length(models) == 1) {

                       model <- models[[1]]

-                      class(model) <- c(class(model), "trainedByClassifyR")

+                      class(model) <- c("trainedByClassifyR", class(model))

                       models <- NULL

                   } else {

-                      class(models) <- c(class(models), "listOfModels", "trainedByClassifyR")

+                      class(models) <- c("listOfModels", "trainedByClassifyR", class(models))

                   }

               }

@@ -932,6 +955,8 @@ train.DataFrame <- function(x, outcomeTrain, classifier = "randomForest", perfor

                # Generate params for each assay. This could be extended to have different selectionMethods for each type

                  paramsAssays <- mapply(generateModellingParams,

+                                        nFeatures = nFeatures[assayIDs],

+                                        selectionMethod = selectionMethod[assayIDs],

                                         assayIDs = assayIDs,

                                         measurements = assayTrain[assayIDs],

                                         classifier = classifier[assayIDs],

@@ -1049,9 +1074,11 @@ predict.trainedByClassifyR <- function(object, newData, ...)

               newData <- prepareData(newData, useFeatures = allFeatureNames(object))

               # Some classifiers dangerously use positional matching rather than column name matching.

               # newData columns are sorted so that the right column ordering is guaranteed.

-            }

-

+    }

+    

+    predictFunctionUse <- attr(object, "predictFunction")

+    class(object) <- rev(class(object)) # Now want the predict method of the specific model to be picked, so put model class first.

     if (is(object, "listOfModels")) 

-         mapply(function(model, assay) predict(model, assay), object, newData, SIMPLIFY = FALSE)

-    else predict(object, newData) # Object is itself a trained model and it is assumed that a predict method is defined for it.

+         mapply(function(model, assay) predictFunctionUse(model, assay), object, newData, SIMPLIFY = FALSE)

+    else predictFunctionUse(object, newData) # Object is itself a trained model and it is assumed that a predict method is defined for it.

 }

@@ -47,6 +47,14 @@ setMethod("prepareData", "matrix",

   prepareData(S4Vectors::DataFrame(measurements, check.names = FALSE), outcome, ...)

 })

+#' @rdname prepareData

+#' @export

+setMethod("prepareData", "data.frame",

+  function(measurements, outcome, ...)

+{

+  prepareData(S4Vectors::DataFrame(measurements, check.names = FALSE), outcome, ...)

+})

+

 #' @rdname prepareData

 #' @export

 setMethod("prepareData", "DataFrame",

new file mode 100644

@@ -0,0 +1,9 @@

+# Automated Selection of Previously Selected Features

+randomSelection <- function(measurementsTrain, classesTrain, nFeatures, verbose = 3)

+{

+  if(verbose == 3)

+    message("Choosing random features.")

+

+  sample(ncol(measurementsTrain), nFeatures) # Return indices, not identifiers.

+}

+attr(randomSelection, "name") <- "randomSelection"

\ No newline at end of file

@@ -26,7 +26,7 @@

 #' @param results A list of \code{\link{ClassifyResult}} objects.

 #' @param topRanked A sequence of thresholds of number of the best features to

 #' use for overlapping.

-#' @param comparison Default: within. The aspect of the experimental design to

+#' @param comparison Default: \code{"within"}. The aspect of the experimental design to

 #' compare. Can be any characteristic that all results share or special value

 #' "within" to compared between all pairwise iterations of cross-validation.

 #' @param referenceLevel The level of the comparison factor to use as the

@@ -13,7 +13,7 @@

 #' a matrix of pre-calculated metrics, for backwards compatibility.

 #' @param classes If \code{results} is a matrix, this is a factor vector of the

 #' same length as the number of columns that \code{results} has.

-#' @param comparison Default: "auto". The aspect of the experimental

+#' @param comparison Default: \code{"auto"}. The aspect of the experimental

 #' design to compare. Can be any characteristic that all results share.

 #' @param metric Default: \code{"auto"}. The name of the

 #' performance measure or "auto". If the results are classification then

@@ -36,7 +36,7 @@

 #' 

 #' @aliases selectionPlot selectionPlot,list-method

 #' @param results A list of \code{\link{ClassifyResult}} objects.

-#' @param comparison Default: within. The aspect of the experimental design to

+#' @param comparison Default: \code{"within"}. The aspect of the experimental design to

 #' compare. Can be any characteristic that all results share or either one of

 #' the special values \code{"within"} to compare between all pairwise

 #' iterations of cross-validation. or \code{"size"}, to draw a bar chart of the

@@ -129,10 +129,12 @@

       tuneCombo <- tuneCombosSelect[rowIndex, , drop = FALSE]

       if(tuneCombo != "none") # Add real parameters before function call.

         paramList <- append(paramList, tuneCombo)

+      if(attr(featureRanking, "name") == "randomSelection")

+        paramList <- append(paramList, nFeatures = topNfeatures)

       do.call(featureRanking, paramList)

     })

-    if(attr(featureRanking, "name") %in% c("previousSelection", "Union Selection")) # Actually selection not ranking.

+    if(attr(featureRanking, "name") %in% c("randomSelection", "previousSelection", "Union Selection")) # Actually selection not ranking.

       return(list(NULL, rankings[[1]], NULL))

     if(crossValParams@tuneMode == "none") # No parameters to choose between.

@@ -509,6 +511,8 @@

         "KS" = KolmogorovSmirnovRanking,

         "KL" = KullbackLeiblerRanking,

         "CoxPH" = coxphRanking,

+        "previousSelection" = previousSelection,

+        "randomSelection" = randomSelection,

         "selectMulti" = selectMulti

     )

 }

@@ -16,7 +16,7 @@ ModellingParams(

 )

 }

 \arguments{

-\item{balancing}{Default: "downsample". A character value specifying what kind

+\item{balancing}{Default: \code{"downsample"}. A character value specifying what kind

 of class balancing to do, if any.}

 \item{transformParams}{Parameters used for feature transformation inside of C.V.

@@ -30,7 +30,7 @@

 \item{...}{Parameters not used by the \code{ClassifyResult} method but passed to

 the \code{list} method.}

-\item{mode}{Default: "merge". Whether to merge all predictions of all

+\item{mode}{Default: \code{"merge"}. Whether to merge all predictions of all

 iterations of cross-validation into one set or keep them separate. Keeping

 them separate will cause separate ROC curves to be computed for each

 iteration and confidence intervals to be drawn with the solid line being the

new file mode 100644

@@ -0,0 +1,19 @@

+% Generated by roxygen2: do not edit by hand

+% Please edit documentation in R/crissCrossValidate.R

+\name{crissCrossPlot}

+\alias{crissCrossPlot}

+\title{A function to plot the output of the crissCrossValidate function.}

+\usage{

+crissCrossPlot(crissCrossResult, includeValues = FALSE)

+}

+\arguments{

+\item{crissCrossResult}{The output of the crissCrossValidate function.}

+

+\item{includeValues}{If TRUE, then the values of the matrix will be included in the plot.}

+}

+\description{

+This function has been designed to give a heatmap output of the crissCrossValidate function.

+}

+\author{

+Harry Robertson

+}

new file mode 100644

@@ -0,0 +1,64 @@

+% Generated by roxygen2: do not edit by hand

+% Please edit documentation in R/crissCrossValidate.R

+\name{crissCrossValidate}

+\alias{crissCrossValidate}

+\title{A function to perform pairwise cross validation}

+\usage{

+crissCrossValidate(

+  measurements,

+  outcomes,

+  nFeatures = 20,

+  selectionMethod = "auto",

+  selectionOptimisation = "Resubstitution",

+  trainType = c("modelTrain", "modelTest"),

+  performanceType = "auto",

+  doRandomFeatures = FALSE,

+  classifier = "auto",

+  nFolds = 5,

+  nRepeats = 20,

+  nCores = 1

+)

+}

+\arguments{

+\item{measurements}{A \code{list} of either \code{\link{DataFrame}}, \code{\link{data.frame}} or \code{\link{matrix}} class measurements.}

+

+\item{outcomes}{A \code{list} of vectors that respectively correspond to outcomes of the samples in \code{measurements} list.}

+

+\item{nFeatures}{The number of features to be used for modelling.}

+

+\item{selectionMethod}{Default: \code{"auto"}. A character keyword of the feature algorithm to be used. If \code{"auto"}, t-test (two categories) /

+F-test (three or more categories) ranking and top \code{nFeatures} optimisation is done. Otherwise, the ranking method is per-feature Cox proportional

+hazards p-value.}

+

+\item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise nFeatures.}

+

+\item{trainType}{Default: \code{"modelTrain"}. A keyword specifying whether a fully trained model is used to make predictions on the test

+set or if only the feature identifiers are chosen using the training data set and a number of training-predictions are made by cross-validation

+in the test set.}

+

+\item{performanceType}{Default: \code{"auto"}. If \code{"auto"}, then balanced accuracy for classification or C-index for survival. Otherwise, any one of the

+options described in \code{\link{calcPerformance}} may otherwise be specified.}

+

+\item{doRandomFeatures}{Default: \code{FALSE}. Whether to perform random feature selection to establish a baseline performance. Either \code{FALSE} or \code{TRUE}

+are permitted values.}

+

+\item{classifier}{Default: \code{"auto"}. A character keyword of the modelling algorithm to be used. If \code{"auto"}, then a random forest is used

+for a classification task or Cox proportional hazards model for a survival task.}

+

+\item{nFolds}{A numeric specifying the number of folds to use for cross-validation.}

+

+\item{nRepeats}{A numeric specifying the the number of repeats or permutations to use for cross-validation.}

+

+\item{nCores}{A numeric specifying the number of cores used if the user wants to use parallelisation.}

+}

+\value{

+A list with elements \code{"real"} for the matrix of pairwise performance metrics using real

+feature selection, \code{"random"} if \code{doRandomFeatures} is \code{TRUE} for metrics of random selection and

+\code{"params"} for a list of parameters used during the execution of this function.

+}

+\description{

+This function has been designed to perform cross-validation and model prediction on datasets in a pairwise manner.

+}

+\author{

+Harry Robertson

+}

@@ -22,10 +22,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -39,10 +39,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -56,10 +56,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -73,10 +73,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -90,10 +90,10 @@ crossValidate(measurements, outcome, ...)

   measurements,

   outcome,

   nFeatures = 20,

-  selectionMethod = "t-test",

+  selectionMethod = "auto",

   selectionOptimisation = "Resubstitution",

   performanceType = "auto",

-  classifier = "randomForest",

+  classifier = "auto",

   multiViewMethod = "none",

   assayCombinations = "all",

   nFolds = 5,

@@ -110,7 +110,9 @@ crossValidate(measurements, outcome, ...)

 \method{train}{DataFrame}(

   x,

   outcomeTrain,

-  classifier = "randomForest",

+  selectionMethod = "auto",

+  nFeatures = 20,

+  classifier = "auto",

   performanceType = "auto",

   multiViewMethod = "none",

   assayIDs = "all",

@@ -141,8 +143,8 @@ or assays. If a numeric vector these will be optimised over using \code{selectio

 a different number of features will be used for each assay. If a named list of vectors, the respective number of features will be optimised over. 

 Set to NULL or "all" if all features should be used.}

-\item{selectionMethod}{Default: "auto". A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

-and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"} t-test (two categories) / F-test (three or more categories) ranking

+\item{selectionMethod}{Default: \code{"auto"}. A character vector of feature selection methods to compare. If a named character vector with names corresponding to different assays, 

+and performing multiview classification, the respective classification methods will be used on each assay. If \code{"auto"}, t-test (two categories) / F-test (three or more categories) ranking

 and top \code{nFeatures} optimisation is done. Otherwise, the ranking method is per-feature Cox proportional hazards p-value.}

 \item{selectionOptimisation}{A character of "Resubstitution", "Nested CV" or "none" specifying the approach used to optimise \code{nFeatures}.}

@@ -5,10 +5,13 @@

 \alias{prepareData,matrix-method}

 \alias{prepareData,DataFrame-method}

 \alias{prepareData,MultiAssayExperiment-method}

+\alias{prepareData,data.frame-method}

 \title{Convert Different Data Classes into DataFrame and Filter Features}

 \usage{

 \S4method{prepareData}{matrix}(measurements, outcome, ...)

+\S4method{prepareData}{data.frame}(measurements, outcome, ...)

+

 \S4method{prepareData}{DataFrame}(

   measurements,

   outcome,

@@ -37,7 +37,7 @@

 \item{topRanked}{A sequence of thresholds of number of the best features to

 use for overlapping.}

-\item{comparison}{Default: within. The aspect of the experimental design to

+\item{comparison}{Default: \code{"within"}. The aspect of the experimental design to

 compare. Can be any characteristic that all results share or special value

 "within" to compared between all pairwise iterations of cross-validation.}

@@ -61,7 +61,7 @@ a matrix of pre-calculated metrics, for backwards compatibility.}

 \item{...}{Parameters not used by the \code{ClassifyResult} method that does

 list-packaging but used by the main \code{list} method.}

-\item{comparison}{Default: "auto". The aspect of the experimental

+\item{comparison}{Default: \code{"auto"}. The aspect of the experimental

 design to compare. Can be any characteristic that all results share.}

 \item{metric}{Default: \code{"auto"}. The name of the