@@ -3,16 +3,16 @@ Type: Package

 Title: A framework for cross-validated classification problems, with

        applications to differential variability and differential

        distribution testing

-Version: 3.1.7

-Date: 2022-06-30

+Version: 3.1.8

+Date: 2022-07-12

 Author: Dario Strbenac, Ellis Patrick, John Ormerod, Graham Mann, Jean Yang

 Maintainer: Dario Strbenac <dario.strbenac@sydney.edu.au>

 VignetteBuilder: knitr

 Encoding: UTF-8

 biocViews: Classification, Survival

 Depends: R (>= 4.1.0), methods, S4Vectors (>= 0.18.0), MultiAssayExperiment (>= 1.6.0), BiocParallel, survival

-Imports: grid, utils, dplyr, tidyr, rlang, randomForest

-Suggests: limma, genefilter, edgeR, car, Rmixmod, ggplot2 (>= 3.0.0), gridExtra (>= 2.0.0), cowplot,

+Imports: grid, genefilter, utils, dplyr, tidyr, rlang, randomForest

+Suggests: limma, edgeR, car, Rmixmod, ggplot2 (>= 3.0.0), gridExtra (>= 2.0.0), cowplot,

         BiocStyle, pamr, PoiClaClu, parathyroidSE, knitr, htmltools, gtable,

         scales, e1071, rmarkdown, IRanges, robustbase, glmnet, class

 Description: The software formalises a framework for classification in R.

@@ -22,7 +22,6 @@ export(SelectParams)

 export(TrainParams)

 export(TransformParams)

 export(actualOutcomes)

-export(allFeatureNames)

 export(bartlettRanking)

 export(calcCVperformance)

 export(calcExternalPerformance)

@@ -42,6 +41,7 @@ export(elasticNetFeatures)

 export(elasticNetGLMpredictInterface)

 export(elasticNetGLMtrainInterface)

 export(featureSetSummary)

+export(featuresInfo)

 export(fisherDiscriminant)

 export(forestFeatures)

 export(generateCrossValParams)

@@ -105,7 +105,6 @@ exportMethods(SelectParams)

 exportMethods(TrainParams)

 exportMethods(TransformParams)

 exportMethods(actualOutcomes)

-exportMethods(allFeatureNames)

 exportMethods(bartlettRanking)

 exportMethods(calcCVperformance)

 exportMethods(calcExternalPerformance)

@@ -124,6 +123,7 @@ exportMethods(elasticNetFeatures)

 exportMethods(elasticNetGLMpredictInterface)

 exportMethods(elasticNetGLMtrainInterface)

 exportMethods(featureSetSummary)

+exportMethods(featuresInfo)

 exportMethods(fisherDiscriminant)

 exportMethods(forestFeatures)

 exportMethods(getLocationsAndScales)

@@ -50,30 +50,32 @@

 #' @author Dario Strbenac

 #' @examples

 #' 

-#'   predicted <- do.call(rbind, list(data.frame(data.frame(sample = LETTERS[c(1, 8, 15, 3, 11, 20, 19, 18)],

-#'                                Healthy = c(0.89, 0.68, 0.53, 0.76, 0.13, 0.20, 0.60, 0.25),

-#'                                Cancer = c(0.11, 0.32, 0.47, 0.24, 0.87, 0.80, 0.40, 0.75),

+#'   predicted <- do.call(rbind, list(DataFrame(data.frame(sample = LETTERS[seq(1, 20, 2)],

+#'                                Healthy = c(0.89, 0.68, 0.53, 0.76, 0.13, 0.20, 0.60, 0.25, 0.10, 0.30),

+#'                                Cancer = c(0.11, 0.32, 0.47, 0.24, 0.87, 0.80, 0.40, 0.75, 0.90, 0.70),

 #'                                fold = 1)),

-#'                     data.frame(sample = LETTERS[c(11, 18, 15, 4, 6, 10, 11, 12)],

-#'                                Healthy = c(0.45, 0.56, 0.33, 0.56, 0.33, 0.20, 0.60, 0.40),

-#'                                Cancer = c(0.55, 0.44, 0.67, 0.44, 0.67, 0.80, 0.40, 0.60),

+#'                     DataFrame(sample = LETTERS[seq(2, 20, 2)],

+#'                                Healthy = c(0.45, 0.56, 0.33, 0.56, 0.65, 0.33, 0.20, 0.60, 0.40, 0.80),

+#'                                Cancer = c(0.55, 0.44, 0.67, 0.44, 0.35, 0.67, 0.80, 0.40, 0.60, 0.20),

 #'                                fold = 2)))

 #'   actual <- factor(c(rep("Healthy", 10), rep("Cancer", 10)), levels = c("Healthy", "Cancer"))

-#'   result1 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name",

-#'                                                          "Cross-validation"),

-#'                             value = c("Melanoma", "t-test", "Random Forest", "2 Permutations, 2 Folds")),

-#'                             LETTERS[1:20], LETTERS[10:1],

-#'                             list(1:100, c(1:9, 11:101)), list(sample(10, 10), sample(10, 10)),

+#'   result1 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name", "Cross-validation"),

+#'                             value = c("Melanoma", "t-test", "Random Forest", "2-fold")),

+#'                             LETTERS[1:20], DataFrame(`Original Feature` = paste("Gene", LETTERS[1:10]),

+#'                             `Renamed Feature` = paste("Feature", 1:10, sep = ''), check.names = FALSE),

+#'                             list(paste("Gene", LETTERS[1:10]), paste("Gene", LETTERS[c(5:1, 6:10)])),

+#'                             list(paste("Gene", LETTERS[1:3]), paste("Gene", LETTERS[1:5])),

 #'                             list(function(oracle){}), NULL, predicted, actual)

 #'   

 #'   predicted[c(2, 6), "Healthy"] <- c(0.40, 0.60)

 #'   predicted[c(2, 6), "Cancer"] <- c(0.60, 0.40)

-#'   result2 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name",

-#'                                                          "Cross-validation"),

-#'                             value = c("Example", "Bartlett Test", "Differential Variability", "2 Permutations, 2 Folds")),

-#'                             LETTERS[1:20], LETTERS[10:1], list(1:100, c(1:5, 11:105)),

-#'                             list(sample(10, 10), sample(10, 10)), list(function(oracle){}),

-#'                             NULL, predicted, actual)

+#'   result2 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name", "Cross-validation"),

+#'                                       value = c("Example", "Bartlett Test", "Differential Variability", "2-fold")),

+#'                             LETTERS[1:20], DataFrame(`Original Feature` = paste("Gene", LETTERS[1:10]),

+#'                             `Renamed Feature` = paste("Feature", 1:10, sep = ''), check.names = FALSE),

+#'                             list(paste("Gene", LETTERS[1:10]), paste("Gene", LETTERS[c(5:1, 6:10)])),

+#'                             list(paste("Gene", LETTERS[1:3]), paste("Gene", LETTERS[1:5])),

+#'                             list(function(oracle){}), NULL, predicted, actual)

 #'   ROCplot(list(result1, result2), plotTitle = "Cancer ROC")

 #'

 #' @usage NULL

@@ -70,13 +70,14 @@

 #' @author Dario Strbenac

 #' @examples

 #' 

-#'   predictTable <- data.frame(sample = paste("A", 1:10, sep = ''),

-#'                              class = factor(sample(LETTERS[1:2], 50, replace = TRUE)))

+#'   predictTable <- DataFrame(sample = paste("A", 1:10, sep = ''),

+#'                             class = factor(sample(LETTERS[1:2], 50, replace = TRUE)))

 #'   actual <- factor(sample(LETTERS[1:2], 10, replace = TRUE))                             

-#'   result <- ClassifyResult(DataFrame(),

-#'                            paste("A", 1:10, sep = ''), paste("Gene", 1:50, sep = ''),

-#'                            list(1:50, 1:50), list(1:5, 6:15), list(function(oracle){}), NULL,

-#'                            predictTable, actual)

+#'   result <- ClassifyResult(DataFrame(characteristic = "Data Set", value = "Example"),

+#'                            paste("A", 1:10, sep = ''), DataFrame(`Original Feature` = paste("Gene", 1:50),

+#'                            `Renamed Feature` = paste("Feature", 1:50, sep = '')),

+#'                            list(paste("Gene", 1:50), paste("Gene", 1:50)), list(paste("Gene", 1:5), paste("Gene", 1:10)),

+#'                            list(function(oracle){}), NULL, predictTable, actual)

 #'   result <- calcCVperformance(result) 

 #'   performance(result)

 #' 

@@ -1217,7 +1217,7 @@ setClassUnion("ModellingParamsOrNULL", c("ModellingParams", "NULL"))

 #' @aliases ClassifyResult ClassifyResult-class

 #' ClassifyResult,DataFrame,character,characterOrDataFrame-method

 #' show,ClassifyResult-method sampleNames sampleNames,ClassifyResult-method

-#' allFeatureNames allFeatureNames,ClassifyResult-method

+#' featuresInfo featuresInfo,ClassifyResult-method

 #' predictions predictions,ClassifyResult-method actualOutcomes

 #' actualOutcomes,ClassifyResult-method features features,ClassifyResult-method

 #' models models,ClassifyResult-method performance

@@ -1237,9 +1237,8 @@ setClassUnion("ModellingParamsOrNULL", c("ModellingParams", "NULL"))

 #' package, the function names will automatically be generated and therefore it

 #' is not necessary to specify them.}

 #' \item{\code{originalNames}}{All sample names.}

-#' \item{\code{originalFeatures}}{All feature names. Character vector

-#' or \code{\link{DataFrame}} with one row for each feature if the data set has multiple kinds

-#' of measurements on the same set of samples.}

+#' \item{\code{featuresInfo}}{A \code{\link{DataFrame}} containing all feature names in original format

+#' and a safe format without any unusual symbols that R would automatically convert into another format and cause trouble.}

 #' \item{\code{rankedFeatures}}{All features, from most to least important. Character vector

 #' or a data frame if data set has multiple kinds of measurements on the same set of samples.}

 #' \item{\code{chosenFeatures}}{Features selected at each fold. Character

@@ -1267,7 +1266,7 @@ setClassUnion("ModellingParamsOrNULL", c("ModellingParams", "NULL"))

 #' \describe{

 #' \item{\code{sampleNames(result)}}{Returns a vector of sample names present in the data set.}}

 #' \describe{

-#' \item{\code{allFeatureNames(result)}}{Returns a vector of features present in the data set.}}

+#' \item{\code{featuresInfo(result)}}{Returns a table of features present in the data set. Shows original names and renamed names to ensure no unusual symbols in names.}}

 #' \describe{

 #' \item{\code{actualOutcomes(result)}}{Returns the known outcomes of each sample.}}

 #' \describe{

@@ -1308,7 +1307,7 @@ setClassUnion("ModellingParamsOrNULL", c("ModellingParams", "NULL"))

 #' @importFrom S4Vectors as.data.frame

 #' @usage NULL

 #' @export

-setGeneric("ClassifyResult", function(characteristics, originalNames, originalFeatures, ...)

+setGeneric("ClassifyResult", function(characteristics, originalNames, featuresInfo, ...)

 standardGeneric("ClassifyResult"))

 #' @rdname ClassifyResult-class

@@ -1316,13 +1315,13 @@ standardGeneric("ClassifyResult"))

 setClass("ClassifyResult", representation(

   characteristics = "DataFrame",

   originalNames = "character",

-  originalFeatures = "characterOrDataFrame",

+  featuresInfo = "DataFrame",

   rankedFeatures = "listOrNULL",

   chosenFeatures = "listOrNULL",

   actualOutcomes = "factorOrSurv",

   models = "list",

   tune = "listOrNULL",

-  predictions = "data.frame",

+  predictions = "DataFrame",

   performance = "listOrNULL",

   importance = "DataFrameOrNULL",

   modellingParams = "ModellingParamsOrNULL",

@@ -1332,11 +1331,11 @@ setClass("ClassifyResult", representation(

 #' @usage NULL

 #' @export

 setMethod("ClassifyResult", c("DataFrame", "character", "characterOrDataFrame"),

-          function(characteristics, originalNames, originalFeatures,

+          function(characteristics, originalNames, featuresInfo,

                    rankedFeatures, chosenFeatures, models, tunedParameters, predictions, actualOutcomes, importance = NULL, modellingParams = NULL, finalModel = NULL)

           {

             new("ClassifyResult", characteristics = characteristics,

-                originalNames = originalNames, originalFeatures = originalFeatures,

+                originalNames = originalNames, featuresInfo = featuresInfo,

                 rankedFeatures = rankedFeatures, chosenFeatures = chosenFeatures,

                 models = models, tune = tunedParameters,

                 predictions = predictions, actualOutcomes = actualOutcomes, importance = importance, modellingParams = modellingParams, finalModel = finalModel)

@@ -1383,16 +1382,16 @@ setMethod("sampleNames", c("ClassifyResult"),

 #' @rdname ClassifyResult-class

 #' @usage NULL

 #' @export

-setGeneric("allFeatureNames", function(object, ...)

-standardGeneric("allFeatureNames"))

+setGeneric("featuresInfo", function(object, ...)

+standardGeneric("featuresInfo"))

 #' @rdname ClassifyResult-class

 #' @usage NULL

 #' @export

-setMethod("allFeatureNames", c("ClassifyResult"),

+setMethod("featuresInfo", c("ClassifyResult"),

           function(object)

           {

-            object@originalFeatures

+            object@featuresInfo

           })

 #' @rdname ClassifyResult-class

@@ -52,7 +52,7 @@

 #' 

 #' # Compare randomForest and SVM classifiers.

 #' result <- crossValidate(measurements, classes, classifier = c("randomForest", "SVM"))

-#' # performancePlot(result)

+#' performancePlot(result)

 #' 

 #' 

 #' # Compare performance of different datasets. 

@@ -125,11 +125,13 @@ setMethod("crossValidate", "DataFrame",

   Using an ordinary GLM instead.")

                   classifier <- "GLM"

               }

+              

               classifier <- cleanClassifier(classifier = classifier,

                                             measurements = measurements)

               # Which data-types or data-views are present?

               datasetIDs <- unique(mcols(measurements)[, "dataset"])

+              if(is.null(datasetIDs)) datasetIDs <- 1

               ##!!!!! Do something with data combinations

@@ -161,9 +163,10 @@ setMethod("crossValidate", "DataFrame",

                               sapply(selectionMethod[[dataIndex]], function(selectionIndex) {

                                   # Loop over classifiers

                                   set.seed(seed)

+                                  measurementsUse <- measurements

+                                  if(!is.null(mcols(measurements))) measurementsUse <- measurements[, mcols(measurements)[, "dataset"] == dataIndex, drop = FALSE]

                                   CV(

-                                      measurements = measurements[, mcols(measurements)$dataset == dataIndex, drop = FALSE],

-                                      classes = classes,

+                                      measurements = measurementsUse, classes = classes,

                                       nFeatures = nFeatures[dataIndex],

                                       selectionMethod = selectionIndex,

                                       selectionOptimisation = selectionOptimisation,

@@ -201,7 +204,7 @@ setMethod("crossValidate", "DataFrame",

                   # This allows someone to answer which combinations of the datasets might be most useful.

-                  if(is.null(dataCombinations)) dataCombinations <- do.call("c", sapply(seq_len(length(datasetIDs)),function(n)combn(datasetIDs, n, simplify = FALSE)))

+                  if(is.null(dataCombinations)) dataCombinations <- do.call("c", sapply(seq_along(datasetIDs),function(n)combn(datasetIDs, n, simplify = FALSE)))

                   result <- sapply(dataCombinations, function(dataIndex){

                       CV(measurements = measurements[, mcols(measurements)$dataset %in% dataIndex],

@@ -310,7 +313,7 @@ setMethod("crossValidate", "MultiAssayExperiment",

                    characteristicsLabel = NULL)

           {

               targets <- c(names(measurements), "sampleInfo")

-              omicsTargets <- setdiff("sampleInfo", targets)              

+              omicsTargets <- setdiff(targets, "sampleInfo")  

               if(length(omicsTargets) > 0)

               {

                   if(any(anyReplicated(measurements[, , omicsTargets])))

@@ -352,9 +355,6 @@ setMethod("crossValidate", "data.frame", # data.frame of numeric measurements.

                    characteristicsLabel = NULL)

           {

               measurements <- DataFrame(measurements)

-              message(paste("You have", ncol(measurements), "features and", nrow(measurements), "samples and only one data-type."))

-              mcols(measurements)$dataset <- "dataset"

-              mcols(measurements)$feature <- colnames(measurements)

               crossValidate(measurements = measurements,

                             classes = classes, 

                             nFeatures = nFeatures,

@@ -386,8 +386,6 @@ setMethod("crossValidate", "matrix", # Matrix of numeric measurements.

                    characteristicsLabel = NULL)

           {

               measurements <- S4Vectors::DataFrame(measurements, check.names = FALSE)

-              mcols(measurements)$dataset <- "dataset"

-              mcols(measurements)$feature <- colnames(measurements)

               crossValidate(measurements = measurements,

                             classes = classes, 

                             nFeatures = nFeatures,

@@ -407,7 +405,7 @@ setMethod("crossValidate", "matrix", # Matrix of numeric measurements.

 ###!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

 #' @rdname crossValidate

 #' @export

-setMethod("crossValidate", "list", # data.frame of numeric measurements.

+setMethod("crossValidate", "list",

           function(measurements,

                    classes, 

                    nFeatures = 20,

@@ -481,14 +479,16 @@ setMethod("crossValidate", "list", # data.frame of numeric measurements.

 ######################################

 cleanNFeatures <- function(nFeatures, measurements){

     #### Clean up

-    obsFeatures <- unlist(as.list(table(mcols(measurements)[, "dataset"])))

+    if(!is.null(mcols(measurements)))

+      obsFeatures <- unlist(as.list(table(mcols(measurements)[, "dataset"])))

+    else obsFeatures <- ncol(measurements)

     if(is.null(nFeatures) || length(nFeatures) == 1 && nFeatures == "all") nFeatures <- as.list(obsFeatures)

     if(is.null(names(nFeatures)) & length(nFeatures) == 1) nFeatures <- as.list(pmin(obsFeatures, nFeatures))

     if(is.null(names(nFeatures)) & length(nFeatures) > 1) nFeatures <- sapply(obsFeatures, function(x)pmin(x, nFeatures), simplify = FALSE)

     #if(is.null(names(nFeatures)) & length(nFeatures) > 1) stop("nFeatures needs to be a named numeric vector or list with the same names as the datasets.")

-    if(!all(names(obsFeatures) %in% names(nFeatures))) stop("nFeatures needs to be a named numeric vector or list with the same names as the datasets.")

-    if(all(names(obsFeatures) %in% names(nFeatures)) & is(nFeatures, "numeric")) nFeatures <- as.list(pmin(obsFeatures, nFeatures[names(obsFeatures)]))

-    if(all(names(obsFeatures) %in% names(nFeatures)) & is(nFeatures, "list")) nFeatures <- mapply(pmin, nFeatures[names(obsFeatures)], obsFeatures, SIMPLIFY = FALSE)

+    if(!is.null(names(obsFeatures)) && !all(names(obsFeatures) %in% names(nFeatures))) stop("nFeatures needs to be a named numeric vector or list with the same names as the datasets.")

+    if(!is.null(names(obsFeatures)) && all(names(obsFeatures) %in% names(nFeatures)) & is(nFeatures, "numeric")) nFeatures <- as.list(pmin(obsFeatures, nFeatures[names(obsFeatures)]))

+    if(!is.null(names(obsFeatures)) && all(names(obsFeatures) %in% names(nFeatures)) & is(nFeatures, "list")) nFeatures <- mapply(pmin, nFeatures[names(obsFeatures)], obsFeatures, SIMPLIFY = FALSE)

     nFeatures

 }

@@ -496,30 +496,32 @@ cleanNFeatures <- function(nFeatures, measurements){

 ######################################

 cleanSelectionMethod <- function(selectionMethod, measurements){

     #### Clean up

-    obsFeatures <- unlist(as.list(table(mcols(measurements)[, "dataset"])))

+    if(!is.null(mcols(measurements)))

+      obsFeatures <- unlist(as.list(table(mcols(measurements)[, "dataset"])))

+    else return(list(selectionMethod))

-    if(is.null(names(selectionMethod)) & length(selectionMethod) == 1) selectionMethod <- sapply(names(obsFeatures), function(x)selectionMethod, simplify = FALSE)

-    if(is.null(names(selectionMethod)) & length(selectionMethod) > 1) selectionMethod <- sapply(names(obsFeatures), function(x)selectionMethod, simplify = FALSE)

+    if(is.null(names(selectionMethod)) & length(selectionMethod) == 1 & !is.null(names(obsFeatures))) selectionMethod <- sapply(names(obsFeatures), function(x) selectionMethod, simplify = FALSE)

+    if(is.null(names(selectionMethod)) & length(selectionMethod) > 1 & !is.null(names(obsFeatures))) selectionMethod <- sapply(names(obsFeatures), function(x) selectionMethod, simplify = FALSE)

     #if(is.null(names(selectionMethod)) & length(selectionMethod) > 1) stop("selectionMethod needs to be a named character vector or list with the same names as the datasets.")

-    if(!all(names(obsFeatures) %in% names(selectionMethod))) stop("selectionMethod needs to be a named character vector or list with the same names as the datasets.")

-    if(all(names(obsFeatures) %in% names(selectionMethod)) & is(selectionMethod, "character")) selectionMethod <- as.list(selectionMethod[names(obsFeatures)])

+    if(!is.null(names(obsFeatures)) && !all(names(obsFeatures) %in% names(selectionMethod))) stop("selectionMethod needs to be a named character vector or list with the same names as the datasets.")

+    if(!is.null(names(obsFeatures)) && all(names(obsFeatures) %in% names(selectionMethod)) & is(selectionMethod, "character")) selectionMethod <- as.list(selectionMethod[names(obsFeatures)])

     selectionMethod

-

 }

 ######################################

 ######################################

 cleanClassifier <- function(classifier, measurements){

     #### Clean up

-    obsFeatures <- unlist(as.list(table(mcols(measurements)[, "dataset"])))

+    if(!is.null(mcols(measurements)))

+      obsFeatures <- unlist(as.list(table(mcols(measurements)[, "dataset"])))

+    else return(list(classifier))

-    if(is.null(names(classifier)) & length(classifier) == 1) classifier <- sapply(names(obsFeatures), function(x)classifier, simplify = FALSE)

-    if(is.null(names(classifier)) & length(classifier) > 1) classifier <- sapply(names(obsFeatures), function(x)classifier, simplify = FALSE)

+    if(is.null(names(classifier)) & length(classifier) == 1 & !is.null(names(obsFeatures))) classifier <- sapply(names(obsFeatures), function(x)classifier, simplify = FALSE)

+    if(is.null(names(classifier)) & length(classifier) > 1 & !is.null(names(obsFeatures))) classifier <- sapply(names(obsFeatures), function(x)classifier, simplify = FALSE)

     #if(is.null(names(classifier)) & length(classifier) > 1) stop("classifier needs to be a named character vector or list with the same names as the datasets.")

-    if(!all(names(obsFeatures) %in% names(classifier))) stop("classifier needs to be a named character vector or list with the same names as the datasets.")

-    if(all(names(obsFeatures) %in% names(classifier)) & is(classifier, "character")) classifier <- as.list(classifier[names(obsFeatures)])

+    if(!is.null(names(obsFeatures)) && !all(names(obsFeatures) %in% names(classifier))) stop("classifier needs to be a named character vector or list with the same names as the datasets.")

+    if(!is.null(names(obsFeatures)) && all(names(obsFeatures) %in% names(classifier)) & is(classifier, "character")) classifier <- as.list(classifier[names(obsFeatures)])

     classifier

-

 }

@@ -644,7 +646,7 @@ generateModellingParams <- function(datasetIDs,

     classifiers <- c("randomForest", "GLM", "elasticNetGLM", "SVM", "DLDA",

                      "naiveBayes", "mixturesNormals", "kNN",

-                     "elasticNetPreval", "CoxPH", "CoxNet")

+                     "CoxPH", "CoxNet")

     # Check classifier

     if(!classifier %in% classifiers)

         stop(paste("Classifier must exactly match of these (be careful of case):", paste(classifiers, collapse = ", ")))

@@ -659,7 +661,6 @@ generateModellingParams <- function(datasetIDs,

         "naiveBayes" = naiveBayesParams(),

         "mixturesNormals" = mixModelsParams(),

         "kNN" = kNNparams(),

-        "elasticNetPreval" = elasticNetPreval(),

         "CoxPH" = coxphParams(),

         "CoxNet" = coxnetParams()

     )

@@ -876,6 +877,7 @@ CV <- function(measurements,

     # Which data-types or data-views are present?

     datasetIDs <- unique(mcols(measurements)[, "dataset"])

+    if(is.null(datasetIDs)) datasetIDs <- 1

     if(is.null(dataCombinations)) dataCombinations <- datasetIDs

     if(is.null(characteristicsLabel)) characteristicsLabel <- "none"

@@ -896,8 +898,7 @@ CV <- function(measurements,

                                                multiViewMethod = multiViewMethod

     )

-

-    characteristics = S4Vectors::DataFrame(characteristic = c("dataset", "classifier", "selectionMethod", "multiViewMethod", "characteristicsLabel"), value = c(paste(datasetIDs, collapse = ", "), paste(classifier, collapse = ", "),  paste(selectionMethod, collapse = ", "), multiViewMethod, characteristicsLabel))

+    characteristics = S4Vectors::DataFrame(characteristic = c(if(length(datasetIDs) > 1) "Data Set" else NULL, "Classifier Name", "Selection Name", "multiViewMethod", "characteristicsLabel"), value = c(if(length(datasetIDs) > 1) paste(datasetIDs, collapse = ", ") else NULL, paste(classifier, collapse = ", "),  paste(selectionMethod, collapse = ", "), multiViewMethod, characteristicsLabel))

     classifyResults <- runTests(measurements, classes, crossValParams = crossValParams, modellingParams = modellingParams, characteristics = characteristics)

@@ -238,8 +238,8 @@ setMethod("elasticNetGLMpredictInterface", c("multnet", "MultiAssayExperiment"),

 #' @param model A fitted multinomial GLM which was created by

 #' \code{\link[glmnet]{glmnet}}.

 #' @return An \code{list} object. The first element is a vector or data frame

-#' of ranked features, the second is a vector or data frame of selected

-#' features.

+#' of ranked feature indicies, the second is a vector or data frame of selected

+#' feature indices of the \code{DataFrame} processed.

 #' @author Dario Strbenac

 #' @examples

 #' 

@@ -282,15 +282,7 @@ setMethod("elasticNetFeatures", "multnet",

             whichCoefficientColumn <- which(abs(model[["lambda"]] - attr(model, "tune")[["lambda"]]) < 0.00001)[1]

             coefficientsUsed <- sapply(model[["beta"]], function(classCoefficients) classCoefficients[, whichCoefficientColumn])

             featureScores <- rowSums(abs(coefficientsUsed))

-            rankedFeatures <- inputFeatures[order(featureScores, decreasing = TRUE)]

-            selectedFeatures <- inputFeatures[featureScores != 0]

-            

-            # Colon is a reserved symbol for separating data name and feature name, which is necessary for identifiability of MultiAssayExperiment features. It is not permitted in feature names.

-            if(grepl(':', selectedFeatures[1]) == TRUE) # Convert to data.frame.

-            {

-              selectedFeatures <- do.call(rbind, strsplit(selectedFeatures, ':'))

-              rankedFeatures <- do.call(rbind, strsplit(rankedFeatures, ':'))

-              colnames(selectedFeatures) <- colnames(rankedFeatures) <- c("dataset", "feature")

-            }

-            list(unique(rankedFeatures), selectedFeatures)

+            rankedFeaturesIndices <- order(featureScores, decreasing = TRUE)

+            selectedFeaturesIndices <- which(featureScores != 0)

+            list(rankedFeaturesIndices, selectedFeaturesIndices)

           })

\ No newline at end of file

@@ -70,7 +70,6 @@

 #'     colnames(genesMatrix) <- paste("Gene", 1:ncol(genesMatrix))

 #'     

 #'     CVparams <- CrossValParams("k-Fold")

-#'       

 #'     trainParams <- TrainParams(GLMtrainInterface)

 #'     predictParams <- PredictParams(GLMpredictInterface)

 #'     modParams <- ModellingParams(selectParams = NULL, trainParams = trainParams,

@@ -240,7 +240,7 @@ setMethod("NSCpredictInterface", c("pamrtrained", "MultiAssayExperiment"), funct

 #' @param classesTrain A vector of class labels of class \code{\link{factor}} of the

 #' same length as the number of samples in \code{measurementsTrain}.

 #' @return A list with the first element being empty (no feature ranking is

-#' provided) and second element being the selected features.

+#' provided) and second element being the selected features' indices.

 #' @author Dario Strbenac

 #' @seealso \code{\link[pamr]{pamr.listgenes}} for the function that is

 #' interfaced to.

@@ -278,12 +278,7 @@ setMethod("NSCfeatures", "pamrtrained",

             minError <- min(model[["errors"]])

             threshold <- model[["threshold"]][max(which(model[["errors"]] == minError))]

             params <- c(list(model), list(list(x = t(as.matrix(measurementsTrain)), y = measurementsTrain, geneid = 1:ncol(measurementsTrain))), threshold)

-            chosen <- as.numeric(do.call(pamr::pamr.listgenes, params)[, 1])

+            chosenIndices <- as.numeric(do.call(pamr::pamr.listgenes, params)[, 1])

-            if(is.matrix(measurementsTrain) || is.null(S4Vectors::mcols(measurementsTrain)))

-              chosen <- colnames(measurementsTrain)[chosen]

-            else

-              chosen <- S4Vectors::mcols(measurementsTrain)[chosen, ]

-            

-            list(NULL, chosen)

+            list(NULL, chosenIndices)

           })

\ No newline at end of file

@@ -193,8 +193,8 @@ setMethod("randomForestPredictInterface", c("randomForest", "MultiAssayExperimen

 #' @aliases forestFeatures forestFeatures,randomForest-method

 #' @param forest A trained random forest which was created by \code{\link{randomForest}}.

 #' @return An \code{list} object. The first element is a vector or data frame

-#' of features, ranked from best to worst using the Gini index. The second

-#' element is a vector or data frame of features used in at least one tree.

+#' of feature indices, ranked from best to worst using the Gini index. The second

+#' element is a vector or data frame of feature indices used in at least one tree.

 #' @author Dario Strbenac

 #' @examples

 #'

@@ -226,17 +226,7 @@ setGeneric("forestFeatures", function(forest, ...)

 setMethod("forestFeatures", "randomForest",

           function(forest)

           {

-            inputFeatures <- rownames(randomForest::importance(forest))

-            rankedFeatures <- inputFeatures[order(randomForest::importance(forest), decreasing = TRUE)]

-            selectedFeatures <- inputFeatures[randomForest::varUsed(forest) > 0]

-            selectedFeatures <- selectedFeatures[na.omit(match(rankedFeatures, selectedFeatures))]

-            

-            # Colon is a reserved symbol for separating data name and feature name, which is necessary for identifiability of MultiAssayExperiment features. It is not permitted in feature names.

-            if(grepl(':', selectedFeatures[1]) == TRUE) # Convert to data.frame.

-            {

-              selectedFeatures <- do.call(rbind, strsplit(selectedFeatures, ':'))

-              rankedFeatures <- do.call(rbind, strsplit(rankedFeatures, ':'))

-              colnames(selectedFeatures) <- colnames(rankedFeatures) <- c("dataset", "feature")

-            }

-            list(rankedFeatures, selectedFeatures)

+            rankedFeaturesIndices <- order(randomForest::importance(forest), decreasing = TRUE)

+            selectedFeaturesIndices <- which(randomForest::varUsed(forest) > 0)

+            list(rankedFeaturesIndices, selectedFeaturesIndices)

           })

\ No newline at end of file

@@ -146,7 +146,6 @@ setMethod("SVMpredictInterface", c("svm", "DataFrame"), function(model, measurem

   # Prediction function depends on test data having same set of columns in same order as

   # selected features used for training.

-  colnames(measurementsTest) <- make.names(colnames(measurementsTest)) # svm silently converts feature names.

   measurementsTest <- measurementsTest[, colnames(model[["SV"]])]

   classPredictions <- predict(model, measurementsTest, probability = TRUE)

@@ -54,26 +54,28 @@

 #' @author Dario Strbenac

 #' @examples

 #' 

-#'   predicted <- data.frame(sample = sample(LETTERS[1:10], 80, replace = TRUE),

-#'                           permutation = rep(1:2, each = 40),

-#'                           class = factor(rep(c("Healthy", "Cancer"), 40)))

+#'   predicted <- DataFrame(sample = sample(LETTERS[1:10], 80, replace = TRUE),

+#'                          permutation = rep(1:2, each = 40),

+#'                          class = factor(rep(c("Healthy", "Cancer"), 40)))

 #'   actual <- factor(rep(c("Healthy", "Cancer"), each = 5))

 #'   result1 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name",

 #'                                                          "Cross-validation"),

 #'                             value = c("Example", "t-test", "Differential Expression", "2 Permutations, 2 Folds")),

-#'                             LETTERS[1:10], LETTERS[10:1], list(1:100, c(1:9, 11:101)),

-#'                             list(c(1:3), c(2, 5, 6), 1:4, 5:8),

+#'                             LETTERS[1:10], DataFrame(`Original Feature` = paste("Gene", LETTERS[1:10]),

+#'                             `Renamed Feature` = paste("Feature", 1:10, sep = '')), list(paste("Gene", 1:100), paste("Gene", c(10:1, 11:100)), paste("Gene", 1:100), paste("Gene", 1:100)),

+#'                             list(paste("Gene", 1:3), paste("Gene", c(2, 5, 6)), paste("Gene", 1:4), paste("Gene", 5:8)),

 #'                             list(function(oracle){}), NULL, predicted, actual)

 #'   result1 <- calcCVperformance(result1, "Macro F1")

 #' 

-#'   predicted <- data.frame(sample = sample(LETTERS[1:10], 80, replace = TRUE),

+#'   predicted <- DataFrame(sample = sample(LETTERS[1:10], 80, replace = TRUE),

 #'                           permutation = rep(1:2, each = 40),

 #'                           class = factor(rep(c("Healthy", "Cancer"), 40)))

 #'                                

 #'   result2 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name",

 #'                                                          "Cross-validation"),

 #'                             value = c("Example", "Bartlett Test", "Differential Variability", "2 Permutations, 2 Folds")),

-#'                             LETTERS[1:10], LETTERS[10:1], list(1:100, c(1:5, 11:105)),

+#'                             LETTERS[1:10], DataFrame(`Original Feature` = paste("Gene", LETTERS[1:10]),

+#'                             `Renamed Feature` = paste("Feature", 1:10, sep = '')), list(paste("Gene", 1:100), paste("Gene", c(10:1, 11:100)), paste("Gene", 1:100), paste("Gene", 1:100)),

 #'                             list(c(1:3), c(4:6), c(1, 6, 7, 9), c(5:8)),

 #'                             list(function(oracle){}), NULL, predicted, actual)

 #'   result2 <- calcCVperformance(result2, "Macro F1")

@@ -131,7 +131,7 @@

 #'                                         colData = cbind(clinicalData, class = classes))

 #'   targetFeatures <- DataFrame(dataset = "RNA", feature = "Gene 50")                                     

 #'   plotFeatureClasses(dataContainer, targets = targetFeatures, classesColumn = "class",

-#'                      groupBy = c("sampleInfo", "Gender"),

+#'                      groupBy = c("sampleInfo", "Gender"), # Table name, feature name.

 #'                      xAxisLabel = bquote(log[2]*'(expression)'), dotBinWidth = 0.5)

 #' 

 #' @importFrom dplyr mutate n

@@ -175,7 +175,7 @@ setMethod("plotFeatureClasses", "DataFrame", function(measurements, classes, tar

                     facets = factor(paste(groupingName, "is", groupBy), levels = paste(groupingName, "is", levelsOrder)))

   }

-  splitDataset <- .splitDataAndOutcomes(measurements, classes)

+  splitDataset <- .splitDataAndOutcomes(measurements, classes, restrict = NULL)

   measurements <- splitDataset[["measurements"]]

   classes <- splitDataset[["outcomes"]]

@@ -96,19 +96,21 @@ setMethod("previousSelection", "DataFrame",

   previousIDs <- chosenFeatureNames(classifyResult)[[.iteration]]

   if(is.character(previousIDs))

   {

-    commonFeatures <- intersect(previousIDs, colnames(measurementsTrain))

-    overlapPercent <- length(commonFeatures) / length(previousIDs) * 100

-  } else { # A data.frame describing the data set and variable name of the chosen feature.

+    featuresIDs <- colnames(measurementsTrain)

+    IDsRows <- match(previousIDs, featuresInfo(classifyResult)[, "Original Feature"])

+    safeIDs <- featuresInfo(classifyResult)[IDsRows, "Renamed Feature"]

+  } else { # A data frame describing the data set and variable name of the chosen feature.

     featuresIDs <- do.call(paste, S4Vectors::mcols(measurementsTrain)[, c("dataset", "feature")])

-    selectedIDs <-  do.call(paste, previousIDs)

-    selectedColumns <- match(selectedIDs, featuresIDs)

-    commonFeatures <- sum(!is.na(selectedColumns))

-    overlapPercent <- commonFeatures / nrow(previousIDs) * 100

+    IDsRows <-  match(do.call(paste, previousIDs), do.call(paste(featuresInfo(classifyResult)[, c("Original Dataset", "Original Feature")])))

+    safeIDs <- do.call(paste, featuresInfo(classifyResult)[IDsRows, c("Renamed Dataset", "Renamed Feature")])

   }

+  

+  commonFeatures <- intersect(safeIDs, featuresIDs)

+  overlapPercent <- length(commonFeatures) / length(safeIDs) * 100

   if(overlapPercent < minimumOverlapPercent)

     signalCondition(simpleError(paste("Number of features in common between previous and current data set is lower than", minimumOverlapPercent, "percent.")))

-  

-  S4Vectors::mcols(measurementsTrain)[selectedColumns, ] # Each row is about one column.

+

+  match(safeIDs, featuresIDs) # Return indices, not identifiers.

 })

 #' @rdname previousSelection

@@ -117,7 +119,7 @@ setMethod("previousSelection", "MultiAssayExperiment",

           function(measurementsTrain, ...)

           {

             sampleInfoColumns <- colnames(MultiAssayExperiment::colData(sampleInfoColumns))

-            dataTable <- wideFormat(measurementsTrain, colDataCols = sampleInfoColumns, check.names = FALSE, collapse = ':')

+            dataTable <- MultiAssayExperiment::wideFormat(measurementsTrain, colDataCols = sampleInfoColumns, check.names = FALSE, collapse = ':')

             S4Vectors::mcols(dataTable)[, "sourceName"] <- gsub("colDataCols", "sampleInfo", S4Vectors::mcols(dataTable)[, "sourceName"])

             previousSelection(dataTable, ...)

           })

\ No newline at end of file

@@ -84,10 +84,7 @@ setMethod("bartlettRanking", "DataFrame", # Sample information data or one of th

   pValues <- apply(measurementsTrain, 2, function(featureColumn)

     stats::bartlett.test(featureColumn, classesTrain)[["p.value"]])

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(pValues), ]

-  else

-    colnames(measurementsTrain)[order(pValues)]

+  order(pValues) # From smallest to largest.

 })

 # One or more omics data sets, possibly with sample information data.

@@ -59,10 +59,7 @@ setMethod("coxphRanking", "DataFrame", function(measurementsTrain, survivalTrain

     s$waldtest["pvalue"]

   })

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(pValues), ]

-  else

-    colnames(measurementsTrain)[order(pValues)]

+  order(pValues) # From smallest to largest.

 })

 # One or more omics data sets, possibly with clinical data.

@@ -102,11 +102,7 @@ setMethod("DMDranking", "DataFrame", # sampleInfo data or one of the other input

   if(differences %in% c("both", "scale"))

     divergence <- divergence + scalesDifferences

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-      S4Vectors::mcols(measurementsTrain)[order(divergence, decreasing = TRUE), ]

-  else

-      colnames(measurementsTrain)[order(divergence, decreasing = TRUE)]

-  

+  order(divergence, decreasing = TRUE)

 })

 #' @rdname DMDranking

@@ -87,10 +87,7 @@ setMethod("differentMeansRanking", "DataFrame",

     pValues <- genefilter::rowFtests(measurementsMatrix, classesTrain)[, "p.value"]

   }

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(pValues), ]

-  else

-    colnames(measurementsTrain)[order(pValues)]

+  order(pValues) # From smallest to largest.

 })

 # One or more omics data sets, possibly with sample information data.

@@ -110,10 +110,7 @@ setMethod("edgeRranking", "DataFrame", function(countsTrain, classesTrain, normF

   fit <- do.call(edgeR::glmFit, paramList)

   test <- edgeR::glmLRT(fit, coef = 2:length(levels(classesTrain)))[["table"]]

-  if(!is.null(S4Vectors::mcols(countsTrain)))

-    S4Vectors::mcols(countsTrain)[order(test[, "PValue"]), ]

-  else

-    colnames(countsTrain)[order(test[, "PValue"])]

+  order(test[, "PValue"]) # From smallest to largest.

 })

 # One or more omics data sets, possibly with sample information data.

@@ -75,10 +75,7 @@ setMethod("KolmogorovSmirnovRanking", "DataFrame", # Sample information data or

   KSdistance <- apply(measurementsTrain, 2, function(featureColumn)

                       stats::ks.test(featureColumn[oneClass], featureColumn[otherClass], ...)[["statistic"]])

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(KSdistance, decreasing = TRUE), ]

-  else

-    colnames(measurementsTrain)[order(KSdistance, decreasing = TRUE)]

+  order(KSdistance, decreasing = TRUE)

 })

 # One or more omics data sets, possibly with sample information data.

@@ -88,11 +88,7 @@ setMethod("KullbackLeiblerRanking", "DataFrame", # Sample information data or on

                          ((oneClassDistribution[[2]])^2) / ((otherClassDistribution[[2]])^2) +

                          ((otherClassDistribution[[2]])^2) / ((oneClassDistribution[[2]])^2))

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(divergence, decreasing = TRUE), ]

-  else

-    colnames(measurementsTrain)[order(divergence, decreasing = TRUE)]

-  

+  order(divergence, decreasing = TRUE)

 })

 # One or more omics data sets, possibly with sample information data.

@@ -76,10 +76,7 @@ setMethod("leveneRanking", "DataFrame", # Sample information data or one of the

   pValues <- apply(measurementsTrain, 2, function(featureColumn)

              car::leveneTest(featureColumn, classesTrain)[["Pr(>F)"]][1])

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(pValues), ]

-  else

-    colnames(measurementsTrain)[order(pValues)]

+  order(pValues) # From smallest to largest.

 })

 # One or more omics data sets, possibly with sample information data.

@@ -99,10 +99,7 @@ setMethod("likelihoodRatioRanking", "DataFrame", # Sample information data or on

     switch(alternative[["scale"]], same = allDistribution[[2]], different = classDistribution[[2]])))    

   }))

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(logLikelihoodRatios), ]

-  else

-    colnames(measurementsTrain)[order(logLikelihoodRatios)]

+  order(logLikelihoodRatios) # From smallest to largest.

 })

 # One or more omics data sets, possibly with sample information data.

@@ -77,10 +77,7 @@ setMethod("limmaRanking", "DataFrame",

   linearModel <- limma::eBayes(linearModel)

   linearModel <- linearModel[, -1] # Get rid of intercept.

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    S4Vectors::mcols(measurementsTrain)[order(linearModel[["F.p.value"]]), ]

-  else

-    colnames(measurementsTrain)[order(linearModel[["F.p.value"]])]

+  order(linearModel[["F.p.value"]]) # From smallest to largest.

 })

 # One or more omics data sets, possibly with sample information data.

@@ -114,7 +114,7 @@ setMethod("pairsDifferencesRanking", "DataFrame",

   pairsClassDifferences <- otherClassDifferences - oneClassDifferences

-  featurePairs[order(abs(pairsClassDifferences), decreasing = TRUE)]

+  order(abs(pairsClassDifferences), decreasing = TRUE)

 })

 # One or more omics data sets, possibly with sample information data.

@@ -73,13 +73,15 @@

 #' @author Dario Strbenac

 #' @examples

 #' 

-#'   predicted <- data.frame(sample = sample(10, 100, replace = TRUE),

+#'   predicted <- DataFrame(sample = sample(10, 100, replace = TRUE),

 #'                           permutation = rep(1:2, each = 50),

 #'                           class = rep(c("Healthy", "Cancer"), each = 50))

 #'   actual <- factor(rep(c("Healthy", "Cancer"), each = 5))

 #'   allFeatures <- sapply(1:100, function(index) paste(sample(LETTERS, 3), collapse = ''))

-#'   rankList <- list(allFeatures[1:100], allFeatures[c(15:6, 1:5, 16:100)],

-#'                    allFeatures[c(1:9, 11, 10, 12:100)], allFeatures[c(1:50, 61:100, 60:51)])

+#'   allFeatures <- DataFrame(`Original Feature` = allFeatures, `Renamed Feature` = paste("Feature", 1:100, sep = ''),

+#'   check.names = FALSE)

+#'   rankList <- list(allFeatures[1:100, "Original Feature"], allFeatures[c(15:6, 1:5, 16:100), "Original Feature"],

+#'                    allFeatures[c(1:9, 11, 10, 12:100), "Original Feature"], allFeatures[c(1:50, 61:100, 60:51), "Original Feature"])

 #'   result1 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name",

 #'                                                          "Cross-validation"),

 #'                             value = c("Melanoma", "t-test", "Diagonal LDA", "2 Permutations, 2 Folds")),

@@ -90,8 +92,8 @@

 #'                             predicted, actual)

 #'   

 #'   predicted[, "class"] <- sample(predicted[, "class"])

-#'   rankList <- list(allFeatures[1:100], allFeatures[c(sample(20), 21:100)],

-#'                    allFeatures[c(1:9, 11, 10, 12:100)], allFeatures[c(1:50, 60:51, 61:100)])

+#'   rankList <- list(allFeatures[1:100, "Original Feature"], allFeatures[c(sample(20), 21:100), "Original Feature"],

+#'   allFeatures[c(1:9, 11, 10, 12:100), "Original Feature"], allFeatures[c(1:50, 60:51, 61:100), "Original Feature"])

 #'   result2 <- ClassifyResult(DataFrame(characteristic = c("Data Set", "Selection Name", "Classifier Name",

 #'                                                          "Cross-validations"),

 #'                             value = c("Melanoma", "t-test", "Random Forest", "2 Permutations, 2 Folds")),

@@ -127,7 +129,7 @@ setMethod("rankingPlot", "list",

   if(comparison == "within" && !is.null(referenceLevel))

     stop("'comparison' should not be \"within\" if 'referenceLevel' is not NULL.")

-  nFeatures <- ifelse(is.null(ncol(results[[1]]@originalFeatures)), length(results[[1]]@originalFeatures), nrow(results[[1]]@originalFeatures)) 

+  nFeatures <- nrow(featuresInfo(results[[1]]))

   error <- character()

   if(max(topRanked) > nFeatures)

     error <- paste("'topRanked' is as high as", max(topRanked))

@@ -99,7 +99,7 @@ function(measurementsTrain, outcomesTrain, measurementsTest, outcomesTest,

       stop("'measurementsTrain' DataFrame must have sample identifiers as its row names.")

     if(any(is.na(measurementsTrain)))

       stop("Some data elements are missing and classifiers don't work with missing data. Consider imputation or filtering.")                

-    

+

     splitDatasetTrain <- .splitDataAndOutcomes(measurementsTrain, outcomesTrain)

     # Rebalance the class sizes of the training samples by either downsampling or upsampling

     # or leave untouched if balancing is none.

@@ -111,6 +111,20 @@ function(measurementsTrain, outcomesTrain, measurementsTest, outcomesTest,

     }

   }

+  if(is.null(.iteration) || .iteration != "internal") # Not nested cross-validation.

+  {

+    # Avoid strange symbols in feature identifiers.

+    featuresInfo <- .summaryFeatures(measurementsTrain)

+    if(!is.null(S4Vectors::mcols(measurementsTrain)))

+    {

+      S4Vectors::mcols(measurementsTrain) <- featuresInfo[, c("Renamed Dataset", "Renamed Feature")]

+      S4Vectors::mcols(measurementsTest) <- featuresInfo[, c("Renamed Dataset", "Renamed Feature")]

+    } else {

+      colnames(measurementsTrain) <- featuresInfo[, "Renamed Feature"]

+      colnames(measurementsTest) <- featuresInfo[, "Renamed Feature"]

+    }

+  }

+    

   if(!is.null(modellingParams@selectParams) && max(modellingParams@selectParams@tuneParams[["nFeatures"]]) > ncol(measurementsTrain))

   {

     warning("Attempting to evaluate more features for feature selection than in

@@ -118,12 +132,6 @@ input data. Autmomatically reducing to smaller number.")

     modellingParams@selectParams@tuneParams[["nFeatures"]] <- 1:min(10, ncol(measurementsTrain))

   }

-  # All input features.

-  if(!is.null(S4Vectors::mcols(measurementsTrain)))

-    allFeatures <- S4Vectors::mcols(measurementsTrain)

-  else

-    allFeatures <- colnames(measurementsTrain)

-  

   if(!is.null(crossValParams) && !is.null(crossValParams@adaptiveResamplingDelta))

   { # Iteratively resample training samples until their class probability or risk changes, on average, less than delta.

     delta <- crossValParams@adaptiveResamplingDelta

@@ -159,34 +167,25 @@ input data. Autmomatically reducing to smaller number.")

     if(is.character(measurementsTransformedList[[1]])) return(measurementsTransformedList[[1]]) # An error occurred.

   }

-  rankedFeatures <- NULL

-  selectedFeatures <- NULL

+  rankedFeaturesIndices <- NULL

+  selectedFeaturesIndices <- NULL

   tuneDetailsSelect <- NULL

-  

+

   if(!is.null(modellingParams@selectParams))

   {

     if(length(modellingParams@selectParams@intermediate) != 0)

       modellingParams@selectParams <- .addIntermediates(modellingParams@selectParams)

-    

+ 

     topFeatures <- tryCatch(.doSelection(measurementsTrain, outcomesTrain, crossValParams, modellingParams, verbose),

                             error = function(error) error[["message"]]) 

     if(is.character(topFeatures)) return(topFeatures) # An error occurred.

-    rankedFeatures <- topFeatures[[1]] # Extract for result object.

-    selectedFeatures <- topFeatures[[2]] # Extract for subsetting.

+    

+    rankedFeaturesIndices <- topFeatures[[1]] # Extract for result object.

+    selectedFeaturesIndices <- topFeatures[[2]] # Extract for subsetting.

     tuneDetailsSelect <- topFeatures[[3]]

     if(modellingParams@selectParams@subsetToSelections == TRUE)

-    { # Subset the the data table to only the selected features.

-      if(is.null(S4Vectors::mcols(measurementsTrain)))

-      { # Input was ordinary matrix or DataFrame.

-        measurementsTrain <- measurementsTrain[, selectedFeatures, drop = FALSE]

-      } else { # Input was MultiAssayExperiment. # Match the selected features to the data frame columns

-        selectedIDs <-  do.call(paste, selectedFeatures)

-        featuresIDs <- do.call(paste, S4Vectors::mcols(measurementsTrain)[, c("dataset", "feature")])

-        selectedColumns <- match(selectedIDs, featuresIDs)

-        measurementsTrain <- measurementsTrain[, selectedColumns, drop = FALSE]

-      }

-    }

+      measurementsTrain <- measurementsTrain[, selectedFeaturesIndices, drop = FALSE]

   } 

   # Training stage.

@@ -207,9 +206,9 @@ input data. Autmomatically reducing to smaller number.")

     if(length(extras) > 0)