@@ -3,8 +3,8 @@ Type: Package

 Title: A framework for cross-validated classification problems, with

        applications to differential variability and differential

        distribution testing

-Version: 3.3.13

-Date: 2023-02-13

+Version: 3.3.14

+Date: 2023-02-16

 Authors@R:

     c(

     person(given = "Dario", family = "Strbenac", email = "dario.strbenac@sydney.edu.au", role = c("aut", "cre")),

@@ -72,7 +72,7 @@ Collate:

     'interfaceXGB.R'

     'performancePlot.R'

     'plotFeatureClasses.R'

-    'precisionPathway.R'

+    'precisionPathways.R'

     'prepareData.R'

     'previousSelection.R'

     'previousTrained.R'

@@ -76,7 +76,8 @@ exportMethods(models)

 exportMethods(performance)

 exportMethods(performancePlot)

 exportMethods(plotFeatureClasses)

-exportMethods(precisionPathwayTrain)

+exportMethods(precisionPathwaysPredict)

+exportMethods(precisionPathwaysTrain)

 exportMethods(predictions)

 exportMethods(prepareData)

 exportMethods(rankingPlot)

@@ -23,7 +23,7 @@

 #' parameters which will be passed into the data cleaning function. The names of the list must be one of \code{"prepare"},

 #' \code{"select"}, \code{"train"}, \code{"predict"}. To remove one of the defaults (see the article titled Parameter Tuning Presets for crossValidate and Their Customisation on

 #' the website), specify the list element to be \code{NULL}. For the valid element names in the \code{"prepare"} list, see \code{?prepareData}.

-#' @param clinicalPredictors If \code{measurements} is a \code{MultiAssayExperiment},

+#' @param clinicalPredictors Default: \code{NULL}. If \code{measurements} is a \code{MultiAssayExperiment},

 #' a character vector of features to use in modelling. This allows avoidance of things like sample IDs,

 #' sample acquisition dates, etc. which are not relevant for outcome prediction.

 #' @param nFeatures The number of features to be used for classification. If this is a single number, the same number of features will be used for all comparisons

@@ -92,6 +92,7 @@

 #' # performancePlot(c(result, resultMerge))

 #' 

 #' @importFrom survival Surv

+#' @usage NULL

 setGeneric("crossValidate", function(measurements, outcome, ...)

     standardGeneric("crossValidate"))

@@ -116,7 +117,7 @@ setMethod("crossValidate", "DataFrame",

               # Check that data is in the right format, if not already done for MultiAssayExperiment input.

               if(!"assay" %in% colnames(S4Vectors::mcols(measurements))) # Assay is put there by prepareData for MultiAssayExperiment, skip if present. 

               {

-                prepParams <- list(measurements, outcome, clinicalPredictors)

+                prepParams <- list(measurements, outcome)

                 if("prepare" %in% names(extraParams))

                   prepParams <- c(prepParams, extraParams[["prepare"]])

                 measurementsAndOutcome <- do.call(prepareData, prepParams)

@@ -9,12 +9,12 @@ elasticNetGLMtrainInterface <- function(measurementsTrain, classesTrain, lambda

   # One-hot encoding needed.    

   measurementsTrain <- MatrixModels::model.Matrix(~ 0 + ., data = measurementsTrain)

-  fitted <- glmnet::glmnet(measurementsTrain, classesTrain, family = "multinomial", ...)

+  fitted <- glmnet::glmnet(measurementsTrain, classesTrain, family = "multinomial", weights = as.numeric(1 / (table(classesTrain)[classesTrain] / length(classesTrain))), ...)

+  # Inverse class size weighting needed to give decent predictions when class imbalance.

-

   if(is.null(lambda)) # fitted has numerous models for automatically chosen lambda values.

   { # Pick one lambda based on resubstitution performance. But not the one that makes all variables excluded from model.

-      lambdaConsider <- colSums(as.matrix(fitted[["beta"]][[1]])) != 0

+    lambdaConsider <- colSums(as.matrix(fitted[["beta"]][[1]])) != 0

     bestLambda <- fitted[["lambda"]][lambdaConsider][which.min(sapply(fitted[["lambda"]][lambdaConsider], function(lambda) # Largest Lambda with minimum balanced error rate.

     {

       classPredictions <- factor(as.character(predict(fitted, measurementsTrain, s = lambda, type = "class")), levels = fitted[["classnames"]])

@@ -9,7 +9,7 @@ GLMtrainInterface <- function(measurementsTrain, classesTrain, ..., verbose = 3)

         fitData <- cbind(measurementsTrain, class = classesTrain)

         classesTrain <- "class" # Column name for glm fit.

     } else {fitData <- measurementsTrain}

-  glm(class ~ . + 0, family = binomial, data = fitData, ...)

+  glm(class ~ . + 0, family = binomial, data = fitData, weights = as.numeric(1 / (table(classesTrain)[classesTrain] / length(classesTrain))), ...)

 }

 attr(GLMtrainInterface, "name") <- "GLMtrainInterface"

deleted file mode 100644

@@ -1,50 +0,0 @@

-# Basically, an ordered list of cross-validations.

-

-setGeneric("precisionPathwayTrain", function(measurements, class, ...)

-    standardGeneric("precisionPathwayTrain"))

-

-#' @rdname precisionPathwayTrain

-#' @export

-setMethod("precisionPathwayTrain", "MultiAssayExperimentOrList", 

-          function(measurements, class, clinicalPredictors = NULL, maxMissingProp = 0.0, topNvariance = NULL,

-                   fixedAssays = "clinical", confidenceCutoff = 0.8, minAssaySamples = 10,

-                   nFeatures = 20, selectionMethod = setNames(c("none", rep("t-test", length(measurements))), c("clinical", names(measurements))),

-                   classifier = setNames(c("elasticNetGLM", rep("randomForest", length(measurements))), c("clinical", names(measurements))),

-                   nFolds = 5, nRepeats = 20, nCores = 1)

-          {

-            if(is.list(measurements)) # Ensure plain list has clinical data.

-            {

-              # One of the tables must be named "clinical".

-              if (!any(names(measurements) == "clinical"))

-                stop("One of the tables must be named \"clinical\".")

-            }

-            prepArgs <- list(measurements, outcomeColumns = class, clinicalPredictors = clinicalPredictors,

-                             maxMissingProp = maxMissingProp, topNvariance = topNvariance)

-            measurementsAndClass <- do.call(prepareData, prepArgs)

-              

-            .precisionPathwayTrain(measurementsAndClass[["measurements"]], measurementsAndClass[["outcome"]],

-                                   fixedAssays = fixedAssays, confidenceCutoff = confidenceCutoff,

-                                   minAssaySamples = minAssaySamples, nFeatures = nFeatures,

-                                   selectionMethod = selectionMethod, classifier = classifier,

-                                   nFolds = nFolds, nRepeats = nRepeats, nCores = nCores)

-          })

-

-# Internal method which carries out all of the processing, obtaining reformatted data from the

-# MultiAssayExperiment and list (of basic rectangular tables) S4 methods.

-.precisionPathwayTrain <- function(measurements, class, fixedAssays = "clinical",

-                   confidenceCutoff = 0.8, minAssaySamples = 10,

-                   nFeatures = 20, selectionMethod = setNames(c(NULL, rep("t-test", length(measurements))), c("clinical", names(measurements))),

-                   classifier = setNames(c("elasticNetGLM", rep("randomForest", length(measurements))), c("clinical", names(measurements))),

-                   nFolds = 5, nRepeats = 20, nCores = 1, ...)

-          {

-            

-            # Step 1: Determine all valid permutations of assays, taking into account the

-            # assays to be used and which assays, if any, must be included.

-            assayIDs <- unique(S4Vectors::mcols(measurements)[["assay"]])

-            assaysPermutations <- .permutations(assayIDs, fixed = data.frame(seq_along(fixedAssays), fixedAssays))

-            

-            # Step 2: Build a classifier for each assay using all of the samples.

-            modelsList <- crossValidate(measurements, class, nFeatures, selectionMethod,

-                                        classifier = classifier, nFolds = nFolds,

-                                        nRepeats = nRepeats, nCores = nCores)

-        }

\ No newline at end of file

new file mode 100644

@@ -0,0 +1,180 @@

+#' Precision Pathways for Sample Prediction Based on Prediction Confidence.

+#' 

+#' Precision pathways allows the evaluation of various permutations of multiomics or multiview data.

+#' Samples are predicted by a particular assay if they were consistently predicted as a particular class

+#' during cross-validation. Otherwise, they are passed onto subsequent assays/tiers for prediction. Balanced accuracy

+#' is used to evaluate overall prediction performance and sample-specific accuracy for individual-level evaluation.

+#'

+#' @param measurements Either a \code{\link{MultiAssayExperiment}} or a list of the basic tabular objects containing the data.

+#' @param class Same as \code{measurements} but only training samples. IF \code{measurements} is a \code{list}, may also be

+#' a vector of classes.

+#' @param clinicalPredictors Default: \code{NULL}. Must be a character vector of clinical features to use in modelling. This allows avoidance of things like sample IDs,

+#' sample acquisition dates, etc. which are not relevant for outcome prediction.

+#' @param maxMissingProp Default: 0.0. A proportion less than 1 which is the maximum

+#' tolerated proportion of missingness for a feature to be retained for modelling.

+#' @param topNvariance Default: NULL. An integer number of most variable features per assay to subset to.

+#' Assays with less features won't be reduced in size.

+#' @param fixedAssays A character vector of assay names specifying any assays which must be at the

+#' beginning of the pathway.

+#' @param confidenceCutoff The minimum confidence of predictions for a sample to be predicted by a particular issue

+#' . If a sample was predicted to belong to a particular class a proportion \eqn{p} times, then the confidence is \eqn{2 \times |p - 0.5|}.

+#' @param minAssaySamples An integer specifying the minimum number of samples a tier may have. If a subsequent tier

+#' would have less than this number of samples, the samples are incorporated into the current tier.

+#' @param nFeatures Default: 20. The number of features to consider during feature selection, if feature selection is done.

+#' @param selectionMethod A named character vector of feature selection methods to use for the assays, one for each. The names must correspond to names of \code{measurements}.

+#' @param classifier A named character vector of modelling methods to use for the assays, one for each. The names must correspond to names of \code{measurements}.

+#' @param nFolds A numeric specifying the number of folds to use for cross-validation.

+#' @param nRepeats A numeric specifying the the number of repeats or permutations to use for cross-validation.

+#' @param nCores A numeric specifying the number of cores used if the user wants to use parallelisation.

+#' @param pathways A set of pathways created by \code{precisionPathwaysTrain} to be used for predicting on a new data set.

+#' @rdname precisionPathways

+#' @return An object of class \code{PrecisionPathways} which is basically a named list that other plotting and

+#' tabulating functions can use.

+#' @examples

+#' # To be determined.

+

+#' @usage NULL

+setGeneric("precisionPathwaysTrain", function(measurements, class, ...)

+    standardGeneric("precisionPathwaysTrain"))

+

+#' @rdname precisionPathways

+#' @export

+setMethod("precisionPathwaysTrain", "MultiAssayExperimentOrList", 

+          function(measurements, class, clinicalPredictors = NULL, maxMissingProp = 0.0, topNvariance = NULL,

+                   fixedAssays = "clinical", confidenceCutoff = 0.8, minAssaySamples = 10,

+                   nFeatures = 20, selectionMethod = setNames(c("none", rep("t-test", length(measurements))), c("clinical", names(measurements))),

+                   classifier = setNames(c("elasticNetGLM", rep("randomForest", length(measurements))), c("clinical", names(measurements))),

+                   nFolds = 5, nRepeats = 20, nCores = 1)

+          {

+            if(is.list(measurements)) # Ensure plain list has clinical data.

+            {

+              # One of the tables must be named "clinical".

+              if (!any(names(measurements) == "clinical"))

+                stop("One of the tables must be named \"clinical\".")

+            }

+            prepArgs <- list(measurements, outcomeColumns = class, clinicalPredictors = clinicalPredictors,

+                             maxMissingProp = maxMissingProp, topNvariance = topNvariance)

+            measurementsAndClass <- do.call(prepareData, prepArgs)

+              

+            .precisionPathwaysTrain(measurementsAndClass[["measurements"]], measurementsAndClass[["outcome"]],

+                                   fixedAssays = fixedAssays, confidenceCutoff = confidenceCutoff,

+                                   minAssaySamples = minAssaySamples, nFeatures = nFeatures,

+                                   selectionMethod = selectionMethod, classifier = classifier,

+                                   nFolds = nFolds, nRepeats = nRepeats, nCores = nCores)

+          })

+

+# Internal method which carries out all of the processing, obtaining reformatted data from the

+# MultiAssayExperiment and list (of basic rectangular tables) S4 methods.

+.precisionPathwaysTrain <- function(measurements, class, fixedAssays = "clinical",

+                   confidenceCutoff = 0.8, minAssaySamples = 10,

+                   nFeatures = 20, selectionMethod = setNames(c(NULL, rep("t-test", length(measurements))), c("clinical", names(measurements))),

+                   classifier = setNames(c("elasticNetGLM", rep("randomForest", length(measurements))), c("clinical", names(measurements))),

+                   nFolds = 5, nRepeats = 20, nCores = 1)

+          {

+            # Step 1: Determine all valid permutations of assays, taking into account the

+            # assays to be used and which assays, if any, must be included.

+            assayIDs <- unique(S4Vectors::mcols(measurements)[["assay"]])

+            assaysPermutations <- .permutations(assayIDs, fixed = data.frame(seq_along(fixedAssays), fixedAssays))

+            permutationIDs <- apply(assaysPermutations, 2, function(permutation) paste(permutation, collapse = '-'))

+            

+            # Step 2: Build a classifier for each assay using all of the samples.

+            modelsList <- crossValidate(measurements, class, nFeatures, selectionMethod,

+                                        classifier = classifier, nFolds = nFolds,

+                                        nRepeats = nRepeats, nCores = nCores)

+            modelsList <- lapply(modelsList, calcCVperformance, "Sample Accuracy") # Add sample accuracy, which can be subset later.

+

+            # Step 3: Loop over each pathway and each assay in order to determine which samples are used at that level

+            # and which are passed onwards.

+            precisionPathways <- lapply(as.data.frame(assaysPermutations), function(permutation)

+            {

+              assaysProcessed <- character()

+              samplesUsed <- character()

+              individualsTableAll <- S4Vectors::DataFrame()

+              tierTableAll <- S4Vectors::DataFrame()

+              breakEarly = FALSE

+              for(assay in permutation)

+              {

+                # Step 3a: Identify all samples which are consistently predicted.

+                modelIndex <- match(assay, assayIDs)

+                allPredictions <- predictions(modelsList[[modelIndex]])

+                allSampleIDs <- sampleNames(modelsList[[modelIndex]])

+                predictionsSamplesCounts <- table(allPredictions[, "sample"], allPredictions[, "class"])

+                confidences <- 2 * abs(predictionsSamplesCounts[, 1] / rowSums(predictionsSamplesCounts) - 0.5)

+                sampleIDsUse <- names(confidences)[confidences > confidenceCutoff]

+                

+                # Check if too few samples left for next round. Include them in this round, if so.

+                remainingIDs <- setdiff(allSampleIDs, c(samplesUsed, sampleIDsUse))

+                if(length(remainingIDs) < minAssaySamples)

+                {

+                  sampleIDsUse <- c(sampleIDsUse, remainingIDs)

+                  breakEarly = TRUE

+                }

+                

+                predictionsSamplesCounts <- predictionsSamplesCounts[sampleIDsUse, ]

+                

+                # Step 3b: Individuals predictions and sample-wise accuracy, tier-wise error.

+                maxVotes <- apply(predictionsSamplesCounts, 1, function(sample) which.max(sample))

+                predictedClasses <- factor(colnames(predictionsSamplesCounts)[maxVotes],

+                                           levels = colnames(predictionsSamplesCounts))    

+                individualsTable <- S4Vectors::DataFrame(Tier = assay,

+                                                         `Sample ID` = sampleIDsUse,

+                                                         `Predicted` = predictedClasses,

+                                                         `Accuracy` = performance(modelsList[[modelIndex]])[["Sample Accuracy"]][sampleIDsUse],

+                                                         check.names = FALSE)

+                knownClasses <- actualOutcome(modelsList[[modelIndex]])[match(sampleIDsUse, allSampleIDs)]

+                balancedAccuracy <- calcExternalPerformance(knownClasses, predictedClasses)

+                tierTable <- S4Vectors::DataFrame(Tier = assay,

+                                                  `Balanced Accuracy` = balancedAccuracy, check.names = FALSE)

+                

+                assaysProcessed <- c(assaysProcessed, assay)

+                individualsTableAll <- rbind(individualsTableAll, individualsTable)

+                tierTableAll <- rbind(tierTableAll, tierTable)

+                samplesUsed <- c(samplesUsed, sampleIDsUse)

+                

+                if(breakEarly == TRUE) break

+              }

+              pathwayString <- paste(assaysProcessed, collapse = '-')

+              parameters = list(confidenceCutoff = confidenceCutoff, minAssaySamples = minAssaySamples)

+              list(models = modelsList, parameters = parameters, pathway = pathwayString,

+                  individuals = individualsTableAll, tiers = tierTableAll)

+            })

+            

+            class(precisionPathways) <- "PrecisionPathways"

+            names(precisionPathways) <- sapply(precisionPathways, "[[", "pathway")

+            precisionPathways

+}

+

+# A nice print method to avoid flooding the screen with lots of tables

+# when result is shown in console.

+print.PrecisionPathways <- function(x)

+{

+  cat("An object of class 'PrecisionPathways'.\n")

+  cat("Pathways:\n")

+  cat(paste(names(x), collapse = '\n'))

+}

+

+#' @usage NULL

+setGeneric("precisionPathwaysPredict", function(pathways, measurements, class, ...)

+    standardGeneric("precisionPathwaysPredict"))

+

+#' @rdname precisionPathways

+#' @export

+setMethod("precisionPathwaysPredict", "MultiAssayExperimentOrList", 

+          function(pathways, measurements, class)

+          {

+            if(is.list(measurements)) # Ensure plain list has clinical data.

+            {

+              # One of the tables must be named "clinical".

+              if (!any(names(measurements) == "clinical"))

+                stop("One of the tables must be named \"clinical\".")

+            }

+            prepArgs <- list(measurements, outcomeColumns = class)

+            measurementsAndClass <- do.call(prepareData, prepArgs)

+              

+            .precisionPathwaysPredict(pathways, measurementsAndClass[["measurements"]], measurementsAndClass[["outcome"]])

+          })

+

+.precisionPathwaysPredict <- function(pathways, measurements, class)

+{

+  # To do.

+}

\ No newline at end of file

@@ -15,8 +15,6 @@

 \alias{predict.trainedByClassifyR}

 \title{Cross-validation to evaluate classification performance.}

 \usage{

-crossValidate(measurements, outcome, ...)

-

 \S4method{crossValidate}{DataFrame}(

   measurements,

   outcome,

@@ -158,7 +156,7 @@ parameters which will be passed into the data cleaning function. The names of th

 \code{"select"}, \code{"train"}, \code{"predict"}. To remove one of the defaults (see the article titled Parameter Tuning Presets for crossValidate and Their Customisation on

 the website), specify the list element to be \code{NULL}. For the valid element names in the \code{"prepare"} list, see \code{?prepareData}.}

-\item{clinicalPredictors}{If \code{measurements} is a \code{MultiAssayExperiment},

+\item{clinicalPredictors}{Default: \code{NULL}. If \code{measurements} is a \code{MultiAssayExperiment},

 a character vector of features to use in modelling. This allows avoidance of things like sample IDs,

 sample acquisition dates, etc. which are not relevant for outcome prediction.}

new file mode 100644

@@ -0,0 +1,80 @@

+% Generated by roxygen2: do not edit by hand

+% Please edit documentation in R/precisionPathways.R

+\name{precisionPathwaysTrain}

+\alias{precisionPathwaysTrain}

+\alias{precisionPathwaysTrain,MultiAssayExperimentOrList-method}

+\alias{precisionPathwaysPredict,MultiAssayExperimentOrList-method}

+\title{Precision Pathways for Sample Prediction Based on Prediction Confidence.}

+\usage{

+\S4method{precisionPathwaysTrain}{MultiAssayExperimentOrList}(

+  measurements,

+  class,

+  clinicalPredictors = NULL,

+  maxMissingProp = 0,

+  topNvariance = NULL,

+  fixedAssays = "clinical",

+  confidenceCutoff = 0.8,

+  minAssaySamples = 10,

+  nFeatures = 20,

+  selectionMethod = setNames(c("none", rep("t-test", length(measurements))),

+    c("clinical", names(measurements))),

+  classifier = setNames(c("elasticNetGLM", rep("randomForest", length(measurements))),

+    c("clinical", names(measurements))),

+  nFolds = 5,

+  nRepeats = 20,

+  nCores = 1

+)

+

+\S4method{precisionPathwaysPredict}{MultiAssayExperimentOrList}(pathways, measurements, class)

+}

+\arguments{

+\item{measurements}{Either a \code{\link{MultiAssayExperiment}} or a list of the basic tabular objects containing the data.}

+

+\item{class}{Same as \code{measurements} but only training samples. IF \code{measurements} is a \code{list}, may also be

+a vector of classes.}

+

+\item{clinicalPredictors}{Default: \code{NULL}. Must be a character vector of clinical features to use in modelling. This allows avoidance of things like sample IDs,

+sample acquisition dates, etc. which are not relevant for outcome prediction.}

+

+\item{maxMissingProp}{Default: 0.0. A proportion less than 1 which is the maximum

+tolerated proportion of missingness for a feature to be retained for modelling.}

+

+\item{topNvariance}{Default: NULL. An integer number of most variable features per assay to subset to.

+Assays with less features won't be reduced in size.}

+

+\item{fixedAssays}{A character vector of assay names specifying any assays which must be at the

+beginning of the pathway.}

+

+\item{confidenceCutoff}{The minimum confidence of predictions for a sample to be predicted by a particular issue

+. If a sample was predicted to belong to a particular class a proportion \eqn{p} times, then the confidence is \eqn{2 \times |p - 0.5|}.}

+

+\item{minAssaySamples}{An integer specifying the minimum number of samples a tier may have. If a subsequent tier

+would have less than this number of samples, the samples are incorporated into the current tier.}

+

+\item{nFeatures}{Default: 20. The number of features to consider during feature selection, if feature selection is done.}

+

+\item{selectionMethod}{A named character vector of feature selection methods to use for the assays, one for each. The names must correspond to names of \code{measurements}.}

+

+\item{classifier}{A named character vector of modelling methods to use for the assays, one for each. The names must correspond to names of \code{measurements}.}

+

+\item{nFolds}{A numeric specifying the number of folds to use for cross-validation.}

+

+\item{nRepeats}{A numeric specifying the the number of repeats or permutations to use for cross-validation.}

+

+\item{nCores}{A numeric specifying the number of cores used if the user wants to use parallelisation.}

+

+\item{pathways}{A set of pathways created by \code{precisionPathwaysTrain} to be used for predicting on a new data set.}

+}

+\value{

+An object of class \code{PrecisionPathways} which is basically a named list that other plotting and

+tabulating functions can use.

+}

+\description{

+Precision pathways allows the evaluation of various permutations of multiomics or multiview data.

+Samples are predicted by a particular assay if they were consistently predicted as a particular class

+during cross-validation. Otherwise, they are passed onto subsequent assays/tiers for prediction. Balanced accuracy

+is used to evaluate overall prediction performance and sample-specific accuracy for individual-level evaluation.

+}

+\examples{

+# To be determined.

+}