@@ -3,8 +3,8 @@ Type: Package

 Title: A framework for cross-validated classification problems, with

        applications to differential variability and differential

        distribution testing

-Version: 3.2.3

-Date: 2022-11-25

+Version: 3.2.4

+Date: 2022-11-27

 Author: Dario Strbenac, Ellis Patrick, Sourish Iyengar, Harry Robertson, Andy Tran, John Ormerod, Graham Mann, Jean Yang

 Maintainer: Dario Strbenac <dario.strbenac@sydney.edu.au>

 VignetteBuilder: knitr

@@ -27,7 +27,7 @@ Description: The software formalises a framework for classification in R.

              may be developed by the user, by creating an interface to the framework.

 License: GPL-3

 Packaged: 2014-10-18 11:16:55 UTC; dario

-RoxygenNote: 7.2.1

+RoxygenNote: 7.2.2

 NeedsCompilation: yes

 Collate:

     'ROCplot.R'

@@ -34,7 +34,7 @@ randomForestPredictInterface <- function(forest, measurementsTest, ..., returnTy

            score = classScores,

            both = data.frame(class = classPredictions, classScores, check.names = FALSE))

   } else { # It is "Survival".

-      rowSums(predictions$survival)

+      -1 * rowSums(predictions$survival) # Make it a risk score.

   }

 }

@@ -15,7 +15,7 @@

 #' \code{characteristicsList['x']} to aggregate to a single number by taking

 #' the mean. This is particularly meaningful when the cross-validation is

 #' leave-k-out, when k is small.

-#' @param performanceName Default: \code{"auto"}. The name of the

+#' @param metric Default: \code{"auto"}. The name of the

 #' performance measure or "auto". If the results are classification then

 #' balanced accuracy will be displayed. Otherwise, the results would be survival risk

 #' predictions and then C-index will be displayed. This is one of the names printed

@@ -81,7 +81,7 @@

 #'                             list(function(oracle){}), NULL, predicted, actual)

 #'   result2 <- calcCVperformance(result2, "Macro F1")

 #'   

-#'   performancePlot(list(result1, result2), performanceName = "Macro F1",

+#'   performancePlot(list(result1, result2), metric = "Macro F1",

 #'                   title = "Comparison")

 #' 

 #' @importFrom rlang sym

@@ -99,7 +99,7 @@ setMethod("performancePlot", "ClassifyResult", function(results, ...) {

 #' @rdname performancePlot

 #' @export

 setMethod("performancePlot", "list",

-          function(results, performanceName = "auto",

+          function(results, metric = "auto",

                    characteristicsList = list(x = "auto"), aggregate = character(), coloursList = list(), orderingList = list(),

                    densityStyle = c("box", "violin"), yLimits = NULL, fontSizes = c(24, 16, 12, 12), title = NULL,

                    margin = grid::unit(c(1, 1, 1, 1), "lines"), rotate90 = FALSE, showLegend = TRUE)

@@ -118,34 +118,34 @@ setMethod("performancePlot", "list",

     else

       stop("No characteristic is present for all results but must be.")

   }

-  if(performanceName == "auto")

-      performanceName <- ifelse("risk" %in% colnames(results[[1]]@predictions), "C-index", "Balanced Accuracy")

+  if(metric == "auto")

+      metric <- ifelse("risk" %in% colnames(results[[1]]@predictions), "C-index", "Balanced Accuracy")

   ggplot2::theme_set(ggplot2::theme_classic() + ggplot2::theme(panel.border = ggplot2::element_rect(fill = NA)))

-  performanceNames <- unlist(lapply(results, function(result)

+  metrics <- unlist(lapply(results, function(result)

     if(!is.null(result@performance)) names(result@performance)))

-  namesCounts <- table(performanceNames)

+  namesCounts <- table(metrics)

   commonNames <- names(namesCounts)[namesCounts == length(results)]

-  if(!performanceName %in% commonNames)

+  if(!metric %in% commonNames)

   {

-    warning(paste(performanceName, "not found in all elements of results. Calculating it now."))

-    results <- lapply(results, function(result) calcCVperformance(result, performanceName))

+    warning(paste(metric, "not found in all elements of results. Calculating it now."))

+    results <- lapply(results, function(result) calcCVperformance(result, metric))

   }

-  ifelse(performanceName == "Matthews Correlation Coefficient", baseline <- 0, baseline <- 0.5)

+  ifelse(metric == "Matthews Correlation Coefficient", baseline <- 0, baseline <- 0.5)

   plotData <- do.call(rbind, mapply(function(result, index)

                     {

-                      if(!performanceName %in% names(result@performance))

-                        stop(performanceName, " not calculated for element ", index, " of results list.")

+                      if(!metric %in% names(result@performance))

+                        stop(metric, " not calculated for element ", index, " of results list.")

                       row <- result@characteristics[, "characteristic"] == characteristicsList[["x"]] 

                       if(any(row) && result@characteristics[row, "value"] %in% aggregate)

-                        performance <- mean(result@performance[[performanceName]])

+                        performance <- mean(result@performance[[metric]])

                       else

-                        performance <- result@performance[[performanceName]]

+                        performance <- result@performance[[metric]]

                       rows <- match(unlist(characteristicsList), result@characteristics[, "characteristic"])

                       summaryTable <- data.frame(as.list(result@characteristics[rows, "value"]), performance)

-                      colnames(summaryTable) <- c(characteristicsList, performanceName)

+                      colnames(summaryTable) <- c(characteristicsList, metric)

                       summaryTable

                     }, results, 1:length(results), SIMPLIFY = FALSE))

@@ -182,12 +182,12 @@ setMethod("performancePlot", "list",

   if(any(analysisGroupSizes > 1))

   {

     multiPlotData <- do.call(rbind, analysisGrouped[analysisGroupSizes > 1])

-    performancePlot <- performancePlot + densityStyle(data = multiPlotData, ggplot2::aes(x = !!characteristicsList[['x']], y = !!(rlang::sym(performanceName)), fill = !!fillVariable, colour = !!lineVariable))

+    performancePlot <- performancePlot + densityStyle(data = multiPlotData, ggplot2::aes(x = !!characteristicsList[['x']], y = !!(rlang::sym(metric)), fill = !!fillVariable, colour = !!lineVariable))

   }

   if(any(analysisGroupSizes == 1))

   {

     singlePlotData <- do.call(rbind, analysisGrouped[analysisGroupSizes == 1])

-    performancePlot <- performancePlot + ggplot2::geom_bar(data = singlePlotData, stat = "identity", ggplot2::aes(x = !!characteristicsList[['x']], y = !!(rlang::sym(performanceName)), fill = !!fillVariable, colour = !!lineVariable))

+    performancePlot <- performancePlot + ggplot2::geom_bar(data = singlePlotData, stat = "identity", ggplot2::aes(x = !!characteristicsList[['x']], y = !!(rlang::sym(metric)), fill = !!fillVariable, colour = !!lineVariable))

   }

   if(!is.null(yLimits)) yLimits = c(0, 1)

@@ -15,7 +15,12 @@

 #' same length as the number of columns that \code{results} has.

 #' @param comparison Default: "auto". The aspect of the experimental

 #' design to compare. Can be any characteristic that all results share.

-#' @param metric Default: "Sample Error". The sample-wise metric to plot.

+#' @param metric Default: \code{"auto"}. The name of the

+#' performance measure or "auto". If the results are classification then

+#' sample accuracy will be displayed. Otherwise, the results would be survival risk

+#' predictions and then a sample C-index will be displayed. Valid values are \code{"Sample Error"},

+#' \code{"Sample Error"} or \code{"Sample C-index"}. If the metric is not stored in the

+#' results list, the performance metric will be calculated automatically.

 #' @param featureValues If not NULL, can be a named factor or named numeric

 #' vector specifying some variable of interest to plot above the heatmap.

 #' @param featureName A label describing the information in

@@ -66,8 +71,8 @@

 #'                             value = c("Example", "Bartlett Test", "Differential Variability", "2 Permutations, 2 Folds")),

 #'                             LETTERS[1:10], features, list(1:100), list(sample(10, 10)),

 #'                             list(function(oracle){}), NULL, predicted, actual)

-#'   result1 <- calcCVperformance(result1, "Sample Error")

-#'   result2 <- calcCVperformance(result2, "Sample Error")

+#'   result1 <- calcCVperformance(result1)

+#'   result2 <- calcCVperformance(result2)

 #'   groups <- factor(rep(c("Male", "Female"), length.out = 10))

 #'   names(groups) <- LETTERS[1:10]

 #'   cholesterol <- c(4.0, 5.5, 3.9, 4.9, 5.7, 7.1, 7.9, 8.0, 8.5, 7.2)

@@ -95,10 +100,10 @@ setMethod("samplesMetricMap", "ClassifyResult", function(results, ...) {

 setMethod("samplesMetricMap", "list", 

           function(results,

                    comparison = "auto",

-                   metric = c("Sample Error", "Sample Accuracy", "Sample C-index"),

+                   metric = "auto",

                    featureValues = NULL, featureName = NULL,

-                   metricColours = list(c("#3F48CC", "#6F75D8", "#9FA3E5", "#CFD1F2", "#FFFFFF"),

-                                        c("#880015", "#A53F4F", "#C37F8A", "#E1BFC4", "#FFFFFF")),

+                   metricColours = list(c("#FFFFFF", "#CFD1F2", "#9FA3E5", "#6F75D8", "#3F48CC"),

+                                        c("#FFFFFF", "#E1BFC4", "#C37F8A", "#A53F4F", "#880015")),

                    classColours = c("#3F48CC", "#880015"), groupColours = c("darkgreen", "yellow2"),

                    fontSizes = c(24, 16, 12, 12, 12),

                    mapHeight = 4, title = switch(metric, `Sample Error` = "Error Comparison", `Sample Accuracy` = "Accuracy Comparison", `Sample C-index` = "Risk Score Comparison"),

@@ -125,6 +130,17 @@ setMethod("samplesMetricMap", "list",

       stop("No characteristic is present for all results but must be.")

     }

   }

+  isSurvival <- "risk" %in% colnames(results[[1]]@predictions)

+  validMetrics <- c("Sample Error", "Sample Accuracy", "Sample C-index")

+  if(metric == "auto")

+    metric <- ifelse(isSurvival, "Sample C-index", "Sample Accuracy")

+  else

+    if(!metric %in% validMetrics) stop("metric must be one of ", validMetrics, " but is ", metric, '.')   

+  if(isSurvival && is.list(metricColours)) metricColours <- metricColours[[1]]

+  metricText <- gsub("Sample ", '', metric) # For legend labelling.

+  if(showXtickLabels == FALSE && xAxisLabel == "Sample Name") xAxisLabel <- "Sample"

+     

+  

   resultsWithComparison <- sum(sapply(results, function(result) any(result@characteristics[, "characteristic"] == comparison)))

   if(resultsWithComparison < length(results))

     stop("Not all results have comparison characteristic ", comparison, ' but need to.')

@@ -135,13 +151,17 @@ setMethod("samplesMetricMap", "list",

     compareFactor <- sapply(results, function(result) {

                      useRow <- result@characteristics[, "characteristic"] == comparison

                      result@characteristics[useRow, "value"]

-                    })  

-  metric <- match.arg(metric)

-  metricText <- switch(metric, `Sample Error` = "Error", `Sample Accuracy` = "Accuracy", `Sample C-index` = "C-index")

+                    })

-  allCalculated <- all(sapply(results, function(result) metric %in% names(performance(result))))

-  if(!allCalculated)

-    stop("One or more classification results lack the calculated sample-specific metric.")

+  metrics <- unlist(lapply(results, function(result)

+    if(!is.null(result@performance)) names(result@performance)))

+  namesCounts <- table(metrics)

+  commonNames <- names(namesCounts)[namesCounts == length(results)]

+  if(!metric %in% commonNames)

+  {

+    warning(paste(metric, "not found in all elements of results. Calculating it now."))

+    results <- lapply(results, function(result) calcCVperformance(result, metric))

+  }

   if(!is.null(featureValues) && is.null(featureName))

     stop("featureValues is specified by featureNames isn't. Specify both.")

   if(!is.null(featureValues) && is.null(names(featureValues)))

@@ -10,7 +10,7 @@

 \S4method{performancePlot}{list}(

   results,

-  performanceName = "auto",

+  metric = "auto",

   characteristicsList = list(x = "auto"),

   aggregate = character(),

   coloursList = list(),

@@ -29,7 +29,7 @@

 \item{...}{Not used by end user.}

-\item{performanceName}{Default: \code{"auto"}. The name of the

+\item{metric}{Default: \code{"auto"}. The name of the

 performance measure or "auto". If the results are classification then

 balanced accuracy will be displayed. Otherwise, the results would be survival risk

 predictions and then C-index will be displayed. This is one of the names printed

@@ -122,7 +122,7 @@ calculated, and a barchart is plotted.

                             list(function(oracle){}), NULL, predicted, actual)

   result2 <- calcCVperformance(result2, "Macro F1")

-  performancePlot(list(result1, result2), performanceName = "Macro F1",

+  performancePlot(list(result1, result2), metric = "Macro F1",

                   title = "Comparison")

 }

@@ -12,11 +12,11 @@

 \S4method{samplesMetricMap}{list}(

   results,

   comparison = "auto",

-  metric = c("Sample Error", "Sample Accuracy", "Sample C-index"),

+  metric = "auto",

   featureValues = NULL,

   featureName = NULL,

-  metricColours = list(c("#3F48CC", "#6F75D8", "#9FA3E5", "#CFD1F2", "#FFFFFF"),

-    c("#880015", "#A53F4F", "#C37F8A", "#E1BFC4", "#FFFFFF")),

+  metricColours = list(c("#FFFFFF", "#CFD1F2", "#9FA3E5", "#6F75D8", "#3F48CC"),

+    c("#FFFFFF", "#E1BFC4", "#C37F8A", "#A53F4F", "#880015")),

   classColours = c("#3F48CC", "#880015"),

   groupColours = c("darkgreen", "yellow2"),

   fontSizes = c(24, 16, 12, 12, 12),

@@ -64,7 +64,12 @@ list-packaging but used by the main \code{list} method.}

 \item{comparison}{Default: "auto". The aspect of the experimental

 design to compare. Can be any characteristic that all results share.}

-\item{metric}{Default: "Sample Error". The sample-wise metric to plot.}

+\item{metric}{Default: \code{"auto"}. The name of the

+performance measure or "auto". If the results are classification then

+sample accuracy will be displayed. Otherwise, the results would be survival risk

+predictions and then a sample C-index will be displayed. Valid values are \code{"Sample Error"},

+\code{"Sample Error"} or \code{"Sample C-index"}. If the metric is not stored in the

+results list, the performance metric will be calculated automatically.}

 \item{featureValues}{If not NULL, can be a named factor or named numeric

 vector specifying some variable of interest to plot above the heatmap.}

@@ -142,8 +147,8 @@ values will be discretised to.

                             value = c("Example", "Bartlett Test", "Differential Variability", "2 Permutations, 2 Folds")),

                             LETTERS[1:10], features, list(1:100), list(sample(10, 10)),

                             list(function(oracle){}), NULL, predicted, actual)

-  result1 <- calcCVperformance(result1, "Sample Error")

-  result2 <- calcCVperformance(result2, "Sample Error")

+  result1 <- calcCVperformance(result1)

+  result2 <- calcCVperformance(result2)

   groups <- factor(rep(c("Male", "Female"), length.out = 10))

   names(groups) <- LETTERS[1:10]

   cholesterol <- c(4.0, 5.5, 3.9, 4.9, 5.7, 7.1, 7.9, 8.0, 8.5, 7.2)

@@ -258,14 +258,11 @@ DDresults

 The naive Bayes kernel classifier by default uses the vertical distance between class densities but it can instead use the horizontal distance to the nearest non-zero density cross-over point to confidently classify samples in the tails of the densities.

-Now, the classification error for each sample is also calculated for both the differential means and differential distribution classifiers and both *ClassifyResult* objects generated so far are plotted with *samplesMetricMap*.

+The per-sample classification accuracy is automatically calculated for both the differential means and differential distribution classifiers and plotted with *samplesMetricMap*.

 ```{r, fig.width = 10, fig.height = 7}

-DMresults <- calcCVperformance(DMresults, "Sample Error")

-DDresults <- calcCVperformance(DDresults, "Sample Error")

 resultsList <- list(Abundance = DMresults, Distribution = DDresults)

-samplesMetricMap(resultsList, metric = "Sample Error", xAxisLabel = "Sample",

-                              showXtickLabels = FALSE)

+samplesMetricMap(resultsList, showXtickLabels = FALSE)

 ```

 The benefit of this plot is that it allows the easy identification of samples which are hard to classify and could be explained by considering additional information about them. Differential distribution class prediction appears to be biased to the majority class (No Asthma).