@@ -385,3 +385,19 @@ is stored in the assayData slot.

 2010-02-07 B. Carvalho - committed version 1.5.25

  * Updated requirement of affyio to support Win64

+

+2010-02-20 R. Scharpf - committed version 1.5.25

+

+** begin adding support for ff

+

+** added use.poe option for nonpolymorphic loci

+

+** crlmm does not return anything, but saves the following objects:

+      - genotypes.rda

+      - confs.rda

+      - snp_rmaResult.rda  (gns, sns, SNR, SKW, mixtureParams, cdfName)

+      - A.rda

+      - B.rda

+      - cn_rmaResult.rda (SKW)

+

+

@@ -1,17 +1,17 @@

 Package: crlmm

 Type: Package

 Title: Genotype Calling (CRLMM) and Copy Number Analysis tool for Affymetrix SNP 5.0 and 6.0 and Illumina arrays.

-Version: 1.5.24

-Date: 2010-02-05

+Version: 1.5.25

+Date: 2010-02-20

 Author: Rafael A Irizarry, Benilton S Carvalho <bcarvalh@jhsph.edu>, Robert Scharpf <rscharpf@jhsph.edu>, Matt Ritchie <mritchie@wehi.edu.au>

 Maintainer: Benilton S Carvalho <bcarvalh@jhsph.edu>, Robert Scharpf <rscharpf@jhsph.edu>, Matt Ritchie <mritchie@wehi.EDU.AU>

 Description: Faster implementation of CRLMM specific to SNP 5.0 and 6.0 arrays, as well as a copy number tool specific to 5.0, 6.0, and Illumina platforms

 License: Artistic-2.0

-Depends: methods, Biobase (>= 2.7.2), R (>= 2.11.0), oligoClasses (>= 1.9.21)

+Depends: R (>= 2.11.0), methods, Biobase (>= 2.7.2), oligoClasses (>= 1.9.28)

 Imports: affyio (>= 1.15.2), preprocessCore, utils, stats, genefilter, splines, mvtnorm, ellipse, SNPchip

-Suggests: hapmapsnp5, hapmapsnp6, genomewidesnp5Crlmm (>= 1.0.2),genomewidesnp6Crlmm (>= 1.0.2), snpMatrix

+Enhances: ff

+Suggests: hapmapsnp5, hapmapsnp6, genomewidesnp5Crlmm (>= 1.0.2),genomewidesnp6Crlmm (>= 1.0.2), snpMatrix, metaArray

 Collate: AllGenerics.R

-         methods-AlleleSet.R

 	 methods-CNSet.R

 	 methods-eSet.R

          methods-SnpSuperSet.R

@@ -1,3 +1,4 @@

+## crlmm NAMESPACE

 useDynLib("crlmm", .registration=TRUE)

 ## Biobase

@@ -23,7 +24,7 @@ importClassesFrom(oligoClasses, SnpSuperSet, AlleleSet)

 importMethodsFrom(oligoClasses, allele, calls, "calls<-", confs,

 		  "confs<-", cnConfidence, "cnConfidence<-", 

-		  isSnp, chromosome, position, CA, "CA<-", CB, "CB<-")

+		  isSnp, chromosome, position, CA, "CA<-", CB, "CB<-", A, B)

 importFrom(oligoClasses, chromosome2integer, celfileDate, list.celfiles)

@@ -49,14 +50,14 @@ importFrom(mvtnorm, dmvnorm)

 importFrom(ellipse, ellipse)

-exportMethods(A, B, copyNumber)

+exportMethods(copyNumber)

 export(cnOptions, crlmm, crlmmIllumina, crlmmCopynumber, ellipse, readIdatFiles, snprma, getParam) 

-##export(##viterbi.CNSet, ##move to VanillaICE

-##	combineIntensities,

-##	whichPlatform,

-##	isValidCdfName,

-##	crlmmWrapper,

-##        withinGenotypeMoments,  

-##	locationAndScale, getParam.SnpSuperSet)

-export(thresholdModelParams, computeCopynumber.CNSet, nuphiAllele, coefs, biasAdjNP)

+##export everything that does not start with a .

+##exportPattern("^[^\\.]")

+

+##export(thresholdModelParams, computeCopynumber.CNSet, nuphiAllele, coefs, biasAdjNP, 

+##       nonpolymorphic.poe, crlmmWrapper,

+##       loadIlluminaCallSet, loadIlluminaRG, loadIlluminaCnrma,

+##       cnrma, 

+##       crlmmGT, oneBatch)

@@ -1,9 +1,7 @@

-setGeneric("A", function(object) standardGeneric("A"))

-setGeneric("B", function(object) standardGeneric("B"))

 ##setGeneric("A<-", function(object, value) standardGeneric("A<-"))

 ##setGeneric("B<-", function(object, value) standardGeneric("B<-"))

-setGeneric("getParam", function(object, name, batch) standardGeneric("getParam"))

+setGeneric("getParam", function(object, name, ...) standardGeneric("getParam"))

 setGeneric("cnIndex", function(object) standardGeneric("cnIndex"))

 setGeneric("cnNames", function(object) standardGeneric("cnNames"))

 setGeneric("computeCopynumber", function(object, cnOptions) standardGeneric("computeCopynumber"))

@@ -153,148 +153,387 @@ predictGender <- function(res, cdfName="genomewidesnp6", SNRMin=5){

 	return(gender)

 }

-combineIntensities <- function(res, NP=NULL, callSet){

-	rownames(res$B) <- rownames(res$A) <- res$gns

-	colnames(res$B) <- colnames(res$A) <- res$sns

-	if(!is.null(NP)){

-		blank <- matrix(NA, nrow(NP), ncol(NP))

-		dimnames(blank) <- dimnames(NP)

-		A <- rbind(res$A, NP)

-		B <- rbind(res$B, blank)

+##crlmmCopynumber <- function(filenames, cnOptions, object, ...){

+##	if(!missing(object)){

+##		stopifnot(class(object) == "CNSet")

+##		createIntermediateObjects <- FALSE

+##	} else {

+##		createIntermediateObjects <- TRUE

+##		## 33G for 1239 files

+##		crlmmResults <- crlmmWrapper(filenames, cnOptions, ...)

+##		snprmaResult <- crlmmResults[["snprmaResult"]]

+##		cnrmaResult <- crlmmResults[["cnrmaResult"]]

+##		callSet <- crlmmResults[["callSet"]]

+##		rm(crlmmResults); gc()

+##		annotation(callSet) <- cnOptions[["cdfName"]]

+##		stopifnot(identical(featureNames(callSet), snprmaResult[["gns"]]))

+##		path <- system.file("extdata", package=paste(annotation(callSet), "Crlmm", sep=""))	

+##		load(file.path(path, "snpProbes.rda"))

+##		snpProbes <- get("snpProbes")

+##		load(file.path(path, "cnProbes.rda"))

+##		cnProbes <- get("cnProbes")	

+##		k <- grep("chr", colnames(snpProbes))

+##		if(length(k) < 1) stop("chr or chromosome not in colnames(snpProbes)")

+##	}

+##	set.seed(cnOptions[["seed"]])  ##for reproducibility

+##	cnFile <- cnOptions[["cnFile"]]

+##	chromosome <- cnOptions[["chromosome"]]

+##	SNRmin <- cnOptions[["SNRmin"]]

+##	for(CHR in chromosome){

+##		cat("Chromosome ", CHR, "\n")

+##		if(createIntermediateObjects){

+##			snps <- rownames(snpProbes)[snpProbes[, k] == CHR]

+##			cnps <- rownames(cnProbes)[cnProbes[, k] == CHR]

+##			index.snps <- match(snps, featureNames(callSet))

+##			index.nps <- match(cnps, rownames(cnrmaResult[["NP"]]))

+##			if(!is.null(cnrmaResult)){

+##				npI <- cnrmaResult$NP[index.nps, ]

+##			} else npI <- NULL

+##			snpI <- list(A=snprmaResult$A[index.snps, ],

+##				     B=snprmaResult$B[index.snps, ],

+##				     sns=snprmaResult$sns,

+##				     gns=snprmaResult$gns[index.snps],

+##				     SNR=snprmaResult$SNR,

+##				     SKW=snprmaResult$SKW,

+##				     mixtureParams=snprmaResult$mixtureParams,

+##				     cdfName=snprmaResult$cdfName)

+##			cnOptions[["batch"]] <- cnOptions[["batch"]][snpI[["SNR"]]  >= SNRmin]

+##			cnSet <- combineIntensities(res=snpI,

+##						    NP=npI,

+##						    callSet=callSet[index.snps, ])

+##			if(any(cnSet$SNR > SNRmin)){

+##				message(paste("Excluding samples with SNR < ", SNRmin))

+##				cnSet <- cnSet[, cnSet$SNR >= SNRmin]

+##			}

+##			rm(snpI, npI, snps, cnps, index.snps, index.nps); gc()

+##			pData(cnSet)$batch <- cnOptions[["batch"]]

+##			featureData(cnSet) <- lm.parameters(cnSet, cnOptions)

+##		} else {

+##			cnSet <- object

+##		}

+##		if(CHR != 24){

+##			## 60G for 1239 files

+##			cnSet <- computeCopynumber(cnSet, cnOptions)

+##		} else{

+##			message("Copy number estimates not available for chromosome Y.  Saving only the 'callSet' object for this chromosome")

+##			alleleSet <- cnSet

+##			save(alleleSet, file=file.path(cnOptions[["outdir"]], paste("alleleSet_", CHR, ".rda", sep="")))

+##			rm(cnSet, alleleSet); gc()

+##			next()

+##		}

+##		if(length(chromosome) == 1){

+##			if(cnOptions[["save.cnset"]]){

+##				save(cnSet, file=file.path(paste(cnFile, "_", CHR, ".rda", sep="")))				

+##				##save(cnSet, file=file.path(cnOptions[["outdir"]], paste("cnSet_", CHR, ".rda", sep="")))

+##			}

+##		}

+##		if(length(chromosome) > 1){

+##			message(paste("Saving ", file.path(cnOptions[["outdir"]], paste(cnFile, "_", CHR, ".rda", sep=""))))

+##			save(cnSet, file=file.path(paste(cnFile, "_", CHR, ".rda", sep="")))				

+##			##save(cnSet, file=file.path(cnOptions[["outdir"]], paste("cnSet_", CHR, ".rda", sep="")))

+##			rm(cnSet); gc()

+##		} else {

+##			return(cnSet)

+##		}

+##	}

+##	saved.objects <- list.files(cnOptions[["outdir"]], pattern=cnFile, full.names=TRUE)		

+##	return(saved.objects)

+##}

+

+

+

+crlmmCopynumber <- function(filenames, cnOptions, object, ...){

+	cnOpts <- cnOptions

+	batchSize <- cnOptions[["crlmmBatchSize"]]

+	batch <- cnOptions[["batch"]]

+	outdir <- cnOptions[["outdir"]]

+	##if necessary, split plates in separate batches

+	if(length(filenames) > batchSize){

+		L <- length(table(batch))

+		message("Processing the files in")

+		batchList <- split(names(table(batch)), rep(1:L, each=10, length.out=L))

+		for(i in seq(along=batchList)){

+			index <- as.character(batch) %in% as.character(batchList[[i]])

+			fns <- filenames[index]

+			cnOpts[["batch"]] <- cnOptions[["batch"]][index]

+			cnOpts[["AFile"]] <- paste(outdir, "/", paste(cnOptions[["AFile"]], i, sep="_crlmmBatch"), ".rda", sep="")

+			cnOpts[["BFile"]] <- paste(outdir, "/", paste(cnOptions[["BFile"]], i, sep="_crlmmBatch"), ".rda", sep="")

+			cnOpts[["callsFile"]] <- paste(outdir, "/", paste(cnOptions[["callsFile"]], i, sep="_crlmmBatch"), ".rda", sep="")

+			cnOpts[["confsFile"]] <- paste(outdir, "/", paste(cnOptions[["confsFile"]], i, sep="_crlmmBatch"), ".rda", sep="")

+			cnOpts[["snprmaFile"]] <- paste(outdir, "/", paste(cnOptions[["snprmaFile"]], i, sep="_crlmmBatch"), ".rda", sep="")

+			cnOpts[["cnrmaFile"]] <- paste(outdir, "/", paste(cnOptions[["cnrmaFile"]], i, sep="_crlmmBatch"), ".rda", sep="")

+			cnOpts[["cnFile"]] <- file.path(outdir, paste(cnOptions[["cnFile"]], i, sep="_crlmmBatch"))

+			cnOpts[["rgFile"]] <- file.path(outdir, paste(cnOptions[["rgFile"]], i, sep="_crlmmBatch"))

+			crlmmCopynumber.crlmmBatch(fns, cnOpts, object, ...)

+		}

 	} else {

-		A <- res$A

-		B <- res$B

+		cnOpts[["AFile"]] <- paste(outdir, "/", cnOptions[["AFile"]], ".rda", sep="")

+		cnOpts[["BFile"]] <- paste(outdir, "/", cnOptions[["BFile"]], ".rda", sep="")

+		cnOpts[["callsFile"]] <- paste(outdir, "/", cnOptions[["callsFile"]], ".rda", sep="")

+		cnOpts[["confsFile"]] <- paste(outdir, "/", cnOptions[["confsFile"]], ".rda", sep="")

+		cnOpts[["snprmaFile"]] <- paste(outdir, "/",cnOptions[["snprmaFile"]], ".rda", sep="")

+		cnOpts[["cnrmaFile"]] <- paste(outdir, "/", cnOptions[["cnrmaFile"]], ".rda", sep="")

+		cnOpts[["cnFile"]] <- file.path(outdir, cnOptions[["cnFile"]])

+		cnOpts[["rgFile"]] <- file.path(outdir, cnOptions[["rgFile"]])

+		crlmmCopynumber.crlmmBatch(filenames, cnOpts, object, ...)

 	}

-	dimnames(B) <- dimnames(A)

-	index.snps <- match(res$gns, rownames(A))

-	callsConfs <- calls <- matrix(NA, nrow(A), ncol(A), dimnames=dimnames(A))

-	

-	calls[index.snps, ] <- calls(callSet)

-	callsConfs[index.snps, ] <- assayData(callSet)[["callProbability"]]

-	fd <- data.frame(matrix(NA, nrow(calls), length(fvarLabels(callSet))))

-	fd[index.snps, ] <- fData(callSet)

-	rownames(fd) <- rownames(A)

-	colnames(fd) <- fvarLabels(callSet)

-	fD <- new("AnnotatedDataFrame",

-		  data=data.frame(fd),

-		  varMetadata=data.frame(labelDescription=colnames(fd), row.names=colnames(fd)))

-	superSet <- new("CNSet",

-			CA=matrix(NA, nrow(A), ncol(A), dimnames=dimnames(A)),

-			CB=matrix(NA, nrow(A), ncol(A), dimnames=dimnames(A)),

-			alleleA=A,

-			alleleB=B,

-			call=calls,

-			callProbability=callsConfs,

-			featureData=fD,

-			phenoData=phenoData(callSet),

-			experimentData=experimentData(callSet),

-			protocolData=protocolData(callSet),

-			annotation=annotation(callSet))

-	return(superSet)

 }

-harmonizeDimnamesTo <- function(object1, object2){

-	#object2 should be a subset of object 1

-	object2 <- object2[featureNames(object2) %in% featureNames(object1), ]

-	object1 <- object1[match(featureNames(object2), featureNames(object1)), ]

-	object1 <- object1[, match(sampleNames(object2), sampleNames(object1))]

-	stopifnot(all.equal(featureNames(object1), featureNames(object2)))

-	stopifnot(all.equal(sampleNames(object1), sampleNames(object2)))

-	return(object1)

-}

-crlmmCopynumber <- function(filenames, cnOptions, object, ...){

+crlmmCopynumber.crlmmBatch <- function(filenames, cnOptions, object, ...){

+	set.seed(cnOptions[["seed"]])  ##for reproducibility

 	if(!missing(object)){

 		stopifnot(class(object) == "CNSet")

 		createIntermediateObjects <- FALSE

 	} else {

 		createIntermediateObjects <- TRUE

-		crlmmResults <- crlmmWrapper(filenames, cnOptions, ...)

-		snprmaResult <- crlmmResults[["snprmaResult"]]

-		cnrmaResult <- crlmmResults[["cnrmaResult"]]

-		callSet <- crlmmResults[["callSet"]]

-		rm(crlmmResults); gc()

-		annotation(callSet) <- cnOptions[["cdfName"]]

-		stopifnot(identical(featureNames(callSet), snprmaResult[["gns"]]))

-		path <- system.file("extdata", package=paste(annotation(callSet), "Crlmm", sep=""))	

-		load(file.path(path, "snpProbes.rda"))

-		snpProbes <- get("snpProbes")

-		load(file.path(path, "cnProbes.rda"))

-		cnProbes <- get("cnProbes")	

-		k <- grep("chr", colnames(snpProbes))

-		if(length(k) < 1) stop("chr or chromosome not in colnames(snpProbes)")

+		## 33G for 1239 files

+		cnOptions[["save.it"]] <- TRUE

+		##crlmmResults <- crlmmWrapper(filenames, cnOptions, ...)

+		crlmmWrapper(filenames, cnOptions, ...)

 	}

-	set.seed(cnOptions[["seed"]])  ##for reproducibility

+	load.it <- cnOptions[["load.it"]]

+	outdir <- cnOptions[["outdir"]]

+	snprmaFile <- cnOptions[["snprmaFile"]]

+	cnFile <- cnOptions[["cnFile"]]

+	cnrmaFile <- cnOptions[["cnrmaFile"]]

+	callsFile <- cnOptions[["callsFile"]]

+	confsFile <- cnOptions[["confsFile"]]

+	AFile <- cnOptions[["AFile"]]

+	BFile <- cnOptions[["BFile"]]

+	path <- system.file("extdata", package=paste(cnOptions[["cdfName"]], "Crlmm", sep=""))

+	load(file.path(path, "snpProbes.rda"))

+	snpProbes <- get("snpProbes")

+	load(file.path(path, "cnProbes.rda"))

+	cnProbes <- get("cnProbes")	

+	k <- grep("chr", colnames(snpProbes))

+	message("Attaching quantile-normalized intensities...")

+	##cwd <- getwd()

+	##setwd(dirname(snprmaFile[1]))

+	load(AFile)

+	if(isPackageLoaded("ff")) open(A)

+	load(BFile)

+	if(isPackageLoaded("ff")) open(B)	

+	message("Loading genotype calls...")	

+	load(callsFile)

+	calls <- get("calls")

+	if(isPackageLoaded("ff")) open(calls)	

+	load(confsFile)

+	confs <- get("confs")

+	if(isPackageLoaded("ff")) open(confs)	

+	fns <- rownames(A)

+	nr <- nrow(A); nc <- ncol(A)

+	if(isPackageLoaded("ff")){

+		##initialize CA, CB

+		if(!file.exists(file.path(outdir, "CA.rda"))){

+			CA <- ff(dim=c(nr, nc), vmode="integer", finalizer="close", dimnames=dimnames(A))

+

+		} else{

+			load(file.path(outdir, "CA.rda"))

+			if(!all(dim(CA) == dim(A)))

+				CA <- ff(dim=c(nr, nc), vmode="integer", finalizer="close", dimnames=dimnames(A), overwrite=TRUE)				

+			open(CA)

+		}

+		if(!file.exists(file.path(outdir, "CB.rda"))){		

+			CB <- ff(dim=c(nr, nc), vmode="integer", finalizer="close", dimnames=dimnames(A))

+		} else{

+			load(file.path(outdir, "CB.rda"))

+			if(!all(dim(CB) == dim(A)))

+				CB <- ff(dim=c(nr, nc), vmode="integer", finalizer="close", dimnames=dimnames(A), overwrite=TRUE)							

+			open(CB)

+		}

+	}

+	##cnFile <- cnOptions[["cnFile"]]

 	chromosome <- cnOptions[["chromosome"]]

 	SNRmin <- cnOptions[["SNRmin"]]

+	ll <- 1

+	scanDates <- data.frame(ScanDate=sapply(filenames, celfileDate))

+	rownames(scanDates) <- basename(rownames(scanDates))

+	protocoldata <- new("AnnotatedDataFrame",

+			    data=scanDates,

+			    varMetadata=data.frame(labelDescription=colnames(scanDates),

+			    row.names=colnames(scanDates)))

+	load(file.path(cnOptions[["outdir"]], "sampleStats.rda"))

+	sampleStats <- data.frame(sampleStats, batch=cnOptions[["batch"]])

+	pD <- new("AnnotatedDataFrame",

+		  data=sampleStats,

+		  varMetadata=data.frame(labelDescription=colnames(sampleStats)))

+	batch <- cnOptions[["batch"]]

 	for(CHR in chromosome){

+		snps <- rownames(snpProbes)[snpProbes[, k] == CHR]

+		cns <- rownames(cnProbes)[cnProbes[, k] == CHR]		

+		index.snps <- match(snps, rownames(A))

+		index.cn <- match(cns, rownames(A))

+		row.index <- c(index.snps, index.cn)

 		cat("Chromosome ", CHR, "\n")

-		if(createIntermediateObjects){

-			snps <- rownames(snpProbes)[snpProbes[, k] == CHR]

-			cnps <- rownames(cnProbes)[cnProbes[, k] == CHR]

-			index.snps <- match(snps, featureNames(callSet))

-			index.nps <- match(cnps, rownames(cnrmaResult[["NP"]]))

-			if(!is.null(cnrmaResult)){

-				npI <- cnrmaResult$NP[index.nps, ]

-			} else npI <- NULL

-			snpI <- list(A=snprmaResult$A[index.snps, ],

-				     B=snprmaResult$B[index.snps, ],

-				     sns=snprmaResult$sns,

-				     gns=snprmaResult$gns[index.snps],

-				     SNR=snprmaResult$SNR,

-				     SKW=snprmaResult$SKW,

-				     mixtureParams=snprmaResult$mixtureParams,

-				     cdfName=snprmaResult$cdfName)

-			cnOptions[["batch"]] <- cnOptions[["batch"]][snpI[["SNR"]]  >= SNRmin]

-			cnSet <- combineIntensities(res=snpI,

-						    NP=npI,

-						    callSet=callSet[index.snps, ])

+		for(i in seq(along=unique(batch))){

+			PLATE <- unique(batch)[i]

+			sample.index <- which(batch==PLATE)

+			##cnOptions[["batch"]] <- cnOptions[["batch"]][snpI[["SNR"]]  >= SNRmin]

+			if(isPackageLoaded("ff")){

+				ca <- CA[row.index, sample.index]

+				cb <- CB[row.index, sample.index]

+			} else{

+				dns <- dimnames(A[row.index, sample.index])

+				cb <- ca <- matrix(NA, nr=length(row.index), nc=length(sample.index), dimnames=dns)

+			}

+			cnSet <- new("CNSet",

+				     alleleA=A[row.index, sample.index],

+				     alleleB=B[row.index, sample.index],

+				     call=calls[row.index, sample.index],

+				     callProbability=confs[row.index, sample.index],

+				     CA=ca,

+				     CB=cb,

+				     featureData=annotatedDataFrameFrom(A[row.index, sample.index], byrow=TRUE),

+				     phenoData=pD[sample.index, ],

+				     protocolData=protocoldata[sample.index, ],

+				     annotation=cnOptions[["cdfName"]])			

 			if(any(cnSet$SNR > SNRmin)){

 				message(paste("Excluding samples with SNR < ", SNRmin))

 				cnSet <- cnSet[, cnSet$SNR >= SNRmin]

 			}

-			rm(snpI, npI, snps, cnps, index.snps, index.nps); gc()

-			pData(cnSet)$batch <- cnOptions[["batch"]]

+			##rm(snpI, NP, gtSet); gc()

 			featureData(cnSet) <- lm.parameters(cnSet, cnOptions)

-		} else {

-			cnSet <- object

-		}

-		if(CHR != 24){		

+		} 

+		if(CHR != 24){

+			## 60G for 1239 files

 			cnSet <- computeCopynumber(cnSet, cnOptions)

+			if(isPackageLoaded("ff")){

+				CA[row.index, sample.index] <- cnSet@assayData[["CA"]]

+				CB[row.index, sample.index] <- cnSet@assayData[["CB"]]

+			}

 		} else{

 			message("Copy number estimates not available for chromosome Y.  Saving only the 'callSet' object for this chromosome")

 			alleleSet <- cnSet

-			save(alleleSet, file=file.path(cnOptions[["outdir"]], paste("alleleSet_", CHR, ".rda", sep="")))

-			rm(cnSet, alleleSet); gc()

+			rm(cnSet) ; gc()

+			save(alleleSet, file=file.path(cnOptions[["outdir"]], paste("alleleSet_", PLATE, "_", CHR, ".rda", sep="")))

+			rm(alleleSet); gc()

 			next()

 		}

 		if(length(chromosome) == 1){

 			if(cnOptions[["save.cnset"]]){

-				save(cnSet, file=file.path(cnOptions[["outdir"]], paste("cnSet_", CHR, ".rda", sep="")))

+				save(cnSet, file=paste(cnFile, "_", PLATE, "_", CHR, ".rda", sep=""))				

 			}

 		}

 		if(length(chromosome) > 1){

-			save(cnSet, file=file.path(cnOptions[["outdir"]], paste("cnSet_", CHR, ".rda", sep="")))

+			save(cnSet, file=paste(cnFile, "_", PLATE, "_", CHR, ".rda", sep=""))

+			##save(cnSet, file=file.path(cnOptions[["outdir"]], paste("cnSet_", CHR, ".rda", sep="")))

 			rm(cnSet); gc()

 		} else {

+			if(isPackageLoaded("ff")) {

+				close(CA); close(CB)

+				save(CA, file=file.path(cnOptions[["outdir"]], "CA.rda"))

+				save(CB, file=file.path(cnOptions[["outdir"]], "CB.rda"))		

+			}			

 			return(cnSet)

 		}

 	}

-	saved.objects <- list.files(cnOptions[["outdir"]], pattern="cnSet", full.names=TRUE)		

-	return(saved.objects)

+	if(isPackageLoaded("ff")) {

+		close(CA); close(CB)

+		save(CA, file=file.path(cnOptions[["outdir"]], "CA.rda"))

+		save(CB, file=file.path(cnOptions[["outdir"]], "CB.rda"))		

+	}

 }

+loadIlluminaRG <- function(rgFile, crlmmFile, load.it, save.it,...){

+##	if(missing(rgFile)){

+##		##stop("must specify 'rgFile'.")

+##		rgFile <- file.path(dirname(crlmmFile), "rgFile.rda")

+##		message("rgFile not specified.  Using ", rgFile)

+##	}

+	if(!load.it){

+		RG <- readIdatFiles(...)

+		if(save.it) save(RG, file=rgFile)

+	}

+	if(load.it & !file.exists(rgFile)){

+		message("load.it is TRUE, but rgFile not present.  Attempting to read the idatFiles.")

+		RG <- readIdatFiles(...)

+		if(save.it) save(RG, file=rgFile)

+	}

+	if(load.it & file.exists(rgFile)){

+		message("Loading RG file")

+		load(rgFile)

+		RG <- get("RG")

+	}

+	return(RG)

+}

+

+loadIlluminaCallSet <- function(crlmmFile, snprmaFile, RG, load.it, save.it, cdfName){

+	if(!file.exists(crlmmFile) | !load.it){		

+		callSet <- crlmmIllumina(RG=RG,

+					 cdfName=cdfName,

+					 sns=sampleNames(RG),

+					 returnParams=TRUE,

+					 save.it=TRUE,

+					 intensityFile=snprmaFile)

+		if(save.it) save(callSet, file=crlmmFile)

+	} else {

+		message("Loading ", crlmmFile, "...")

+		load(crlmmFile)

+		callSet <- get("callSet")

+	}

+	protocolData(callSet) <- protocolData(RG)

+	return(callSet)

+}

+

+

+##loadAffyCallSet <- function(filenames, confsFile, callsFile, snprmaFile, load.it, save.it,  cdfName){

+##

+####		if(save.it){

+####			message("Saving callSet to", callsFile)

+####			##save(callSet, cnrmaResult, file=callsFile)

+####			save(callSet, file=callsFile)

+####		}

+####	} else {

+####		message("Loading ", callsFile, "...")

+####		##load(snprmaFile)				

+####		load(callsFile)

+####		callSet <- get("callSet")

+####		##cnrmaResult <- get("cnrmaResult")

+####	}

+##

+##}

+##	if(platform=="affymetrix") {

+##		protocolData(callSet)[["ScanDate"]] <- as.character(celDates(filenames))

+##		sampleNames(protocolData(callSet)) <- sampleNames(callSet)

+##	}

+

+##loadAffyCnrma <- function(filenames, cnrmaFile, cdfName, outdir, load.it, save.it, use.bigmemory=FALSE){

+##	if(!file.exists(cnrmaFile) | !load.it){	

+##		message("Quantile normalizing the copy number probes...")

+##		cnrmaResult <- cnrma2(filenames=filenames, cdfName=cdfName, outdir=outdir, cnrmaFile=cnrmaFile)

+##	} else cnrmaResult <- attach.big.matrix("NP.desc", outdir)

+##	return(cnrmaResult)

+##}

+

+loadIlluminaCnrma <- function(){

+	if(exists("cnAB")){

+		np.A <- as.integer(cnAB$A)

+		np.B <- as.integer(cnAB$B)

+		np <- ifelse(np.A > np.B, np.A, np.B)

+		np <- matrix(np, nrow(cnAB$A), ncol(cnAB$A))

+		rownames(np) <- cnAB$gns

+		colnames(np) <- cnAB$sns

+		cnAB$NP <- np

+		##sampleNames(callSet) <- res$sns

+		sampleNames(callSet) <- cnAB$sns

+		cnrmaResult <- get("cnAB")

+	} else cnrmaResult <- NULL

+	return(cnrmaResult)

+}

 crlmmWrapper <- function(filenames, cnOptions, ...){

 	cdfName <- cnOptions[["cdfName"]]

 	load.it <- cnOptions[["load.it"]]

 	save.it <- cnOptions[["save.it"]]

-	splitByChr <- cnOptions[["splitByChr"]]

-	crlmmFile <- cnOptions[["crlmmFile"]]

-	intensityFile=cnOptions[["intensityFile"]]

+	callsFile <- cnOptions[["callsFile"]]

+	confsFile <- cnOptions[["confsFile"]]

+	AFile=cnOptions[["AFile"]]

+	BFile=cnOptions[["BFile"]]	

+	snprmaFile=cnOptions[["snprmaFile"]]

+	cnrmaFile=cnOptions[["cnrmaFile"]]

 	rgFile=cnOptions[["rgFile"]]

-	##use.ff=cnOptions[["use.ff"]]

 	outdir <- cnOptions[["outdir"]]

 	if(missing(cdfName)) stop("cdfName is missing -- a valid cdfName is required.  See crlmm:::validCdfNames()")

 	platform <- whichPlatform(cdfName)

@@ -307,148 +546,57 @@ crlmmWrapper <- function(filenames, cnOptions, ...){

 			stop(cdfName, " is not a valid entry.  See crlmm:::validCdfNames(platform)")

 		} else cat("TRUE\n")

 	}

-	if(missing(intensityFile)) stop("must specify 'intensityFile'.")

-	if(missing(crlmmFile)) stop("must specify 'crlmmFile'.")

-	if(platform == "illumina"){

-		if(missing(rgFile)){

-			##stop("must specify 'rgFile'.")

-			rgFile <- file.path(dirname(crlmmFile), "rgFile.rda")

-			message("rgFile not specified.  Using ", rgFile)

-		}

-		if(!load.it){

-			RG <- readIdatFiles(...)

-			if(save.it) save(RG, file=rgFile)

-		}

-		if(load.it & !file.exists(rgFile)){

-			message("load.it is TRUE, but rgFile not present.  Attempting to read the idatFiles.")

-			RG <- readIdatFiles(...)

-			if(save.it) save(RG, file=rgFile)

-		}

-		if(load.it & file.exists(rgFile)){

-			message("Loading RG file")

-			load(rgFile)

-			RG <- get("RG")

-		}

-	}

-	if(!(file.exists(dirname(crlmmFile)))) stop(dirname(crlmmFile), " does not exist.")

-	if(!(file.exists(dirname(intensityFile)))) stop(dirname(intensityFile), " does not exist.")

-	##---------------------------------------------------------------------------

-	## FIX

-	outfiles <- file.path(dirname(crlmmFile), paste("crlmmSetList_", 1:24, ".rda", sep=""))

-	if(load.it & all(file.exists(outfiles))){

-		load(outfiles[1])

-		crlmmSetList <- get("crlmmSetList")

-		if(!all(sampleNames(crlmmSetList) == basename(filenames))){

-			stop("load.it is TRUE, but sampleNames(crlmmSetList != basename(filenames))")

-		} else{

-			return("load.it is TRUE and 'crlmmSetList_<CHR>.rda' objects found. Nothing to do...")

-		}

-	}

+	if(platform == "illumina") RG <- loadIlluminaRG(rgFile, callsFile, load.it, save.it)

+	if(!(file.exists(dirname(callsFile)))) stop(dirname(callsFile), " does not exist.")

+	if(!(file.exists(dirname(snprmaFile[1])))) stop(dirname(snprmaFile[1]), " does not exist.")

 	if(load.it){

-		if(!file.exists(crlmmFile)){

-			message("load.it is TRUE, but ", crlmmFile, " does not exist.  Rerunning the genotype calling algorithm") 

-			load.it <- FALSE

+		if(!file.exists(callsFile)){

+			message("load.it is TRUE, but ", callsFile, " does not exist.  Rerunning the genotype calling algorithm") 

+			##load.it <- FALSE

 		}

 	}

 	if(platform == "affymetrix"){

-		if(!file.exists(crlmmFile) | !load.it){

-			callSet <- crlmm(filenames=filenames,

-					     cdfName=cdfName,

-					     save.it=TRUE,

-					     load.it=load.it,

-					     intensityFile=intensityFile)

-			message("Quantile normalizing the copy number probes...")		

-			cnrmaResult <- cnrma(filenames=filenames, cdfName=cdfName, outdir=outdir)

-			if(save.it){

-				message("Saving callSet and cnrmaResult to", crlmmFile)

-				save(callSet, cnrmaResult, file=crlmmFile)

-			}

-		} else {

-			message("Loading ", crlmmFile, "...")

-			load(intensityFile)				

-			load(crlmmFile)

-			callSet <- get("callSet")

-			cnrmaResult <- get("cnrmaResult")

-		}

-		scanDates <- data.frame(ScanDate=sapply(filenames, celfileDate))

-		protocolData(callSet) <- new("AnnotatedDataFrame",

-					     data=scanDates,

-					     varMetadata=data.frame(labelDescription=colnames(scanDates),

-					     row.names=colnames(scanDates)))

-	}

-	if(platform == "illumina"){

-		if(!file.exists(crlmmFile) | !load.it){		

-			callSet <- crlmmIllumina(RG=RG,

-						 cdfName=cdfName,

-						 sns=sampleNames(RG),

-						 returnParams=TRUE,

-						 save.it=TRUE,

-						 intensityFile=intensityFile)

-			if(save.it) save(callSet, file=crlmmFile)

-		} else {

-			message("Loading ", crlmmFile, "...")

-			load(crlmmFile)

-			callSet <- get("callSet")

+		if(!file.exists(callsFile) | !load.it){

+			crlmm(filenames=filenames,

+			      cdfName=cdfName,

+			      save.it=TRUE,

+			      load.it=load.it,

+			      intensityFile=snprmaFile,

+			      callsFile=callsFile,

+			      confsFile=confsFile,

+			      AFile=AFile,

+			      BFile=BFile)

 		}

-		protocolData(callSet) <- protocolData(RG)

 	}

-	if(platform=="affymetrix") {

-		protocolData(callSet)[["ScanDate"]] <- as.character(celDates(filenames))

-		sampleNames(protocolData(callSet)) <- sampleNames(callSet)

-	}	

-	load(intensityFile)

-	snprmaResult <- get("res")

-	if(platform=="illumina"){

-		if(exists("cnAB")){

-			np.A <- as.integer(cnAB$A)

-			np.B <- as.integer(cnAB$B)

-			np <- ifelse(np.A > np.B, np.A, np.B)

-			np <- matrix(np, nrow(cnAB$A), ncol(cnAB$A))

-			rownames(np) <- cnAB$gns

-			colnames(np) <- cnAB$sns

-			cnAB$NP <- np

-			##sampleNames(callSet) <- res$sns

-			sampleNames(callSet) <- cnAB$sns

-			cnrmaResult <- get("cnAB")

-		} else cnrmaResult <- NULL

+	gc()

+	if(platform == "illumina") {

+		browser()

+		callSet <- loadIlluminaCallSet(callsFile, snprmaFile, RG, load.it, save.it, cdfName)

 	}

-	if(platform=="affymetrix"){

-		if(exists("cnrmaResult")){

-			cnrmaResult <- get("cnrmaResult")

-		} else cnrmaResult <- NULL

+	if(platform == "affymetrix"){

+		if(!file.exists(cnrmaFile) | !load.it){	

+			message("Quantile normalizing the copy number probes...")

+			## updates A matrix and saves cnrmaFile

+			cnrma(filenames=filenames, cdfName=cdfName, outdir=outdir, cnrmaFile=cnrmaFile, AFile=AFile)

+		} 

 	}

-	crlmmResults <- list(snprmaResult=snprmaResult,

-			     cnrmaResult=cnrmaResult,

-			     callSet=callSet)

-

+##	if(!is.null(cnrmaResult)){

+##		for(CHR in chromosome){

+##			cat(CHR, " ")

+##			cnps <- rownames(cnProbes)[cnProbes[, k] == CHR]

+##			index.nps <- match(cnps, rownames(cnrmaResult[["NP"]]))

+##			NP <- cnrmaResult$NP[index.nps, ]

+##			save(NP, file=file.path(tmpdir, paste("NP_", CHR, ".rda", sep="")))

+##			rm(NP); gc()

+##		}

+##	}

 	if(!save.it){

 		message("Cleaning up")		

-		unlink(intensityFile)

+		unlink(snprmaFile); unlink(cnrmaFile)

 	}

-	return(crlmmResults)

 }

-validCdfNames <- function(){

-	c("genomewidesnp6",

-	  "genomewidesnp5",

-	  "human370v1c",

-	  "human370quadv3c",

-	  "human550v3b",

-	  "human650v3a",

-	  "human610quadv1b",

-	  "human660quadv1a",

-	  "human1mduov3b")

-}

-isValidCdfName <- function(cdfName){

-	chipList <- validCdfNames()

-	result <- cdfName %in% chipList	

-	if(!(result)){

-		warning("cdfName must be one of the following: ",

-			chipList)

-	}

-	return(result)

-}

 whichPlatform <- function(cdfName){

 	index <- grep("genomewidesnp", cdfName)

@@ -463,7 +611,50 @@ whichPlatform <- function(cdfName){

 # steps: quantile normalize hapmap: create 1m_reference_cn.rda object

-cnrma <- function(filenames, cdfName, sns, seed=1, verbose=FALSE, outdir){

+##cnrma <- function(filenames, cdfName, sns, seed=1, verbose=FALSE, outdir){

+##	if(missing(cdfName)) stop("must specify cdfName")

+##	pkgname <- getCrlmmAnnotationName(cdfName)

+##	require(pkgname, character.only=TRUE) || stop("Package ", pkgname, " not available")

+##	if (missing(sns)) sns <- basename(filenames)

+##        loader("npProbesFid.rda", .crlmmPkgEnv, pkgname)

+##	fid <- getVarInEnv("npProbesFid")

+##	set.seed(seed)

+##	idx2 <- sample(length(fid), 10^5) ##for skewness. no need to do everything

+##	SKW <- vector("numeric", length(filenames))

+##

+##	

+##	NP <- matrix(NA, length(fid), length(filenames))

+##	verbose <- TRUE

+##	if(verbose){

+##		message("Processing ", length(filenames), " files.")

+##		if (getRversion() > '2.7.0') pb <- txtProgressBar(min=0, max=length(filenames), style=3)

+##	}

+##	if(cdfName=="genomewidesnp6"){

+##		loader("1m_reference_cn.rda", .crlmmPkgEnv, pkgname)

+##	}

+##	if(cdfName=="genomewidesnp5"){

+##		loader("5.0_reference_cn.rda", .crlmmPkgEnv, pkgname)

+##	}

+##	reference <- getVarInEnv("reference")

+##	##if(!is.matrix(reference)) stop("target distribution for quantile normalization not available.")

+##	for(i in seq(along=filenames)){

+##		y <- as.matrix(read.celfile(filenames[i], intensity.means.only=TRUE)[["INTENSITY"]][["MEAN"]][fid])

+##		x <- log2(y[idx2])

+##		SKW[i] <- mean((x-mean(x))^3)/(sd(x)^3)

+##		rm(x)

+##		NP[, i] <- as.integer(normalize.quantiles.use.target(y, target=reference))

+##		if (verbose)

+##			if (getRversion() > '2.7.0') setTxtProgressBar(pb, i)

+##			else cat(".")

+##	}

+##	dimnames(NP) <- list(names(fid), sns)

+##	##dimnames(NP) <- list(map[, "man_fsetid"], sns)

+##	res3 <- list(NP=NP, SKW=SKW)

+##	cat("\n")

+##	return(res3)

+##}

+

+cnrma <- function(filenames, cdfName, sns, seed=1, verbose=FALSE, outdir, cnrmaFile, AFile){

 	if(missing(cdfName)) stop("must specify cdfName")

 	pkgname <- getCrlmmAnnotationName(cdfName)

 	require(pkgname, character.only=TRUE) || stop("Package ", pkgname, " not available")

@@ -473,17 +664,15 @@ cnrma <- function(filenames, cdfName, sns, seed=1, verbose=FALSE, outdir){

 	set.seed(seed)

 	idx2 <- sample(length(fid), 10^5) ##for skewness. no need to do everything

 	SKW <- vector("numeric", length(filenames))

-##	if(bigmemory){

-##		NP <- filebacked.big.matrix(length(pnsa), length(filenames),

-##					    type="integer",

-##					    init=as.integer(0),

-##					    backingpath=outdir,

-##					    backingfile="NP.bin",

-##					    descriptorfile="NP.desc")

-##	} else{

-		NP <- matrix(NA, length(fid), length(filenames))

-##	}

-	verbose <- TRUE

+	load(AFile)

+	if(isPackageLoaded("ff")) open(A)

+	path <- system.file("extdata", package=pkgname)

+	load(file.path(path, "cnProbes.rda"))

+	cnProbes <- get("cnProbes")

+	cnps <- rownames(cnProbes)

+	cnps <- cnps[cnps %in% rownames(A)]

+	index <- match(cnps, rownames(A), nomatch=0)

+	index <- index[index != 0]

 	if(verbose){

 		message("Processing ", length(filenames), " files.")

 		if (getRversion() > '2.7.0') pb <- txtProgressBar(min=0, max=length(filenames), style=3)

@@ -500,17 +689,21 @@ cnrma <- function(filenames, cdfName, sns, seed=1, verbose=FALSE, outdir){

 		y <- as.matrix(read.celfile(filenames[i], intensity.means.only=TRUE)[["INTENSITY"]][["MEAN"]][fid])

 		x <- log2(y[idx2])

 		SKW[i] <- mean((x-mean(x))^3)/(sd(x)^3)

-		rm(x)

-		NP[, i] <- as.integer(normalize.quantiles.use.target(y, target=reference))

+		rm(x); gc()

+		A[index, i] <- as.integer(normalize.quantiles.use.target(y, target=reference))

+		##NP[, i] <- as.integer(normalize.quantiles.use.target(y, target=reference))

 		if (verbose)

 			if (getRversion() > '2.7.0') setTxtProgressBar(pb, i)

 			else cat(".")

+		rm(y); gc()

 	}

-	dimnames(NP) <- list(names(fid), sns)

-	##dimnames(NP) <- list(map[, "man_fsetid"], sns)

-	res3 <- list(NP=NP, SKW=SKW)

-	cat("\n")

-	return(res3)

+	message("Done")

+	cnrmaResults <- list(SKW=SKW)

+	save(cnrmaResults, file=cnrmaFile)

+	if(isPackageLoaded("ff")) close(A)

+	save(A, file=AFile)

+	rm(list=ls()); gc()

+	return()

 }

 getFlags <- function(object, PHI.THR){

@@ -555,21 +748,29 @@ thresholdCopynumber <- function(object){

 	return(object)

 }