@@ -1,11 +1,10 @@

 Package: RGMQL

 Type: Package

 Title: GenoMetric Query Language for R/Bioconductor

-Version: 0.99.24

+Version: 0.99.25

 Author: Simone Pallotta, Marco Masseroli

 Maintainer: Simone Pallotta <simonepallotta@hotmail.com>

-Description: This RGMQL package brings the GenoMetric Query Language (GMQL) 

-  functionalities into the R environment.

+Description: This RGMQL package brings the GenoMetric Query Language (GMQL) functionalities into the R environment.

   GMQL is a high-level, declarative language to query and compare multiple and heterogeneous genomic 

   datasets for biomedical knowledge discovery. It allows expressing easily queries 

   and processing over genomic regions and their metadata, in a way similar to 

@@ -64,7 +64,7 @@ extend <-function(input_data, metadata = NULL)

                                     dispatch = TRUE)

     else

         metadata_matrix <- .jnull("java/lang/String")

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response <- WrappeR$extend(metadata_matrix,input_data$value)

     error <- strtoi(response[1])

@@ -47,7 +47,7 @@

 #' @export

 #'

 filter_and_extract <- function(data, metadata = NULL, 

-                               metadata_prefix = NULL, regions = NULL)

+                                metadata_prefix = NULL, regions = NULL)

 {

     if(is(data,"GRangesList"))

         .extract_from_GRangesList(data,metadata,metadata_prefix,regions)

@@ -38,7 +38,7 @@ import_gmql <- function(dataset_path, is_gtf)

     datasetName <- sub("/*[/]$","",datasetName)

     if(basename(datasetName) !="files")

         datasetName <- paste0(datasetName,"/files")

-  

+    

     if(!dir.exists(datasetName))

         stop("Directory does not exists")

@@ -101,7 +101,7 @@ join <- function(right_input_data, left_input_data,

                 warning("only 4 element per list, we cut the rest")

                 length(list_pred)=4

             }

-      

+            

             if(!all(sapply(list_pred, function(x) {is(x,"DISTAL")} )))

                 stop("All elements should be DISTAL object")

             })

@@ -120,18 +120,18 @@ join <- function(right_input_data, left_input_data,

     }

     else

         genomatrix <- .jnull("java/lang/String")

-      

+    

     if(!is.null(joinBy))

-        join_condition_matrix <- .jarray(.join_condition(joinBy),

+        join_condition_matrix <- .jarray(.join_condition(joinBy), 

                                             dispatch = TRUE)

     else

         join_condition_matrix <- .jnull("java/lang/String")

-  

+    

     ouput <- toupper(region_output)

     if(!identical(ouput,"CONTIG") && !identical(ouput,"LEFT") && 

         !identical(ouput,"RIGHT") && !identical(ouput,"INT"))

         stop("region_output must be contig,left,right or int (intersection)")

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response <- WrappeR$join(genomatrix,join_condition_matrix, 

                                 ouput,right_input_data$value,

@@ -88,13 +88,13 @@ map <- function(left_input_data, right_input_data, aggregates = NULL,

                                     dispatch = TRUE)

     else

         metadata_matrix = .jnull("java/lang/String")

-

+    

     if(!is.null(joinBy))

         join_condition_matrix <- .jarray(.join_condition(joinBy),

                                             dispatch = TRUE)

     else

         join_condition_matrix <- .jnull("java/lang/String")

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response<-WrappeR$map(join_condition_matrix, metadata_matrix, 

                             left_input_data$value, right_input_data$value)

@@ -28,7 +28,7 @@ execute <- function()

     remote_proc <- WrappeR$is_remote_processing()

     if(!remote_proc)

         .download_or_upload()

-  

+    

     response <- WrappeR$execute()

     error <- strtoi(response[1])

     data <- response[2]

@@ -143,14 +143,14 @@ take <- function(input_data, rows=0L)

     rows <- as.integer(rows[1])

     if(rows<0)

         stop("rows cannot be negative")

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response <- WrappeR$take(input_data$value,rows)

     error <- strtoi(response[1])

     data <- response[2]

     if(error!=0)

         stop(data)

-

+    

     reg <- .jevalArray(WrappeR$get_reg(),simplify = TRUE)

     if(is.null(reg))

         stop("no regions defined")

@@ -160,11 +160,11 @@ take <- function(input_data, rows=0L)

     schema <- .jevalArray(WrappeR$get_schema(),simplify = TRUE)

     if(is.null(schema))

         stop("no schema defined")

-

+    

     reg_data_frame <- as.data.frame(reg)

     list <- split(reg_data_frame, reg_data_frame[1])

     names <- c("seqname","start","end","strand",schema)

-  

+    

     sampleList <- lapply(list, function(x){

         x <- x[-1]

         names(x) <- names

@@ -175,7 +175,7 @@ take <- function(input_data, rows=0L)

         })

     gRange_list <- GRangesList(sampleList)

     meta_list <- .metadata_from_frame_to_list(meta)

-  

+    

     S4Vectors::metadata(gRange_list) <- meta_list

     return(gRange_list)

 }

@@ -57,7 +57,7 @@ merge <- function(input_data, groupBy = NULL)

                                             dispatch = TRUE)

     else

         join_condition_matrix <- .jnull("java/lang/String")

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response <- WrappeR$merge(join_condition_matrix,input_data$value)

     error <- strtoi(response[1])

@@ -70,22 +70,22 @@ order <- function(input_data, metadata_ordering = NULL, mtop = 0, mtopg = 0,

     if(!is.numeric(mtop) || !is.numeric(mtopg) || !is.numeric(rtop) || 

         !is.numeric(rtopg) || !is.numeric(mtopp)|| !is.numeric(rtopp))

         stop("mtop, rtop, rtopg and mtopg must be integer")

-

+    

     if(length(mtop)>1 || length(mtopg)>1 || length(rtop)>1 || length(rtopg)>1

         || length(mtopp)>1 || length(rtopp)>1)

         warning("only first element: rtop, mtop, mtopg, rtopg, rtopp, mtopp")

-

+    

     # we consider only the first element even if input is a vector of Int

     # we cut the other arguments

-

+    

     mtop = as.integer(mtop[1])

     mtopg = as.integer(mtopg[1])

     mtopp = as.integer(mtopp[1])

-

+    

     rtop = as.integer(rtop[1])

     rtopg = as.integer(rtopg[1])

     rtopp = as.integer(rtopp[1])

-

+    

     if(mtop > 0 && mtopg >0)

     {

         warning("cannot be used together.\nWe set mtopg = 0")

@@ -97,43 +97,43 @@ order <- function(input_data, metadata_ordering = NULL, mtop = 0, mtopg = 0,

         warning("cannot be used together.\nWe set mtopp = 0")

         mtopp = 0L

     }

-

+    

     if(mtopg >0 && mtopp>0)

     {

         warning("cannot be used together.\nWe set mtopp = 0")

         mtopp = 0L

     }

-

+    

     if(rtop > 0 && rtopg >0)

     {

         warning("cannot be used together.\nWe set rtopg = 0")

         rtopg = 0L

     }

-

+    

     if(rtop >0 && rtopp>0)

     {

         warning("cannot be used together.\nWe set rtopp = 0")

         rtopp = 0L

     }

-

+    

     if(rtopg >0 && rtopp>0)

     {

         warning("cannot be used together.\nWe set rtopp = 0")

         rtopp = 0L

     }

-

+    

     if(!is.null(metadata_ordering))

         meta_matrix <- .jarray(.ordering_meta(metadata_ordering),

                                     dispatch = TRUE)

     else

         meta_matrix <- .jnull("java/lang/String")

-

+    

     if(!is.null(regions_ordering))

         region_matrix <- .jarray(.ordering_meta(regions_ordering),

                                     dispatch = TRUE)

     else

         region_matrix <- .jnull("java/lang/String")

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response <- WrappeR$order(meta_matrix, mtopg, mtop, mtopp, region_matrix,

                                 rtopg, rtop, rtopp, input_data$value)

@@ -169,6 +169,7 @@ order <- function(input_data, metadata_ordering = NULL, mtop = 0, mtopg = 0,

     }

     else

         stop("only list or character")

+    

 }

@@ -95,34 +95,34 @@ project <-function(input_data, metadata = NULL, metadata_update=NULL,

     {

         if(!is.character(metadata))

             stop("metadata: no valid input")

-

+        

         metadata <- metadata[!metadata %in% ""]

         metadata <- metadata[!duplicated(metadata)]

-

+        

         if(length(metadata)==0)

             metadata <- .jnull("java/lang/String")

-    

+        

         metadata <- .jarray(metadata)

     }

     else

         metadata <- .jnull("java/lang/String")

-

+    

     if(!is.null(regions))

     {

         if(!is.character(regions))

             stop("regions: no valid input")

-

+        

         regions = regions[!regions %in% ""]

         regions = regions[!duplicated(regions)]

-

+        

         if(length(regions)==0)

             regions <- .jnull("java/lang/String")

-    

+        

         regions <- .jarray(regions)

     }

     else

         regions <- .jnull("java/lang/String")

-

+    

     reg_update <- substitute(regions_update)

     if(!is.null(reg_update))

     {

@@ -131,7 +131,7 @@ project <-function(input_data, metadata = NULL, metadata_update=NULL,

     }

     else

         regions_update <- .jnull("java/lang/String")

-  

+    

     meta_update <- substitute(metadata_update)

     if(!is.null(meta_update))

     {

@@ -140,15 +140,15 @@ project <-function(input_data, metadata = NULL, metadata_update=NULL,

     }

     else

         metadata_update <- .jnull("java/lang/String")

-  

+    

     if(length(all_but_meta)>1)

         warning("all_but_meta: no multiple values")

-  

+    

     if(length(all_but_reg)>1)

         warning("all_but_reg: no multiple values")

     all_but_reg <- all_but_reg[1]

     all_but_meta <- all_but_meta[1]

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response <- WrappeR$project(metadata,metadata_update,all_but_meta,

                                 regions,regions_update,

@@ -52,12 +52,12 @@ init_gmql <- function(output_format = "gtf", remote_processing = FALSE,

         !identical(out_format,"COLLECT"))

         stop("output_format must be TAB, GTF or COLLECT")

     .check_logical(remote_processing)

-  

+    

     # mettere attesa da input keyboard, controllare se token già esiste 

     # da sessione precedente

     if(!is.null(url) && !exists("authToken",envir = .GlobalEnv))

         login_gmql(url,username,password)

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     WrappeR$initGMQL(out_format,remote_processing)

 }

@@ -132,7 +132,7 @@ read_dataset <- function(dataset, parser = "CustomParser", is_local=TRUE,

     {

         if(!dir.exists(dataset))

             stop("folder does not exist")

-    

+        

         dataset <- sub("/*[/]$","",dataset)

         if(basename(dataset) !="files")

             dataset <- paste0(dataset,"/files")

@@ -145,16 +145,16 @@ read_dataset <- function(dataset, parser = "CustomParser", is_local=TRUE,

         url <- WrappeR$get_url()

         if(is.null(url))

             stop("You have to log on using login function")

-    

+        

         if(!exists("authToken",envir = .GlobalEnv))

             stop("You have to log on using login function")

-   

+        

         list <- show_schema(url,dataset)

         schema_names <- sapply(list$fields, function(x){x$name})

         schema_type <- sapply(list$fields, function(x){x$type})

         schema_matrix <- cbind(schema_type,schema_names)

         #schema_type <- list$type

-    

+        

         if(is.null(schema_matrix) || length(schema_matrix)==0)

             schema_matrix <- .jnull("java/lang/String")

         else

@@ -203,7 +203,7 @@ read <- function(samples)

 {

     if(!is(samples,"GRangesList"))

         stop("only GrangesList")

-

+    

     meta <- S4Vectors::metadata(samples)

     if(is.null(meta) || length(meta)==0) {

         #repeat meta for each sample in samples list

@@ -212,17 +212,19 @@ read <- function(samples)

 We provide two metadata for you")

         index_meta <- rep(1:len,each = len)

         rep_meta <- rep(c("Provider","Polimi", "Application", "R-GMQL"),

-                        times=len)

+                            times=len)

         meta_matrix <- matrix(rep_meta,ncol = 2,byrow = TRUE)

         meta_matrix <- cbind(index_meta,meta_matrix)

     }

-    else {

+    else 

+    {

         unlist_meta <- unlist(meta)

         names_meta <- names(unlist_meta)

         group_names <- gsub(".*_([0-9]*)\\..*","\\1", names_meta)

         names(unlist_meta) <- NULL

         meta_matrix <- cbind(group_names,names_meta,unlist_meta)

     }

+    

     df <- data.frame(samples)

     df <- df[-2] #delete group_name

     region_matrix <- as.matrix(sapply(df, as.character))

@@ -249,11 +251,11 @@ We provide two metadata for you")

     }

     rownames(schema_matrix) <- NULL

     colnames(schema_matrix) <- NULL

-  

+    

     schema_matrix <- .jarray(schema_matrix,dispatch = TRUE)

     meta_matrix <- .jarray(meta_matrix,dispatch = TRUE)

     region_matrix <- .jarray(region_matrix,dispatch = TRUE)

-  

+    

     WrappeR <- J("it/polimi/genomics/r/Wrapper")

     response <- WrappeR$read(meta_matrix,region_matrix,schema_matrix)

     DataSet(response)

@@ -98,7 +98,7 @@ select <- function(input_data, predicate = NULL, region_predicate = NULL,

     }

     else

         predicate <- .jnull("java/lang/String")

-

+    

     reg_pred <- substitute(region_predicate)

     if(!is.null(reg_pred))

     {

@@ -107,7 +107,7 @@ select <- function(input_data, predicate = NULL, region_predicate = NULL,

     }

     else

         region_predicate <- .jnull("java/lang/String")

-

+    

     if(is.null(semi_join) && is.null(semi_join_dataset))

     {

         join_condition_matrix <- .jnull("java/lang/String")

@@ -22,7 +22,7 @@

                                 full.names = TRUE)

     if(length(schema_name)==0)

         stop("schema not present")

-

+    

     xml_schema <- xml2::read_xml(schema_name)

     list_field <- xml2::as_list(xml_schema)

     vector_field <- unlist(list_field)

@@ -33,10 +33,10 @@

 {

     if(!is.list(meta_data))

         stop("meta_data: invalid input")

-

+    

     if(!all(sapply(meta_data, function(x) is(x,class))))

         stop("All elements must be META_AGGREGATES object")

-

+    

     names <- names(meta_data)

     if(is.null(names))

     {

@@ -46,12 +46,13 @@

     else

     {

         if("" %in% names)

-        stop("No partial names assignment is allowed")

+            stop("No partial names assignment is allowed")

     }

     aggregate_matrix <- t(sapply(meta_data, function(x) {

         new_value = as.character(x)

         matrix <- matrix(new_value)

     }))

+    

     m_names <- matrix(names)

     metadata_matrix <- cbind(m_names,aggregate_matrix)

 }

@@ -94,7 +95,7 @@

 {

     if(!is.character(value))

         stop("no valid data")

-  

+    

     if(length(value)>1)

         stop("no multiple string")

 }

@@ -103,7 +104,7 @@

 {

     if(!is.logical(value))

         stop("no valid data")

-  

+    

     if(length(value)>1)

         stop("no multiple string")

 }

@@ -5,7 +5,6 @@

     .jpackage(pkgname, lib.loc = libname)

     # tools::vignetteEngine("knitr", pattern = "[.]Rmd$", package = "knitr")

     .jinit(force.init = TRUE)

- 

 }

 .onAttach <- function(libname, pkgname) {

@@ -17,7 +17,7 @@ check.OPERATOR <- function(value)

     {

         if(is.character(value) && length(value)>1)

             stop("value: no multiple string")

-    

+        

         if(!is.character(value))

             stop("value: is not a string")

     }

@@ -5,9 +5,7 @@

 ORDER <- function(value)

 {

-  op_list <- list(

-    value = value

-  )

+  op_list <- list(value = value)

   ## Set the name for the class

   class(op_list) <- "ORDER"

   return(op_list)

@@ -17,7 +15,7 @@ check.ORDER <- function(value)

 {

   if(is.character(value) && length(value)>1)

     stop("value: no multiple string")

-

+    

   if(!is.character(value))

     stop("value: is not a string")

 }

@@ -26,8 +24,7 @@ print.ORDER <- function(obj) {

   as.character(obj)

 }

-c.ORDER <- function(...)

-{

+c.ORDER <- function(...) {

   a <- list(...)

 }

@@ -63,7 +60,7 @@ as.character.ORDER <- function(obj) {

 DESC <- function(value)

 {

     check.ORDER(value)

-

+    

     list <- list(value = value)

     ## Set the name for the class

     class(list) <- c("DESC","ORDER")

@@ -98,7 +95,7 @@ DESC <- function(value)

 ASC <- function(value)

 {

     check.ORDER(value)

-

+    

     list <- list(value = value)

     ## Set the name for the class

     class(list) <- c("ASC","ORDER")

@@ -201,11 +201,11 @@ run_query <- function(url, fileName, query, output_gtf = TRUE)

         out <- "GTF"

     else

         out <- "TAB"

-

+    

     URL <- paste0(url,"/queries/run/",fileName,"/",out)

     h <- c('Accept' = "Application/json",

             'Content-Type' = 'text/plain','X-Auth-Token' = authToken)

-

+    

     req <- httr::POST(URL,body = query ,httr::add_headers(h),encode = "json")

     content <- httr::content(req,"parsed")

     if(req$status_code !=200)

@@ -257,7 +257,7 @@ run_query_fromfile <- function(url, fileName, filePath, output_gtf = TRUE)

 {

     if(!file.exists(filePath))

         stop("file does not exist")

-

+    

     query <- readLines(filePath)

     run_query(url,fileName,query,output_gtf)

 }

@@ -325,7 +325,7 @@ compile_query_fromfile <- function(url ,filePath)

 {

     if(!file.exists(filePath))

         stop("file does not exist")

-

+    

     query <- readLines(filePath)

     compile_query(url,query)

 }

@@ -338,11 +338,11 @@ serialize_query <- function(url,output_gtf,base64)

         out <- "gtf"

     else

         out <- "tab"

-  

+    

     URL <- paste0(url,"/queries/dag/",out)

     h <- c('Accept' = "Application/json",

             'Content-Type' = 'text/plain','X-Auth-Token' = authToken)

-  

+    

     req <- httr::POST(URL,body = base64 ,httr::add_headers(h),encode = "json")

     content <- httr::content(req,"parsed")

     if(req$status_code !=200)

@@ -15,73 +15,131 @@ link-citations: true

 # Introduction

-Recent years have seen a tremendous increase in the volume of data generated in the life sciences, especially propelled by the rapid progress of Next Generation Sequencing (NGS) technologies. 

-This high-throughput technologies can produce billions of short DNA or RNA fragments in excess of a few terabytes of data in a single run.

-Next-generation sequencing refers to the deep, in-parallel DNA sequencing technologies providing massively parallel analysis and extremely high-throughput from multiple samples at much reduced cost. Improvement of sequencing technologies and data processing pipelines is rapidly providing sequencing data, with associated high-level features, of many individual genomes in multiple biological and clinical conditions. 

-To make effective use of the produced data, the design of big data algorithms and their efficient implementation on modern high performance computing infrastructures,

-such as clouds, CPU clusters and network infrastructures, is required in order to achieve scalability and performance. 

-For this purpose the GenoMetric Query Language (GMQL) has been proposed as high-level, declarative language to process, query, and compare multiple and heterogeneous genomic datasets for biomedical knowledge discovery [@Bioinformatics2015]

+Recent years have seen a tremendous increase in the volume of data generated 

+in the life sciences, especially propelled by the rapid progress of 

+Next Generation Sequencing (NGS) technologies. 

+This high-throughput technologies can produce billions of short DNA or RNA 

+fragments in excess of a few terabytes of data in a single run.

+Next-generation sequencing refers to the deep, in-parallel DNA sequencing 

+technologies providing massively parallel analysis and extremely 

+high-throughput from multiple samples at much reduced cost. 

+Improvement of sequencing technologies and data processing pipelines 

+is rapidly providing sequencing data, with associated high-level features, 

+of many individual genomes in multiple biological and clinical conditions. 

+To make effective use of the produced data, the design of big data algorithms 

+and their efficient implementation on modern high performance 

+computing infrastructures, such as clouds, CPU clusters 

+and network infrastructures, is required in order to achieve scalability 

+and performance. 

+For this purpose the GenoMetric Query Language (GMQL) has been proposed 

+as high-level, declarative language to process, query, 

+and compare multiple and heterogeneous genomic datasets for biomedical 

+knowledge discovery [@Bioinformatics2015]

 ## Purpose

-A very important emerging problem is to make sense of the enormous amount and variety of NGS data becoming available, i.e. to discover how different genomic regions and their products interact and cooperate with each other. 

-To this aim, the integration of several heterogeneous DNA feature data is required.

-Such big genomic feature data are collected within numerous and heterogeneous files, usually distributed within different repositories, lacking an attribute-based organization and a systematic description of their metadata. 

-These heterogeneous data can contain the hidden answer to very important biomedical questions.

-To inveil them, standard tools already available for knowledge extraction are too specialized or present powerful features, but have a rough interface not well-suited for scientists/biologists.

-GMQL addresses these aspects using cloud-based technologies (including Apache Hadoop, mapReduce, and Spark), and focusing on genomic data operations written as simple queries with implicit iterations over thousands of heterogeneous samples, computed efficiently [@IEEE7484654].

-This RGMQL package makes easy to take advantage of GMQL functionalities also to scientists and biologists with limited knowledge of query and programming languages, but used to the R/Bioconductor environment. 

-This package is built over a GMQL scalable data management engine written in Scala programming language, released as Scala API [@githubrepo] providing a set of functions to combine, manipulate, compare, and extract genomic data from different datasources both from local and remote datasets.

-These functions allow performing complex GMQL processing and queries without knowledge of GMQL syntax, but leveraging on R idiomatic paradigm and logic.

+A very important emerging problem is to make sense of the enormous amount and 

+variety of NGS data becoming available, i.e. to discover how different genomic 

+regions and their products interact and cooperate with each other. 

+To this aim, the integration of several heterogeneous DNA feature data 

+is required.

+Such big genomic feature data are collected within numerous and 

+heterogeneous files, usually distributed within different repositories, 

+lacking an attribute-based organization and a systematic description 

+of their metadata. 

+These heterogeneous data can contain the hidden answer to very important 

+biomedical questions.

+To inveil them, standard tools already available for knowledge extraction 

+are too specialized or present powerful features, but have a rough interface 

+not well-suited for scientists/biologists.

+GMQL addresses these aspects using cloud-based technologies 

+(including Apache Hadoop, mapReduce, and Spark), and focusing on genomic data 

+operations written as simple queries with implicit iterations over thousands 

+of heterogeneous samples, computed efficiently [@IEEE7484654].

+This RGMQL package makes easy to take advantage of GMQL functionalities also 

+to scientists and biologists with limited knowledge of query and 

+programming languages, but used to the R/Bioconductor environment. 

+This package is built over a GMQL scalable data management engine 

+written in Scala programming language, released as Scala API [@githubrepo] 

+providing a set of functions to combine, manipulate, compare, and extract 

+genomic data from different datasources both from local and remote datasets.

+These functions allow performing complex GMQL processing and queries without 

+knowledge of GMQL syntax, but leveraging on R idiomatic paradigm and logic.

 # Genomic Data Model

-The Genomic Data Model (GDM) is based on the notions of datasets and samples[@modeling2016] 

-Datasets are collections of samples, and each sample consists of two parts, the region data, which describe portions of the genome, and the metadata, which describe sample general properties and how observations are collected.

-In contrast to other data models, it clearly divides, and comprehensively manages, observations about genomic regions and metadata.

-GDM provides a flat attribute based organization, just requiring that each dataset is associated with a given data schema, which specifies the attributes and their type of region data.

-The first attributes of such schema are fixed (chr, start, end, strand); they represent the genomic region identifying coordinates.

+The Genomic Data Model (GDM) is based on the notions of datasets 

+and samples[@modeling2016] 

+Datasets are collections of samples, and each sample consists of two parts, 

+the region data, which describe portions of the genome, and the metadata, 

+which describe sample general properties and how observations are collected.

+In contrast to other data models, it clearly divides, and comprehensively 

+manages, observations about genomic regions and metadata.

+GDM provides a flat attribute based organization, just requiring that 

+each dataset is associated with a given data schema, which specifies 

+the attributes and their type of region data.

+The first attributes of such schema are fixed (chr, start, end, strand); 

+they represent the genomic region identifying coordinates.

 In addition, metadata have free attribute-value pair format.

 ## Genomic Region 

-Genomic region data describe a broad variety of biomolecular aspects and are very valuable for biomolecular investigation.

-A genomic region is a portion of a genome, qualified by a quadruple of values called region coordinates:

+Genomic region data describe a broad variety of biomolecular aspects and are 

+very valuable for biomolecular investigation.

+A genomic region is a portion of a genome, qualified by a quadruple of values 

+called region coordinates:

 $$< chr, left, right, strand >$$

-Regions can have an arbitrary number of associated values, according to the processing of DNA, RNA or epigenomic sequencing reads that determined the region.

+Regions can have an arbitrary number of associated values, according to 

+the processing of DNA, RNA or epigenomic sequencing reads that determined 

+the region.

 ## Metadata

-Metadata describe the biological and clinical properties associated with each sample.

-They are usually collected in a broad variety of data structures and formats that constitute barriers to their use and comparison GDM models metadata simply as arbitrary semi-structured attribute-value pairs, where attributes may have multiple values.

+Metadata describe the biological and clinical properties associated with 

+each sample.

+They are usually collected in a broad variety of data structures and formats 

+that constitute barriers to their use and comparison GDM models metadata 

+simply as arbitrary semi-structured attribute-value pairs, 

+where attributes may have multiple values.

 ## Genomic Sample

-Formally, a sample s is a collection of genomic regions modeled as the following triple: $$< id, {< r_i,v_i >}, {m_j} >$$ where:

+Formally, a sample s is a collection of genomic regions modeled as 

+the following triple: $$< id, {< r_i,v_i >}, {m_j} >$$ where:

 * id is the sample identifier

 * Each region is a pair of coordinates $r_i$ and values $v_i$

 * Metadata $m_j$ are attribute-value pairs $< a_j,v_j >$

-Note that the sample id attribute provides a many-to-many connection between regions and metadata of a sample.

-Through the use of a data type system to express region data, and of arbitrary attribute-value pairs for metadata, GDM provides interoperability across datasets in multiple formats produced by different experimental techniques.

+Note that the sample id attribute provides a many-to-many connection between 

+regions and metadata of a sample.

+Through the use of a data type system to express region data, and of arbitrary 

+attribute-value pairs for metadata, GDM provides interoperability across 

+datasets in multiple formats produced by different experimental techniques.

 ## Dataset

-A dataset is a collection of samples uniquely identified, with the same region schema and with each sample consisting of two parts:

+A dataset is a collection of samples uniquely identified, with the same region 

+schema and with each sample consisting of two parts:

 * region data: describing characteristics and location of genomic portions

 * metadata: expressing general properties of the sample

-Each dataset is typically produced within the same project by using the same or equivalent technology and tools, but with different experimental conditions, described by metadata.

+Each dataset is typically produced within the same project by using the same 

+or equivalent technology and tools, but with different experimental 

+conditions, described by metadata.

-Datasets contain large number of information describing regions of a genome, with data encoded in human readable format using plain text files.

+Datasets contain large number of information describing regions of a genome, 

+with data encoded in human readable format using plain text files.

-GMQL datasets are materialized in a standard layout composed of three types of files:

+GMQL datasets are materialized in a standard layout composed of three 

+types of files:

-1. genomic region tab-delimited text files with extension .gdm, or .gtf if in standard GTF format

-2. metadata attribute-value tab-delimited text files with the same fullname (name and extension) of the correspondent region file and extension .meta

+1. genomic region tab-delimited text files with extension .gdm, or .gtf 

+if in standard GTF format

+2. metadata attribute-value tab-delimited text files with the same fullname 

+(name and extension) of the correspondent region file and extension .meta

 3. schema XML file containing region attribute names and types

 All these files reside in unique folder called files.

@@ -89,14 +147,22 @@ All these files reside in unique folder called files.

 <!-- ![GMQL dataset folder](dataset_gmql.png) -->

 In RGMQL package dataset files are considered read-only.

-Once read, genomic information is represented in abstract structure inside the package, mapped to a R GRanges data structure at occurency.

+Once read, genomic information is represented in abstract structure inside 

+the package, mapped to a R GRanges data structure at occurency.

 # GenoMetric Query Language

-The GenoMetric Query Language name stems from the language ability to deal with genomic distances, which are measured as number of nucleotide bases between genomic regions (aligned to the same reference genome) and computed using arithmetic operations between region coordinates.

-GMQL is a high-level, declarative language that allows expressing queries easily over genomic regions and their metadata, in a way similar to what can be done with the Structured Query Language (SQL) over a relational database.

-GMQL approach exhibits two main differences with respect to other tools based on Hadoop, mapReduce framework, and Spark engine technologies to address similar biomedical problems:\newline

+The GenoMetric Query Language name stems from the language ability to deal 

+with genomic distances, which are measured as number of nucleotide bases 

+between genomic regions (aligned to the same reference genome) and computed 

+using arithmetic operations between region coordinates.

+GMQL is a high-level, declarative language that allows expressing queries 

+easily over genomic regions and their metadata, in a way similar to what can 

+be done with the Structured Query Language (SQL) over a relational database.

+GMQL approach exhibits two main differences with respect to other tools 

+based on Hadoop, mapReduce framework, and Spark engine technologies 

+to address similar biomedical problems:\newline

 * GMQL:

@@ -108,29 +174,43 @@ GMQL approach exhibits two main differences with respect to other tools based on

     1. read generally from raw or alligned data from NGS machines

     2. provide no support for metadata management

-GMQL is the appropriate tool for querying numerous processed genomic datasets and very many samples that are becoming available.

-Note however that GMQL performs worse than some other available systems on a small number of small-scale datasets, but these other systems are not cloud-based; hence, they are not adequate for efficient big data processing and, in some cases, they are inherently limited in their data management capacity, as they only work as RAM memory resident processes.

+GMQL is the appropriate tool for querying numerous processed genomic datasets 

+and very many samples that are becoming available.

+Note however that GMQL performs worse than some other available systems on a 

+small number of small-scale datasets, but these other systems are not 

+cloud-based; hence, they are not adequate for efficient big data processing 

+and, in some cases, they are inherently limited in their 

+data management capacity, as they only work as RAM memory resident processes.

 ## Query structure

-A GMQL operation is expressed as a sequence of GMQL operations with the following structure:

+A GMQL operation is expressed as a sequence of GMQL operations with the 

+following structure:

 $$< variable > = operator(< parameters >) < variable >;$$

 where each $< variable >$ stands for a GDM dataset

-This RGMQL package brings GMQL functionalities into R environemnt, allowing users to build directly a GMQL query without knowing the GMQL syntax.

-In RGMQL every GMQL operations is translated into a R function and expressed as:

+This RGMQL package brings GMQL functionalities into R environemnt, 

+allowing users to build directly a GMQL query without knowing the GMQL syntax.

+In RGMQL every GMQL operations is translated into a R function 

+and expressed as:

 $$ variable = operator(variable, parameters)$$

-It is very similar to the GMQL syntax for operation expression although expressed with the R idiomatic paradigm and logic, with parameters totaly builded using R native data structures such as lists, matrices, vectors or R logic conditions.

+It is very similar to the GMQL syntax for operation expression although 

+expressed with the R idiomatic paradigm and logic, with parameters totaly 

+builded using R native data structures such as lists, matrices, 

+vectors or R logic conditions.

 # Processing Environments

-In this section, we show how GMQL processing is built in R, which operations are available in RGMQL, and the difference beetween local and remote dataset processing.

+In this section, we show how GMQL processing is built in R, which operations 

+are available in RGMQL, and the difference beetween local 

+and remote dataset processing.

 ## Local Processing

-RGMQL local processing consumes computational power directly from local CPUs/system while managing datasets (both GMQL or generic text plain datasets).

+RGMQL local processing consumes computational power directly from local 

+CPUs/system while managing datasets (both GMQL or generic text plain datasets).

 ### Initialization

@@ -138,22 +218,27 @@ Load and attach the GMQL package in a R session using library function:

 ```{r, initialization, eval = TRUE}

 library('RGMQL')

 ```

-Before starting using any GMQL operation we need to initialise the GMQL context with the following code:

+Before starting using any GMQL operation we need to initialise the GMQL 

+context with the following code:

 ```{r, init, eval = TRUE}

 init_gmql()

 ```

-The function *init_gmql()* initializes the context of scalable data management engine laid upon Spark and Hadoop.

-Details on this and all other functions are provided in the R documentation for this package (e.g., help(RGMQL)).

+The function *init_gmql()* initializes the context of scalable data management 

+engine laid upon Spark and Hadoop.

+Details on this and all other functions are provided in the R documentation 

+for this package (e.g., help(RGMQL)).

 ### Read Dataset

 After initialization we need to read datasets.

-We already defined above the formal definition of dataset and the power of GMQL to deal with data in a variety of standard tab-delimited text formats.

+We already defined above the formal definition of dataset and the power of 

+GMQL to deal with data in a variety of standard tab-delimited text formats.

 In the following, we show how to get data from different sources.\newline

 We distinguish two different cases:

 1. Local dataset:\newline

-A local dataset is a folder with sample files (region files and correspondent metadata files) on the user computer.

+A local dataset is a folder with sample files (region files and correspondent 

+metadata files) on the user computer.

 As data are already in the user computer, we simply execute:

 ```{r, read GMQL dataset, eval = TRUE}

@@ -161,10 +246,13 @@ gmql_dataset_path <- system.file("example", "EXON", package = "RGMQL")

 data_out = read_dataset(gmql_dataset_path)

 ```

 In this case we are reading a dataset named EXON specified by path.

-It doens't matter what kind of format the data are, *read_dataset()* read many standard tab-delimited text formats without specified any paramter at input.

+It doens't matter what kind of format the data are, *read_dataset()* read many 

+standard tab-delimited text formats without specified any paramter at input.

 2. GRangesList:\newline

-For better integration in the R environment and with other packages, we provide a *read()* function to read directly from R memory/environment using GRangesList as input.

+For better integration in the R environment and with other packages, 

+we provide a *read()* function to read directly from R memory/environment 

+using GRangesList as input.

 ```{r, read GRangesList, eval = TRUE}

 library("GenomicRanges")

@@ -182,19 +270,27 @@ grl <- GRangesList("txA" = gr1, "txB" = gr2)

 data_out <- read(grl)