Benchmarking CMEnt on Repeated Simulated Microarray Data
CMEnt vs comb-p, probeLasso, DMRcate, and Bumphunter across repeated simulateDMRs runs
2026-10-19
Contents
0.1 Introduction
This vignette repeats the simulated 450K benchmark over multiple independently
simulated datasets. Each dataset is produced from the same GSE40279 chr20/chr21
beta-value subset and 100-sample covariate subset used by the single-run
benchmark, with CMEnt::simulateDMRs() parameters drawn from fixed ranges under
a master seed of 42. Method outputs and simulation bundles are cached per
iteration, and summary plots show medians with observed interval ranges across
simulations.
Methods:
- CMEnt: correlation-based seed-guided assembly approach
- comb-p: adjacent p-value combination
- probeLasso: lasso-based feature selection for DMR detection
- DMRcate: Gaussian kernel smoothing approach
- Bumphunter: permutation-based regional testing
0.2 Setup and Dependencies
.loadCMEnt <- function() {
root <- if (file.exists("NAMESPACE")) "." else if (file.exists(file.path("..", "NAMESPACE"))) ".." else NULL
if (!is.null(root) && requireNamespace("pkgload", quietly = TRUE)) {
ok <- tryCatch({
pkgload::load_all(root, quiet = TRUE)
TRUE
}, error = function(e) {
message("Could not load local CMEnt source: ", conditionMessage(e))
FALSE
})
if (ok) return(invisible(TRUE))
}
suppressPackageStartupMessages(library(CMEnt))
}
.loadCMEnt()
suppressPackageStartupMessages({
library(data.table)
library(dplyr)
library(GenomicRanges)
library(ggplot2)
})
getBenchmarkFile <- function(filename) {
folder <- "simulated_microarray450k_multisim"
benchmark_file <- if (getwd() == "notebooks") {
file.path("cached_results", folder, filename)
} else if ("NAMESPACE" %in% dir()) {
file.path("notebooks", "cached_results", folder, filename)
} else {
file.path("cached_results", folder, filename)
}
dir.create(dirname(benchmark_file), showWarnings = FALSE, recursive = TRUE)
benchmark_file
}
getSimulationFile <- function(sim_id, filename) {
getBenchmarkFile(file.path(sprintf("sim_%02d", sim_id), filename))
}
read_cache <- function(path, params = NULL) {
if (!file.exists(path)) return(NULL)
out <- readRDS(path)
if (is.null(params) || identical(out$params, params)) out else NULL
}
save_cache <- function(path, value, params = NULL) {
saveRDS(list(value = value, params = params), path)
}
nsim <- suppressWarnings(as.integer(Sys.getenv("CMENT_BENCHMARK_NSIM", "10")))
nsim <- ifelse(is.na(nsim) || nsim < 1L, 10L, nsim)
plan_file <- getBenchmarkFile("simulation_parameters.tsv")
if (file.exists(plan_file)) {
simulation_plan <- data.table::fread(plan_file)
} else {
set.seed(42)
simulation_plan <- data.frame(
simulation_id = seq_len(nsim),
seed = sample(0:1000, nsim),
delta_beta_max = round(runif(nsim, 0.2, 0.5), 2),
expected_corr = round(runif(nsim, 0.3, 0.7), 2)
)
data.table::fwrite(simulation_plan, plan_file, sep = "\t")
}
NJOBS <- as.integer(
Sys.getenv(
"NJOBS",
unset = as.character(max(1L, min(8L, parallel::detectCores(logical = TRUE) - 1L)))
)
)
doParallel::registerDoParallel(cores = NJOBS)
knitr::kable(simulation_plan, caption = "Simulation parameters generated under master seed 42")| simulation_id | seed | delta_beta_max | expected_corr |
|---|---|---|---|
| 1 | 560 | 0.34 | 0.66 |
| 2 | 996 | 0.42 | 0.36 |
| 3 | 320 | 0.48 | 0.70 |
| 4 | 152 | 0.28 | 0.68 |
| 5 | 73 | 0.34 | 0.33 |
| 6 | 227 | 0.48 | 0.51 |
| 7 | 145 | 0.49 | 0.46 |
| 8 | 633 | 0.24 | 0.66 |
| 9 | 48 | 0.34 | 0.48 |
| 10 | 127 | 0.37 | 0.63 |
0.2.1 Install External Dependencies
r <- getOption("repos")
if (is.null(r) || is.na(r["CRAN"]) || r["CRAN"] %in% c("@CRAN@", "")) {
options(repos = c(CRAN = "https://cloud.r-project.org"))
}
pkg_installed <- function(pkg) pkg %in% rownames(utils::installed.packages())
if (!pkg_installed("benchmarkme")) install.packages("benchmarkme")
if (!pkg_installed("BiocManager")) install.packages("BiocManager")
if (!pkg_installed("DMRcate")) BiocManager::install("DMRcate", update = FALSE, ask = FALSE)
if (!pkg_installed("bumphunter")) BiocManager::install("bumphunter", update = FALSE, ask = FALSE)
if (!pkg_installed("doParallel")) install.packages("doParallel")
if (!pkg_installed("tidyr")) install.packages("tidyr")
if (!pkg_installed("callr")) install.packages("callr")
if (!pkg_installed("ChAMP")) BiocManager::install("ChAMP", update = FALSE, ask = FALSE)
anno_pkg <- "IlluminaHumanMethylation450kanno.ilmn12.hg19"
if (!nzchar(system.file(package = anno_pkg))) BiocManager::install(anno_pkg, update = FALSE, ask = FALSE)Note: comb-p requires the Python combined-pvalues command and bedtools:
touch .combp.sh
{ command -v bedtools >/dev/null 2>&1; } || {
apt-get update && apt-get install -y bedtools
}
{ command -v comb-p >/dev/null 2>&1 && command -v bedtools >/dev/null 2>&1 && echo "$(command -v comb-p) \$@"> .combp.sh; } || {
echo >&2 "comb-p or bedtools not found. Installing with conda...";
apt-get update && apt-get install -y curl git bedtools
if ! command -v conda >/dev/null 2>&1; then
curl -L https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh
rm -rf $HOME/miniconda
bash miniconda.sh -b -p $HOME/miniconda
rm miniconda.sh
MINICONDA_BIN="$HOME/miniconda/bin/conda"
$MINICONDA_BIN init bash
export PATH="$HOME/miniconda/bin:$PATH"
else
MINICONDA_BIN=$(command -v conda)
fi
$MINICONDA_BIN tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
$MINICONDA_BIN tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
$MINICONDA_BIN create -n combp -y python=3.7
$MINICONDA_BIN run -n combp -- pip install git+https://github.com/brentp/combined-pvalues.git numpy==1.21.6
echo "$MINICONDA_BIN run -n combp comb-p \$@" > .combp.sh
}
chmod +x .combp.sh0.3 Load Input Data and Simulate
array_type <- "450K"
genome <- "hg19"
input_beta_path <- getBenchmarkFile("GSE40279_beta_subset.tsv.gz")
input_samplesheet_path <- getBenchmarkFile("GSE40279_sim_samplesheet.tsv")
zenodo_url <- "https://zenodo.org/records/20592944/files/GSE40279_beta_subset.tsv.gz?download=1"
zenodo_samplesheet_url <- "https://zenodo.org/records/20592944/files/GSE40279_sim_samplesheet.tsv?download=1"
if (!file.exists(input_samplesheet_path)) {
curl::curl_download(zenodo_samplesheet_url, input_samplesheet_path)
}
simulation_samplesheet <- data.table::fread(input_samplesheet_path, data.table = FALSE, check.names = FALSE)
rownames(simulation_samplesheet) <- simulation_samplesheet[[1L]]
simulation_samplesheet <- simulation_samplesheet[, -1L, drop = FALSE]
simulation_samplesheet$Sample_Group <- NULL
base_simulation <- list(
delta_jitter = 0.1,
max_gap = 500L,
min_dmr_size = 50L,
max_dmr_size = 2000L,
min_sites = 5L,
max_sites = 500L,
num_dmrs = 500L,
genome = genome,
sample_ids = rownames(simulation_samplesheet),
covariates = c("age..y..ch1", "predictedSex")
)
simulate_one <- function(plan_row) {
sim_file <- getSimulationFile(plan_row$simulation_id, "simulation.rds")
params <- c(base_simulation, list(
seed = plan_row$seed,
delta_beta_max = plan_row$delta_beta_max,
expected_corr = plan_row$expected_corr
))
cached <- read_cache(sim_file, params)
if (!is.null(cached)) return(cached$value)
cat(
sprintf(
"Simulating dataset %d with seed %d, delta_beta_max %.3f, expected_corr %.3f",
plan_row$simulation_id, plan_row$seed, plan_row$delta_beta_max, plan_row$expected_corr
)
)
if (!file.exists(input_beta_path)) {
curl::curl_download(zenodo_url, input_beta_path)
}
set.seed(plan_row$seed)
sim <- CMEnt::simulateDMRs(
beta = input_beta_path,
array = array_type,
genome = genome,
samplesheet = simulation_samplesheet,
sample_group_col = "Sample_Group",
covariates = base_simulation$covariates,
num_dmrs = base_simulation$num_dmrs,
delta_max0 = plan_row$delta_beta_max,
delta_jitter = base_simulation$delta_jitter,
expected_correlation = plan_row$expected_corr,
max_gap = base_simulation$max_gap,
min_dmr_size = base_simulation$min_dmr_size,
max_dmr_size = base_simulation$max_dmr_size,
min_sites = base_simulation$min_sites,
max_sites = base_simulation$max_sites,
verbose = 1
)
sim$pheno$Sample_ID <- rownames(sim$pheno)
sim$pheno$casecontrol <- sim$pheno$Sample_Group == "case"
data.table::fwrite(sim$truth, getSimulationFile(plan_row$simulation_id, "truth.tsv"), sep = "\t")
data.table::fwrite(sim$pheno, getSimulationFile(plan_row$simulation_id, "samplesheet.tsv"), sep = "\t")
save_cache(sim_file, sim, params)
sim
}
benchmark_specs_file <- getBenchmarkFile("benchmark_specs.rds")
if (!file.exists(benchmark_specs_file)) {
benchmark_specs <- list(
cpu = benchmarkme::get_cpu(),
ram = benchmarkme::get_ram(),
platform = benchmarkme::get_platform_info(),
rversion = R.version.string,
njobs = NJOBS
)
saveRDS(benchmark_specs, benchmark_specs_file)
} else {
benchmark_specs <- readRDS(benchmark_specs_file)
}
cpu <- benchmark_specs$cpu
ram <- benchmark_specs$ram
platform <- benchmark_specs$platform
rversion <- benchmark_specs$rversion
njobs <- benchmark_specs$njobs
cat("CPU:", cpu$vendor, cpu$model, "(", cpu$no_of_cores, "cores )\n")CPU: AuthenticAMD AMD EPYC 7B13 ( 8 cores )
cat("RAM:", capture.output(print(ram)), "\n")RAM: [1] 67433791488 attr(,“class”) [1] “ram”
cat("OS:", platform$OS.type, "\n")OS: unix
cat("R Version:", rversion, "\n")R Version: R version 4.6.0 (2026-04-24)
cat("Using", njobs, "parallel jobs\n")Using 7 parallel jobs
0.4 Method Runners
as_truth_gr <- function(sim) {
force_genome(makeGRangesFromDataFrame(
sim$truth,
seqnames.field = "seqnames",
start.field = "start",
end.field = "end",
keep.extra.columns = TRUE,
seqinfo = GenomeInfoDb::Seqinfo(genome = genome)
))
}
force_genome <- function(gr) {
GenomeInfoDb::genome(gr) <- genome
gr
}
logit2 <- function(x) log2(pmax(pmin(x, 1 - 1e-6), 1e-6)) - log2(1 - pmax(pmin(x, 1 - 1e-6), 1e-6))
prepare_array_data <- function(sim) {
beta_handler <- CMEnt::getBetaHandler(sim$simulated, sorted_locs = sim$beta_locs, njobs = njobs)
beta_mat <- as.matrix(beta_handler$getBeta())
locs <- beta_handler$getBetaLocs()
pheno <- sim$pheno[colnames(beta_mat), , drop = FALSE]
pheno$Sample_Group <- factor(pheno$Sample_Group, levels = c("control", "case"))
pheno$casecontrol <- pheno$Sample_Group == "case"
dmps <- CMEnt::findDMPsArray(
beta_handler,
pheno,
sample_group_col = "Sample_Group",
id_col = "Sample_ID",
array = array_type,
sorted_locs = sim$beta_locs,
case_group = "case",
njobs = njobs
)
rownames(dmps) <- dmps$site_id
dmps <- dmps[!is.na(dmps$pval), ]
dmp_locs <- locs[rownames(locs) %in% rownames(dmps), , drop = FALSE]
dmps <- dmps[rownames(dmp_locs), , drop = FALSE]
dmps_gr <- makeGRangesFromDataFrame(
data.frame(
chr = dmp_locs$chr,
start = dmp_locs$start,
end = dmp_locs$end,
id = rownames(dmps),
pval = dmps$pval,
qval = dmps$qval,
delta_beta = dmps$delta_beta
),
keep.extra.columns = TRUE
)
dmps_gr <- sort(dmps_gr)
names(dmps_gr) <- mcols(dmps_gr)$id
list(
beta_handler = beta_handler,
beta_mat = beta_mat,
locs = locs,
pheno = pheno,
dmps = dmps,
dmps_gr = dmps_gr,
seeds = mcols(dmps_gr[mcols(dmps_gr)$qval < 0.05])$id
)
}
run_combp <- function(dmps_gr, params) {
tmpdir <- tempdir()
bed_file <- file.path(tmpdir, "dmps_for_combp.bed")
bed_df <- data.frame(
chrom = as.character(Seqinfo::seqnames(dmps_gr)),
start = start(dmps_gr) - 1L,
end = end(dmps_gr),
pval = mcols(dmps_gr)$pval
)
write.table(bed_df[order(bed_df$chrom, bed_df$start), ], bed_file, quote = FALSE, sep = "\t", row.names = FALSE)
out_prefix <- file.path(tmpdir, "output.")
cmd <- sprintf(
"bash .combp.sh pipeline -c 4 --seed %d --dist %d -p %s --region-filter-p %s --region-filter-n %d %s",
params$seed, params$dist, out_prefix, params$region_filter_p, params$region_filter_n, bed_file
)
ret <- system(cmd, ignore.stdout = FALSE, ignore.stderr = FALSE)
if (ret != 0L) stop(sprintf("comb-p failed with exit code %d", ret))
regions_file <- paste0(out_prefix, "regions-p.bed.gz")
if (!file.exists(regions_file)) stop(sprintf("comb-p output file not found: %s", regions_file))
regions <- data.table::fread(cmd = sprintf("zcat %s", shQuote(regions_file)))
colnames(regions) <- c("chr", "start", "end", "min_p", "n_probes", "z_p", "z_sidak_p")
regions <- regions[regions$z_sidak_p < params$region_sidak_p & regions$n_probes >= params$region_filter_n, ]
if (nrow(regions) == 0L) stop("comb-p returned no regions passing filters")
makeGRangesFromDataFrame(regions, seqnames.field = "chr", start.field = "start", end.field = "end", keep.extra.columns = TRUE)
}
run_probelasso_champ <- function(beta_mat, pheno, params) {
if (!requireNamespace("callr", quietly = TRUE)) return(GRanges(seqinfo = GenomeInfoDb::Seqinfo(genome = params$genome)))
pl_dmr <- tryCatch({
callr::r(
function(beta_mat, sample_group, params) {
if (!requireNamespace("ChAMP", quietly = TRUE)) stop("ChAMP is not installed.")
suppressPackageStartupMessages(library(ChAMP))
result <- ChAMP::champ.DMR(
beta = beta_mat,
pheno = sample_group,
arraytype = params$array_type,
method = "ProbeLasso",
minProbes = params$minProbes,
adjPvalDmr = params$adjPvalDmr,
adjPvalProbe = params$adjPvalProbe,
meanLassoRadius = params$meanLassoRadius,
minDmrSep = params$minDmrSep,
minDmrSize = params$minDmrSize,
PDFplot = FALSE,
Rplot = FALSE,
resultsDir = file.path(tempdir(), "champ_probelasso")
)
pl_dmr <- result$ProbeLassoDMR
if (is.null(pl_dmr) || nrow(pl_dmr) == 0L) return(data.frame())
as.data.frame(pl_dmr)
},
args = list(
beta_mat = beta_mat,
sample_group = as.character(pheno$Sample_Group),
params = params
),
show = TRUE
)
}, error = function(e) {
warning(sprintf("probeLasso failed: %s", e$message))
data.frame()
})
if (nrow(pl_dmr) == 0L) return(GRanges(seqinfo = GenomeInfoDb::Seqinfo(genome = params$genome)))
if (!"seqnames" %in% names(pl_dmr) && "dmrChrom" %in% names(pl_dmr)) pl_dmr$seqnames <- pl_dmr$dmrChrom
if (!"start" %in% names(pl_dmr) && "dmrStart" %in% names(pl_dmr)) pl_dmr$start <- pl_dmr$dmrStart
if (!"end" %in% names(pl_dmr) && "dmrEnd" %in% names(pl_dmr)) pl_dmr$end <- pl_dmr$dmrEnd
makeGRangesFromDataFrame(
pl_dmr, seqnames.field = "seqnames", start.field = "start", end.field = "end",
seqinfo = GenomeInfoDb::Seqinfo(genome = params$genome), keep.extra.columns = TRUE
)
}
run_dmrcate_array <- function(beta_mat, pheno, params) {
if (!requireNamespace("DMRcate", quietly = TRUE)) return(GRanges(seqinfo = GenomeInfoDb::Seqinfo(genome = params$genome)))
design <- model.matrix(~ Sample_Group, data = pheno)
coef_col <- which(colnames(design) == "Sample_Groupcase")
colnames(design)[1] <- "(Intercept)"
colnames(design)[coef_col] <- "case_vs_control"
ann <- DMRcate::cpg.annotate(
"array",
logit2(beta_mat),
what = "M",
arraytype = params$array_type,
analysis.type = "differential",
design = design,
coef = coef_col,
fdr = params$fdr
)
DMRcate::extractRanges(DMRcate::dmrcate(
ann,
C = params$C,
lambda = params$lambda,
min.cpgs = params$min.cpgs,
betacutoff = params$betacutoff
), genome = params$genome)
}
run_bumphunter_array <- function(beta_mat, locs, pheno, params) {
if (!requireNamespace("bumphunter", quietly = TRUE) || !requireNamespace("doParallel", quietly = TRUE)) return(GRanges())
design <- model.matrix(~ Sample_Group, data = pheno)
coef_col <- which(colnames(design) == "Sample_Groupcase")
on.exit({
try(doParallel::stopImplicitCluster(), silent = TRUE)
gc()
}, add = TRUE)
cluster <- bumphunter::clusterMaker(as.character(locs$chr), locs$start, maxGap = params$max_gap)
run_fit <- function(cutoff, B) {
bumphunter::bumphunter(
beta_mat,
design = design,
chr = as.character(locs$chr),
pos = locs$start,
cluster = cluster,
cutoff = cutoff,
maxGap = params$max_gap,
nullMethod = "bootstrap",
B = B,
type = "Beta",
coef = coef_col
)
}
has_informative_null <- function(fit) {
null_values <- unlist(fit$null$value, use.names = FALSE)
null_lengths <- unlist(fit$null$length, use.names = FALSE)
length(null_values) > 0L &&
any(abs(null_values) > 0) &&
length(null_lengths) > 0L &&
any(null_lengths > 0)
}
cutoff <- params$cutoff_start
n_candidates <- Inf
iter <- 0L
while (n_candidates > params$max_dmrs && iter < params$max_iter) {
iter <- iter + 1L
fit0 <- run_fit(cutoff = cutoff, B = 0L)
n_candidates <- nrow(fit0$table)
cutoff <- cutoff * 1.2
rm(fit0)
gc()
}
cutoff <- cutoff / 1.2
cat(sprintf(
"Bumphunter: cutoff=%.4f, candidate_bumps=%d, minL=%d, maxGap=%d, cores=%d, B=%d\n",
cutoff, n_candidates, params$min_l, params$max_gap, njobs, params$B
))
fit <- run_fit(cutoff = cutoff, B = params$B)
adjust_iter <- 0L
while (
nrow(fit$table) > 0L &&
(all(is.na(fit$table$p.value)) || !has_informative_null(fit)) &&
cutoff > params$cutoff_floor &&
adjust_iter < params$max_bootstrap_adjust
) {
prev_cutoff <- cutoff
cutoff <- max(cutoff / 2, params$cutoff_floor)
if (identical(cutoff, prev_cutoff)) break
adjust_iter <- adjust_iter + 1L
cat(sprintf(
"Bumphunter: bootstrap null degenerate at cutoff=%.4f; retrying with cutoff=%.4f\n",
prev_cutoff, cutoff
))
fit <- run_fit(cutoff = cutoff, B = params$B)
}
tab <- fit$table
keep <- tab$L >= params$min_l &
is.finite(tab$p.value) &
tab$p.value <= params$region_p &
is.finite(tab$start) &
is.finite(tab$end)
tab <- tab[keep, , drop = FALSE]
if (nrow(tab) == 0L) return(GRanges())
makeGRangesFromDataFrame(tab, seqnames.field = "chr", start.field = "start", end.field = "end", keep.extra.columns = TRUE)
}
time_method <- function(expr) {
start <- Sys.time()
value <- force(expr)
list(dmrs = value, time = Sys.time() - start)
}0.5 Run Benchmark
run_one_simulation <- function(plan_row) {
sim_dir <- sprintf("sim_%02d", plan_row$simulation_id)
common <- list(delta_beta_max = plan_row$delta_beta_max, simulation_seed = plan_row$seed)
method_specs <- list(
CMEnt = list(
params = c(common, list(
sample_group_col = "Sample_Group", casecontrol_col = "casecontrol",
max_bridge_seeds_gaps = 0L, max_bridge_extension_gaps = 1L,
min_seeds = 2L, min_sites = 3L, ext_site_delta_beta = plan_row$delta_beta_max / 3,
max_lookup_dist = base_simulation$max_gap,
expansion_window = base_simulation$max_dmr_size,
njobs = njobs,
max_pval = 0.05, testing_mode = "parametric", entanglement = "weak",
genome = genome
)),
run = function(params) {
time_method(
do.call(
CMEnt::buildDMRs, c(
list(
beta = dat$beta_handler, seeds = dat$seeds, pheno = dat$pheno,
verbose = 2, score_dmrs = FALSE, extract_motifs = FALSE, annotate_with_genes = FALSE
),
params[setdiff(names(params), c("delta_beta_max", "simulation_seed"))]
)
)
)
}
),
`comb-p` = list(
params = c(
common,
list(
seed = base_simulation$max_gap, dist = base_simulation$max_gap, region_filter_p = 0.01,
region_filter_n = base_simulation$min_sites, region_sidak_p = 0.05
)
),
run = function(params) time_method(run_combp(dat$dmps_gr, params))
),
probeLasso = list(
params = c(
common,
list(
runner = "callr_attach_champ", array_type = array_type, genome = genome,
minProbes = base_simulation$min_sites, adjPvalDmr = 0.05, adjPvalProbe = 0.1,
meanLassoRadius = base_simulation$max_gap,
minDmrSep = base_simulation$max_gap, minDmrSize = base_simulation$min_dmr_size
)
),
run = function(params) time_method(run_probelasso_champ(dat$beta_mat, dat$pheno, params))
),
DMRcate = list(
params = c(
common,
list(
array_type = array_type, genome = genome, fdr = 0.05,
C = 2, lambda = base_simulation$max_gap * 2,
min.cpgs = base_simulation$min_sites, betacutoff = 0.05
)
),
run = function(params) time_method(run_dmrcate_array(dat$beta_mat, dat$pheno, params))
),
Bumphunter = list(
params = c(common, list(
runner = "adaptive_bootstrap_v2",
B = 100L,
max_dmrs = 100000L,
max_gap = base_simulation$max_gap,
min_l = base_simulation$min_sites,
region_p = 0.05,
cutoff_start = plan_row$delta_beta_max / 3,
cutoff_floor = 0.005,
max_iter = 10L,
max_bootstrap_adjust = 6L
)),
run = function(params) time_method(run_bumphunter_array(dat$beta_mat, dat$locs, dat$pheno, params))
)
)
cached_methods <- lapply(names(method_specs), function(method) {
spec <- method_specs[[method]]
cache_file <- getBenchmarkFile(file.path(sim_dir, sprintf("dmrs.%s.rds", gsub("[^A-Za-z0-9]+", "_", method))))
cached <- read_cache(cache_file, spec$params)
if (
!is.null(cached) &&
inherits(cached$value$dmrs, "GenomicRanges") &&
length(cached$value$dmrs) > 0L &&
!is.null(cached$value$max_r_heap_mb)
) {
cached$value
} else {
NULL
}
})
names(cached_methods) <- names(method_specs)
summary_file <- getSimulationFile(plan_row$simulation_id, "simulation_summary.rds")
summary_params <- c(base_simulation, list(
seed = plan_row$seed,
delta_beta_max = plan_row$delta_beta_max,
expected_corr = plan_row$expected_corr
))
cached_summary <- read_cache(summary_file, summary_params)
if (!is.null(cached_summary) && all(!vapply(cached_methods, is.null, logical(1)))) {
truth_dmrs <- cached_summary$value$truth
seeds_n <- cached_summary$value$seeds_n
} else {
sim <- simulate_one(plan_row)
truth_dmrs <- as_truth_gr(sim)
dat <- prepare_array_data(sim)
seeds_n <- nrow(dat$seeds)
save_cache(summary_file, list(truth = truth_dmrs, seeds_n = seeds_n), summary_params)
}
method_outputs <- lapply(names(method_specs), function(method) {
spec <- method_specs[[method]]
cache_file <- getBenchmarkFile(file.path(sim_dir, sprintf("dmrs.%s.rds", gsub("[^A-Za-z0-9]+", "_", method))))
cached <- read_cache(cache_file, spec$params)
out <- if (!is.null(cached) && length(cached$value$dmrs) > 0) {
cached$value
} else {
value <- spec$run(spec$params)
if (!is.null(value) && length(value$dmrs) > 0) {
save_cache(cache_file, value, spec$params)
}
value
}
out$dmrs <- force_genome(out$dmrs)
list(method = method, dmrs = out$dmrs, time = out$time)
})
names(method_outputs) <- names(method_specs)
list(
simulation_id = plan_row$simulation_id,
params = plan_row,
truth = truth_dmrs,
seeds_n = seeds_n,
methods = method_outputs
)
}
benchmark_results <- lapply(seq_len(nrow(simulation_plan)), function(i) {
run_one_simulation(simulation_plan[i, , drop = FALSE])
})Simulating dataset 1 with seed 560, delta_beta_max 0.340, expected_corr 0.660
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 2 with seed 996, delta_beta_max 0.420, expected_corr 0.360
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 3 with seed 320, delta_beta_max 0.480, expected_corr 0.700
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 4 with seed 152, delta_beta_max 0.280, expected_corr 0.680
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 5 with seed 73, delta_beta_max 0.340, expected_corr 0.330
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 6 with seed 227, delta_beta_max 0.480, expected_corr 0.510
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 7 with seed 145, delta_beta_max 0.490, expected_corr 0.460
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 8 with seed 633, delta_beta_max 0.240, expected_corr 0.660
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 9 with seed 48, delta_beta_max 0.340, expected_corr 0.480
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%Simulating dataset 10 with seed 127, delta_beta_max 0.370, expected_corr 0.630
|
| | 0%
|
| | 1%
|
|= | 1%
|
|= | 2%
|
|= | 3%
|
|== | 3%
|
|== | 4%
|
|== | 5%
|
|=== | 5%
|
|=== | 6%
|
|=== | 7%
|
|==== | 7%
|
|==== | 8%
|
|==== | 9%
|
|===== | 9%
|
|===== | 10%
|
|===== | 11%
|
|====== | 11%
|
|====== | 12%
|
|====== | 13%
|
|======= | 13%
|
|======= | 14%
|
|======= | 15%
|
|======== | 15%
|
|======== | 16%
|
|======== | 17%
|
|========= | 17%
|
|========= | 18%
|
|========= | 19%
|
|========== | 19%
|
|========== | 20%
|
|========== | 21%
|
|=========== | 21%
|
|=========== | 22%
|
|=========== | 23%
|
|============ | 23%
|
|============ | 24%
|
|============ | 25%
|
|============= | 25%
|
|============= | 26%
|
|============= | 27%
|
|============== | 27%
|
|============== | 28%
|
|============== | 29%
|
|=============== | 29%
|
|=============== | 30%
|
|=============== | 31%
|
|================ | 31%
|
|================ | 32%
|
|================ | 33%
|
|================= | 33%
|
|================= | 34%
|
|================= | 35%
|
|================== | 35%
|
|================== | 36%
|
|================== | 37%
|
|=================== | 37%
|
|=================== | 38%
|
|=================== | 39%
|
|==================== | 39%
|
|==================== | 40%
|
|==================== | 41%
|
|===================== | 41%
|
|===================== | 42%
|
|===================== | 43%
|
|====================== | 43%
|
|====================== | 44%
|
|====================== | 45%
|
|======================= | 45%
|
|======================= | 46%
|
|======================= | 47%
|
|======================== | 47%
|
|======================== | 48%
|
|======================== | 49%
|
|========================= | 49%
|
|========================= | 50%
|
|========================= | 51%
|
|========================== | 51%
|
|========================== | 52%
|
|========================== | 53%
|
|=========================== | 53%
|
|=========================== | 54%
|
|=========================== | 55%
|
|============================ | 55%
|
|============================ | 56%
|
|============================ | 57%
|
|============================= | 57%
|
|============================= | 58%
|
|============================= | 59%
|
|============================== | 59%
|
|============================== | 60%
|
|============================== | 61%
|
|=============================== | 61%
|
|=============================== | 62%
|
|=============================== | 63%
|
|================================ | 63%
|
|================================ | 64%
|
|================================ | 65%
|
|================================= | 65%
|
|================================= | 66%
|
|================================= | 67%
|
|================================== | 67%
|
|================================== | 68%
|
|================================== | 69%
|
|=================================== | 69%
|
|=================================== | 70%
|
|=================================== | 71%
|
|==================================== | 71%
|
|==================================== | 72%
|
|==================================== | 73%
|
|===================================== | 73%
|
|===================================== | 74%
|
|===================================== | 75%
|
|====================================== | 75%
|
|====================================== | 76%
|
|====================================== | 77%
|
|======================================= | 77%
|
|======================================= | 78%
|
|======================================= | 79%
|
|======================================== | 79%
|
|======================================== | 80%
|
|======================================== | 81%
|
|========================================= | 81%
|
|========================================= | 82%
|
|========================================= | 83%
|
|========================================== | 83%
|
|========================================== | 84%
|
|========================================== | 85%
|
|=========================================== | 85%
|
|=========================================== | 86%
|
|=========================================== | 87%
|
|============================================ | 87%
|
|============================================ | 88%
|
|============================================ | 89%
|
|============================================= | 89%
|
|============================================= | 90%
|
|============================================= | 91%
|
|============================================== | 91%
|
|============================================== | 92%
|
|============================================== | 93%
|
|=============================================== | 93%
|
|=============================================== | 94%
|
|=============================================== | 95%
|
|================================================ | 95%
|
|================================================ | 96%
|
|================================================ | 97%
|
|================================================= | 97%
|
|================================================= | 98%
|
|================================================= | 99%
|
|==================================================| 99%
|
|==================================================| 100%
0.6 Comparative Analysis
get_dmr_stats <- function(dmrs, method, simulation_id) {
data.frame(
simulation_id = simulation_id,
Method = method,
Number_of_DMRs = length(dmrs),
Median_DMR_Width = ifelse(length(dmrs) == 0L, NA_real_, median(width(dmrs))),
Total_Coverage_bp = ifelse(length(dmrs) == 0L, 0, sum(width(reduce(dmrs)))),
Min_Width = ifelse(length(dmrs) == 0L, NA_real_, min(width(dmrs))),
Max_Width = ifelse(length(dmrs) == 0L, NA_real_, max(width(dmrs)))
)
}
evaluate_against_truth <- function(pred, truth) {
pred <- force_genome(pred)
truth <- force_genome(truth)
n_pred <- length(pred)
n_truth <- length(truth)
pred_hit <- sum(IRanges::overlapsAny(pred, truth))
truth_hit <- sum(IRanges::overlapsAny(truth, pred))
precision <- ifelse(n_pred == 0L, 0, pred_hit / n_pred)
recall <- ifelse(n_truth == 0L, 0, truth_hit / n_truth)
pred_red <- reduce(pred)
truth_red <- reduce(truth)
inter_bp <- sum(width(GenomicRanges::intersect(pred_red, truth_red)))
pred_bp <- sum(width(pred_red))
truth_bp <- sum(width(truth_red))
union_bp <- pred_bp + truth_bp - inter_bp
precision_bp <- ifelse(pred_bp == 0L, 0, inter_bp / pred_bp)
recall_bp <- ifelse(truth_bp == 0L, 0, inter_bp / truth_bp)
data.frame(
TP_regions = truth_hit,
FP_regions = n_pred - pred_hit,
FN_regions = n_truth - truth_hit,
Precision_regions = precision,
Recall_regions = recall,
F1_regions = ifelse(precision + recall == 0, 0, 2 * precision * recall / (precision + recall)),
Precision_bp = precision_bp,
Recall_bp = recall_bp,
F1_bp = ifelse(precision_bp + recall_bp == 0, 0, 2 * precision_bp * recall_bp / (precision_bp + recall_bp)),
IoU_bp = ifelse(union_bp == 0L, 0, inter_bp / union_bp)
)
}
format_mean_pm <- function(x, digits = 2L, suffix = "") {
x <- x[is.finite(x)]
if (length(x) == 0L) return(NA_character_)
sd_x <- if (length(x) > 1L) stats::sd(x) else 0
fmt <- paste0("%.", digits, "f")
values <- sprintf(fmt, c(mean(x), sd_x))
paste0(values[1L], suffix, " (+/- ", values[2L], suffix, ")")
}
summarise_mean_pm <- function(df, group_cols, value_cols, digits = 2L) {
metric_digits <- if (length(digits) == 1L) {
stats::setNames(rep(digits, length(value_cols)), value_cols)
} else {
stats::setNames(as.integer(digits), names(digits))
}
df |>
tidyr::pivot_longer(dplyr::all_of(value_cols), names_to = "Metric", values_to = "Value") |>
dplyr::group_by(dplyr::across(dplyr::all_of(c(group_cols, "Metric")))) |>
dplyr::summarise(
Value = format_mean_pm(Value, digits = metric_digits[dplyr::first(Metric)]),
.groups = "drop"
) |>
tidyr::pivot_wider(names_from = Metric, values_from = Value)
}
summarise_benchmark_table <- function(stats_all, runtime_all) {
stats_all |>
dplyr::select(
simulation_id,
Method,
dmrs_number = Number_of_DMRs,
median_width_bp = Median_DMR_Width,
min_width_bp = Min_Width,
max_width_bp = Max_Width,
Total_Coverage_bp
) |>
dplyr::mutate(total_coverage_kb = Total_Coverage_bp / 1000) |>
dplyr::select(-Total_Coverage_bp) |>
dplyr::left_join(
runtime_all |>
dplyr::rename(runtime_sec = Seconds),
by = c("simulation_id", "Method")
) |>
dplyr::rename(method = Method) |>
dplyr::mutate(
relative_runtime_pct = {
cment_mean_runtime <- mean(runtime_sec[method == "CMEnt" & is.finite(runtime_sec)])
if (is.finite(cment_mean_runtime) && cment_mean_runtime > 0) {
runtime_sec / cment_mean_runtime * 100
} else {
NA_real_
}
}
) |>
dplyr::group_by(method) |>
dplyr::summarise(
dmrs_number = format_mean_pm(dmrs_number, digits = 0L),
median_width_bp = format_mean_pm(median_width_bp, digits = 0L),
min_width_bp = format_mean_pm(min_width_bp, digits = 0L),
max_width_bp = format_mean_pm(max_width_bp, digits = 0L),
total_coverage_kb = format_mean_pm(total_coverage_kb, digits = 1L),
runtime_sec = format_mean_pm(runtime_sec, digits = 1L),
relative_runtime_pct = format_mean_pm(relative_runtime_pct, digits = 1L, suffix = "%"),
.groups = "drop"
)
}
stats_all <- do.call(rbind, lapply(benchmark_results, function(res) {
rbind(
do.call(rbind, lapply(res$methods, function(x) get_dmr_stats(x$dmrs, x$method, res$simulation_id))),
get_dmr_stats(res$truth, "Truth", res$simulation_id)
)
}))
truth_eval_all <- do.call(rbind, lapply(benchmark_results, function(res) {
do.call(rbind, lapply(res$methods, function(x) {
out <- evaluate_against_truth(x$dmrs, res$truth)
out$simulation_id <- res$simulation_id
out$Method <- x$method
out
}))
}))
runtime_all <- do.call(rbind, lapply(benchmark_results, function(res) {
do.call(rbind, lapply(res$methods, function(x) {
data.frame(simulation_id = res$simulation_id, Method = x$method, Seconds = as.numeric(x$time, units = "secs"))
}))
}))0.6.1 Basic Statistics
stats_summary <- summarise_benchmark_table(stats_all, runtime_all)
knitr::kable(stats_summary, caption = "Benchmark statistics across simulations, mean (+/- SD)")| method | dmrs_number | median_width_bp | min_width_bp | max_width_bp | total_coverage_kb | runtime_sec | relative_runtime_pct |
|---|---|---|---|---|---|---|---|
| Bumphunter | 366 (+/- 44) | 748 (+/- 66) | 95 (+/- 22) | 1874 (+/- 100) | 291.5 (+/- 58.1) | 6.6 (+/- 0.5) | 51.4% (+/- 3.6%) |
| CMEnt | 497 (+/- 10) | 862 (+/- 35) | 78 (+/- 34) | 1963 (+/- 10) | 447.8 (+/- 20.4) | 12.8 (+/- 1.4) | 100.0% (+/- 11.0%) |
| DMRcate | 413 (+/- 59) | 1147 (+/- 97) | 155 (+/- 35) | 4430 (+/- 965) | 505.4 (+/- 108.8) | 4.5 (+/- 3.5) | 34.8% (+/- 27.5%) |
| Truth | 500 (+/- 0) | 919 (+/- 9) | 118 (+/- 13) | 1969 (+/- 4) | 478.3 (+/- 3.6) | NA | NA |
| comb-p | 483 (+/- 29) | 930 (+/- 23) | 121 (+/- 13) | 1970 (+/- 4) | 467.9 (+/- 20.3) | 7.7 (+/- 0.1) | 60.5% (+/- 0.6%) |
| probeLasso | 76 (+/- 7) | 896 (+/- 51) | 66 (+/- 23) | 12848 (+/- 2936) | 176.1 (+/- 33.7) | 31.1 (+/- 0.4) | 243.1% (+/- 3.3%) |
stats_plot <- stats_all |>
dplyr::select(Method, simulation_id, Number_of_DMRs, Total_Coverage_bp) |>
tidyr::pivot_longer(c(Number_of_DMRs, Total_Coverage_bp), names_to = "Metric", values_to = "Value") |>
dplyr::group_by(Method, Metric) |>
dplyr::mutate(Median = median(Value), Min = min(Value), Max = max(Value)) |>
dplyr::ungroup()
ggplot(stats_plot, aes(Method, Value, colour = Method)) +
geom_point(alpha = 0.45, position = position_jitter(width = 0.08, height = 0)) +
stat_summary(fun = median, geom = "point", size = 2.6, colour = "black") +
stat_summary(fun.min = min, fun.max = max, geom = "errorbar", width = 0.25, colour = "black") +
facet_wrap(~ Metric, scales = "free_y") +
theme_minimal() +
theme(axis.text.x = element_text(angle = 45, hjust = 1), legend.position = "none") +
labs(title = "DMR counts and coverage across simulations", x = "", y = "Value")
Figure 1: DMR counts and total genomic coverage by method across simulated 450K datasets
0.7 Ground-Truth Evaluation
regions_truth_eval <- summarise_mean_pm(
truth_eval_all,
"Method",
c("TP_regions", "FP_regions", "FN_regions", "Precision_regions", "Recall_regions", "F1_regions"),
digits = c(TP_regions = 0L, FP_regions = 0L, FN_regions = 0L, Precision_regions = 2L, Recall_regions = 2L, F1_regions = 2L)
)
bp_truth_eval <- summarise_mean_pm(
truth_eval_all,
"Method",
c("Precision_bp", "Recall_bp", "F1_bp", "IoU_bp"),
digits = 2
)
knitr::kable(regions_truth_eval, caption = "Macro region-level performance against ground truth, mean (+/- SD)")| Method | F1_regions | FN_regions | FP_regions | Precision_regions | Recall_regions | TP_regions |
|---|---|---|---|---|---|---|
| Bumphunter | 0.84 (+/- 0.06) | 134 (+/- 43) | 0 (+/- 0) | 1.00 (+/- 0.00) | 0.73 (+/- 0.09) | 366 (+/- 43) |
| CMEnt | 0.99 (+/- 0.01) | 6 (+/- 10) | 3 (+/- 1) | 0.99 (+/- 0.00) | 0.99 (+/- 0.02) | 494 (+/- 10) |
| DMRcate | 0.92 (+/- 0.08) | 74 (+/- 64) | 0 (+/- 0) | 1.00 (+/- 0.00) | 0.85 (+/- 0.13) | 426 (+/- 64) |
| comb-p | 0.98 (+/- 0.03) | 17 (+/- 29) | 0 (+/- 0) | 1.00 (+/- 0.00) | 0.97 (+/- 0.06) | 483 (+/- 29) |
| probeLasso | 0.27 (+/- 0.02) | 421 (+/- 8) | 0 (+/- 0) | 1.00 (+/- 0.00) | 0.16 (+/- 0.02) | 79 (+/- 8) |
knitr::kable(bp_truth_eval, caption = "Micro base-pair-level performance against ground truth, mean (+/- SD)")| Method | F1_bp | IoU_bp | Precision_bp | Recall_bp |
|---|---|---|---|---|
| Bumphunter | 0.75 (+/- 0.09) | 0.61 (+/- 0.12) | 1.00 (+/- 0.00) | 0.61 (+/- 0.12) |
| CMEnt | 0.96 (+/- 0.02) | 0.93 (+/- 0.04) | 1.00 (+/- 0.00) | 0.93 (+/- 0.04) |
| DMRcate | 0.82 (+/- 0.04) | 0.69 (+/- 0.06) | 0.81 (+/- 0.06) | 0.84 (+/- 0.13) |
| comb-p | 0.99 (+/- 0.02) | 0.97 (+/- 0.04) | 1.00 (+/- 0.00) | 0.98 (+/- 0.04) |
| probeLasso | 0.13 (+/- 0.01) | 0.07 (+/- 0.01) | 0.25 (+/- 0.03) | 0.09 (+/- 0.01) |
metric_plot <- truth_eval_all |>
dplyr::select(Method, simulation_id, F1_regions, F1_bp, IoU_bp) |>
tidyr::pivot_longer(c(F1_regions, F1_bp, IoU_bp), names_to = "Metric", values_to = "Value")
metric_plot$Metric <- factor(
metric_plot$Metric,
levels = c("F1_regions", "F1_bp", "IoU_bp"),
labels = c("F1 (Region)", "F1 (Base-pair)", "IoU (Base-pair)")
)
ggplot(metric_plot, aes(Method, Value, colour = Method)) +
geom_point(alpha = 0.5, position = position_jitter(width = 0.08, height = 0)) +
stat_summary(fun = median, geom = "point", size = 2.6, colour = "black") +
stat_summary(fun.min = min, fun.max = max, geom = "errorbar", width = 0.25, colour = "black") +
facet_wrap(~ Metric, nrow = 1) +
coord_cartesian(ylim = c(0, 1)) +
theme_minimal() +
theme(axis.text.x = element_text(angle = 45, hjust = 1), legend.position = "none") +
labs(title = "Method performance vs ground truth across simulations", x = "", y = "Score")
Figure 2: Region-level F1, base-pair-level F1, and base-pair intersection over union against ground truth across simulated 450K datasets
0.7.1 Runtime
runtime_summary <- summarise_mean_pm(runtime_all, "Method", "Seconds", digits = 2)
knitr::kable(runtime_summary, caption = "Runtime across simulations, mean (+/- SD)")| Method | Seconds |
|---|---|
| Bumphunter | 6.57 (+/- 0.46) |
| CMEnt | 12.77 (+/- 1.40) |
| DMRcate | 4.45 (+/- 3.52) |
| comb-p | 7.73 (+/- 0.08) |
| probeLasso | 31.06 (+/- 0.43) |
ggplot(runtime_all, aes(Method, Seconds, colour = Method)) +
geom_point(alpha = 0.45, position = position_jitter(width = 0.08, height = 0)) +
stat_summary(fun = median, geom = "point", size = 2.6, colour = "black") +
stat_summary(fun.min = min, fun.max = max, geom = "errorbar", width = 0.25, colour = "black") +
scale_y_continuous(labels = scales::comma) +
theme_minimal() +
theme(axis.text.x = element_text(angle = 45, hjust = 1), legend.position = "none") +
labs(title = "Runtime across simulations", x = "", y = "Seconds")
Figure 3: Runtime by method across simulated 450K datasets
0.7.2 Width Distributions
width_data <- do.call(rbind, lapply(benchmark_results, function(res) {
sets <- c(lapply(res$methods, `[[`, "dmrs"), list(Truth = res$truth))
do.call(rbind, lapply(names(sets), function(method) {
if (length(sets[[method]]) == 0L) return(NULL)
data.frame(simulation_id = res$simulation_id, Method = method, Width = width(sets[[method]]))
}))
}))
ggplot(width_data, aes(Method, Width, fill = Method)) +
geom_violin(alpha = 0.55) +
geom_boxplot(width = 0.18, alpha = 0.55, outlier.size = 0.4) +
scale_y_log10(labels = scales::comma) +
theme_minimal() +
theme(axis.text.x = element_text(angle = 45, hjust = 1), legend.position = "none") +
labs(title = "DMR width distributions pooled across simulations", x = "", y = "Width (bp, log scale)")
Figure 4: DMR width distributions by method and ground truth, pooled across simulated 450K datasets
0.8 Summary
best_f1 <- truth_eval_all |>
dplyr::group_by(Method) |>
dplyr::summarise(Median_F1 = median(F1_regions), Median_IoU = median(IoU_bp), .groups = "drop")
cat(sprintf("- Best median region-level F1: **%s** (%.3f)\n", best_f1$Method[which.max(best_f1$Median_F1)], max(best_f1$Median_F1)))- Best median region-level F1: comb-p (0.998)
cat(sprintf("- Best median base-pair IoU: **%s** (%.3f)\n", best_f1$Method[which.max(best_f1$Median_IoU)], max(best_f1$Median_IoU)))- Best median base-pair IoU: comb-p (0.994)
0.9 Session Info
sessionInfo()R version 4.6.1 (2026-06-24) Platform: x86_64-pc-linux-gnu Running under: Ubuntu 24.04.4 LTS
Matrix products: default BLAS: /usr/lib/x86_64-linux-gnu/blas/libblas.so.3.12.0 LAPACK: /usr/lib/x86_64-linux-gnu/lapack/liblapack.so.3.12.0 LAPACK version 3.12.0
locale:
[1] LC_CTYPE=en_US.UTF-8 LC_NUMERIC=C
[3] LC_TIME=en_US.UTF-8 LC_COLLATE=en_US.UTF-8
[5] LC_MONETARY=en_US.UTF-8 LC_MESSAGES=en_US.UTF-8
[7] LC_PAPER=en_US.UTF-8 LC_NAME=C
[9] LC_ADDRESS=C LC_TELEPHONE=C
[11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C
time zone: Europe/Brussels tzcode source: system (glibc)
attached base packages:
[1] stats4 stats graphics grDevices datasets utils methods
[8] base
other attached packages:
[1] ggplot2_4.0.3 GenomicRanges_1.64.0 Seqinfo_1.2.0
[4] IRanges_2.46.0 S4Vectors_0.50.3 BiocGenerics_0.58.1
[7] generics_0.1.4 dplyr_1.2.1 data.table_1.18.6.1
[10] CMEnt_0.99.14 testthat_3.3.2 BiocStyle_2.40.0
loaded via a namespace (and not attached):
[1] later_1.4.8 BiocIO_1.22.0
[3] bitops_1.1-0 filelock_1.0.3
[5] tibble_3.3.1 R.oo_1.27.1
[7] DirichletMultinomial_1.54.1 XML_3.99-0.25
[9] lifecycle_1.0.5 httr2_1.3.0
[11] pwalign_1.8.0 doParallel_1.0.17
[13] rprojroot_2.1.1 processx_3.9.0
[15] lattice_0.23-1 backports_1.5.1
[17] magrittr_2.0.5 limma_3.68.5
[19] plotly_4.12.1 sass_0.4.10
[21] rmarkdown_2.32 jquerylib_0.1.4
[23] yaml_2.3.12 httpuv_1.6.17
[25] otel_0.2.0 pkgbuild_1.4.8
[27] DBI_1.3.0 RColorBrewer_1.1-3
[29] abind_1.4-8 pkgload_1.5.3
[31] purrr_1.2.2 R.utils_2.13.0
[33] RCurl_1.98-1.20 DelayedDataFrame_1.28.0
[35] circlize_0.4.18 inline_0.3.21
[37] seqLogo_1.78.0 pkgdown_2.2.1
[39] permute_0.9-10 DelayedMatrixStats_1.34.0
[41] codetools_0.2-20 DelayedArray_0.38.2
[43] DT_0.34.0 tidyselect_1.2.1
[45] shape_1.4.6.1 futile.logger_1.4.9
[47] ggseqlogo_0.2.2 UCSC.utils_1.8.0
[49] farver_2.1.2 matrixStats_1.5.0
[51] BiocFileCache_3.2.0 showtext_0.9-8
[53] GenomicAlignments_1.48.0 jsonlite_2.0.0
[55] GetoptLong_1.1.1 e1071_1.7-17
[57] iterators_1.0.14 foreach_1.5.2
[59] tools_4.6.1 TFMPvalue_1.0.0
[61] Rcpp_1.1.2 glue_1.8.1
[63] gridExtra_2.3.1 SparseArray_1.12.3
[65] xfun_0.62 MatrixGenerics_1.24.0
[67] GenomeInfoDb_1.48.0 HDF5Array_1.40.0
[69] withr_3.0.3 formatR_1.14
[71] BiocManager_1.30.27 fastmap_1.2.0
[73] bedr_1.1.5 rhdf5filters_1.24.1
[75] caTools_1.18.4 callr_3.8.0
[77] digest_0.6.39 mime_0.13
[79] R6_2.6.1 colorspace_2.1-4
[81] gtools_3.9.5 dichromat_2.0-1
[83] RSQLite_3.53.3 cigarillo_1.2.1
[85] R.methodsS3_1.8.2 h5mread_1.4.1
[87] tidyr_1.3.2 rtracklayer_1.72.0
[89] FNN_1.1.4.1 class_7.3-24
[91] httr_1.4.9 htmlwidgets_1.6.4
[93] S4Arrays_1.12.1 TFBSTools_1.50.0
[95] pkgconfig_2.0.3 gtable_0.3.6
[97] blob_1.3.0 ComplexHeatmap_2.28.0
[99] S7_0.2.2 XVector_0.52.0
[101] brio_1.1.5 htmltools_0.5.9
[103] sysfonts_0.8.9 bookdown_0.48
[105] strex_2.1.0 clue_0.3-68
[107] scales_1.4.0 Biobase_2.72.0
[109] png_0.1-9 knitr_1.52
[111] lambda.r_1.2.6 reshape2_1.4.5
[113] rjson_0.2.23 checkmate_2.3.4
[115] curl_8.1.0 showtextdb_3.0
[117] proxy_0.4-29 cachem_1.1.0
[119] rhdf5_2.56.1 GlobalOptions_0.1.4
[121] stringr_1.6.0 shinycssloaders_1.1.0
[123] parallel_4.6.1 AnnotationDbi_1.74.0
[125] restfulr_0.0.17 desc_1.4.3
[127] pillar_1.11.1 grid_4.6.1
[129] vctrs_0.7.3 promises_1.5.0
[131] dbplyr_2.6.0 beachmat_2.28.0
[133] xtable_1.8-8 cluster_2.1.8.3
[135] evaluate_1.0.5 bsseq_1.48.0
[137] magick_2.9.1 tinytex_0.61
[139] VennDiagram_1.8.2 GenomicFeatures_1.64.0
[141] cli_3.6.6 locfit_1.5-9.12
[143] compiler_4.6.1 futile.options_1.0.1
[145] Rsamtools_2.28.0 rlang_1.3.0
[147] crayon_1.5.3 labeling_0.4.3
[149] ps_1.9.3 plyr_1.8.9
[151] fs_2.1.0 stringi_1.8.9
[153] viridisLite_0.4.3 gridBase_0.4-7
[155] BiocParallel_1.46.0 Biostrings_2.80.2
[157] bspm_0.5.8 Matrix_1.7-6
[159] BSgenome_1.80.0 sparseMatrixStats_1.24.0
[161] bit64_4.8.6 Rhdf5lib_2.0.0
[163] KEGGREST_1.52.2 statmod_1.5.2
[165] shiny_1.14.0 SummarizedExperiment_1.42.0
[167] igraph_2.3.4 memoise_2.0.1
[169] bslib_0.12.0 bit_4.6.0