Benchmarks

library(sceua)

benchmark_packages_available <-
  requireNamespace("rtop", quietly = TRUE) &&
  requireNamespace("SoilHyP", quietly = TRUE) &&
  requireNamespace("bench", quietly = TRUE)

The sceua package is not the first R package with the Shuffled Complex Evolution algorithm for global optimisation. To our knowledge, the rtop and SoilHyP packages have implemented this algorithm before. However, sceua provides the fastest implementation available in R. See benchmarks below.

A non-linear curve-fitting problem

To test the three packages against each other, we first use the same non-linear curve-fitting problem as in the rtop::sceua() documentation. We generate noisy observations from a sine function with three parameters and recover them by minimising the sum of squared errors.

set.seed(1)

fun <- function(x, pars) pars[2] * sin(x * pars[1]) + pars[3]

x <- rnorm(50, sd = 3)
y <- fun(x, pars = c(5, 2, 3)) + rnorm(length(x), sd = 0.3)

OFUN <- function(pars, x, yobs) {
  yvals <- fun(x, pars)
  sum((yvals - yobs)^2)
}

Timing comparison

The rtop and SoilHyP packages contain SCE-UA implementations in pure R. Below is a comparison of these two packages with the Rust-backed sceua package on the same problem.

rtop::sceua() and SoilHyP::SCEoptim() print convergence information to the console. The helper below captures that output and returns only the optimised value.

Code
silent_rtop <- function(...) {
  utils::capture.output(
    res <- rtop::sceua(...),
    type = "output"
  )
  res$value
}

silent_soilhyp <- function(...) {
  utils::capture.output(
    res <- SoilHyP::SCEoptim(...),
    type = "output"
  )
  res$value
}

# The first `rtop` call in a fresh session can land in a different local basin;
# a single warm-up call makes the benchmarked calls comparable to `sceua`.
invisible(silent_rtop(
  OFUN,
  pars = c(0.1, 0.1, 0.1),
  lower = c(-10, 0, -10),
  upper = c(10, 10, 10),
  x = x,
  yobs = y,
  maxn = 10000
))

invisible(silent_soilhyp(
  FUN = OFUN,
  par = c(0.1, 0.1, 0.1),
  lower = c(-10, 0, -10),
  upper = c(10, 10, 10),
  x = x,
  yobs = y,
  control = list(maxeval = 10000)
))
set.seed(1)

bench::mark(
  sceua = sceua::sceua(
    fn = OFUN,
    lower = c(-10, 0, -10),
    upper = c(10, 10, 10),
    initial = c(0.1, 0.1, 0.1),
    x = x,
    yobs = y,
    max_evaluations = 10000
  )$value,
  rtop = silent_rtop(
    OFUN,
    pars = c(0.1, 0.1, 0.1),
    lower = c(-10, 0, -10),
    upper = c(10, 10, 10),
    x = x,
    yobs = y,
    maxn = 10000
  ),
  soilhyp = silent_soilhyp(
    FUN = OFUN,
    par = c(0.1, 0.1, 0.1),
    lower = c(-10, 0, -10),
    upper = c(10, 10, 10),
    x = x,
    yobs = y,
    control = list(maxeval = 10000)
  ),
  iterations = 25,
  relative = FALSE,
  check = FALSE
)
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> # A tibble: 3 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 sceua        1.85ms   3.05ms    303.      1.59MB     12.1
#> 2 rtop       128.35ms 139.78ms      6.51    1.12MB     14.1
#> 3 soilhyp    146.15ms 188.22ms      5.09    2.73MB     13.6

Q. Duan’s test-function benchmarks

The following benchmarks use R translations of the Q. Duan test functions published alongside the Matlab code. We use one aligned strict configuration for all problems: objective convergence is disabled (pcento = 0) and parameter convergence is tightened to 1e-6. The same seed is used for every package so the initial populations are comparable.

Each problem below gets its own table with the best value found, the known optimum, the mean absolute error (MAE = |value - optimum|), the median wall-clock time over 5 timed runs, and a Timing bar scaled to the slowest run within that problem (a full bar marks the slowest package). Every package is timed on every problem; cells show only when a package errors or fails to return a finite value. Per-problem bounds are visible in the folded code chunk below.

Code
goldstein_price <- function(x) {
  x1 <- x[1]
  x2 <- x[2]
  u1 <- (x1 + x2 + 1)^2
  u2 <- 19 - 14 * x1 + 3 * x1^2 - 14 * x2 + 6 * x1 * x2 + 3 * x2^2
  u3 <- (2 * x1 - 3 * x2)^2
  u4 <- 18 - 32 * x1 + 12 * x1^2 + 48 * x2 - 36 * x1 * x2 +
    27 * x2^2
  (1 + u1 * u2) * (30 + u3 * u4)
}

rosenbrock <- function(x) {
  100 * (x[2] - x[1]^2)^2 + (1 - x[1])^2
}

six_hump_camelback <- function(x) {
  x1 <- x[1]
  x2 <- x[2]
  (4 - 2.1 * x1^2 + x1^4 / 3) * x1^2 + x1 * x2 +
    (-4 + 4 * x2^2) * x2^2
}

rastrigin_duan <- function(x) {
  x[1]^2 + x[2]^2 - cos(18 * x[1]) - cos(18 * x[2])
}

griewank_duan <- function(x) {
  divisor <- if (length(x) == 2L) 200 else 4000
  sum(x^2 / divisor) - prod(cos(x / sqrt(seq_along(x)))) + 1
}

shekel <- function(x) {
  a <- rbind(
    c(4, 1, 8, 6, 3, 2, 5, 8, 6, 7),
    c(4, 1, 8, 6, 7, 9, 5, 1, 2, 3.6),
    c(4, 1, 8, 6, 3, 2, 3, 8, 6, 7),
    c(4, 1, 8, 6, 7, 9, 3, 1, 2, 3.6)
  )
  c_values <- c(0.1, 0.2, 0.2, 0.4, 0.4, 0.6, 0.3, 0.7, 0.5, 0.5)

  -sum(vapply(seq_along(c_values), function(i) {
    1 / (sum((x - a[, i])^2) + c_values[i])
  }, numeric(1)))
}

hartman <- function(x) {
  a6 <- rbind(
    c(10, 0.05, 3, 17),
    c(3, 10, 3.5, 8),
    c(17, 17, 1.7, 0.05),
    c(3.5, 0.1, 10, 10),
    c(1.7, 8, 17, 0.1),
    c(8, 14, 8, 14)
  )
  c6 <- c(1, 1.2, 3, 3.2)
  p6 <- rbind(
    c(0.1312, 0.2329, 0.2348, 0.4047),
    c(0.1696, 0.4135, 0.1451, 0.8828),
    c(0.5569, 0.8307, 0.3522, 0.8732),
    c(0.0124, 0.3736, 0.2883, 0.5743),
    c(0.8283, 0.1004, 0.3047, 0.1091),
    c(0.5886, 0.9991, 0.6650, 0.0381)
  )

  -sum(vapply(seq_along(c6), function(i) {
    c6[i] * exp(-sum(a6[, i] * (x - p6[, i])^2))
  }, numeric(1)))
}

duan_control <- list(
  seed = 1L,
  complexes = 5L,
  max_evaluations = 10000L,
  kstop = 5L,
  pcento = 0,
  parameter_epsilon = 1e-6
)

duan_problems <- list(
  goldstein_price = list(
    fn = goldstein_price,
    lower = c(-2, -2),
    upper = c(2, 2),
    initial = c(1, 1),
    optimum = 3
  ),
  rosenbrock = list(
    fn = rosenbrock,
    lower = c(-5, -5),
    upper = c(5, 5),
    initial = c(-3, 4),
    optimum = 0
  ),
  six_hump_camelback = list(
    fn = six_hump_camelback,
    lower = c(-5, -2),
    upper = c(5, 8),
    initial = c(0, 0),
    optimum = -1.031628453489877
  ),
  rastrigin_duan = list(
    fn = rastrigin_duan,
    lower = c(-1, -1),
    upper = c(1, 1),
    initial = c(0.5, 0.5),
    optimum = -2
  ),
  griewank_duan = list(
    fn = griewank_duan,
    lower = rep(-600, 10),
    upper = rep(600, 10),
    initial = rep(100, 10),
    optimum = 0
  ),
  shekel = list(
    fn = shekel,
    lower = rep(0, 4),
    upper = rep(10, 4),
    initial = rep(1, 4),
    optimum = -10.5364098252
  ),
  hartman = list(
    fn = hartman,
    lower = rep(0, 6),
    upper = rep(1, 6),
    initial = rep(0.5, 6),
    optimum = -3.322368011415515
  )
)

sceua_duan_value <- function(problem) {
  set.seed(duan_control$seed)
  n <- length(problem$lower)

  sceua::sceua(
    fn = problem$fn,
    lower = problem$lower,
    upper = problem$upper,
    initial = problem$initial,
    max_evaluations = duan_control$max_evaluations,
    kstop = duan_control$kstop,
    pcento = duan_control$pcento,
    complexes = duan_control$complexes,
    points_per_complex = 2L * n + 1L,
    simplex_size = n + 1L,
    evolution_steps = 2L * n + 1L,
    min_complexes = duan_control$complexes,
    parameter_epsilon = duan_control$parameter_epsilon
  )$value
}

rtop_duan_value <- function(problem) {
  set.seed(duan_control$seed)
  n <- length(problem$lower)

  silent_rtop(
    problem$fn,
    pars = problem$initial,
    lower = problem$lower,
    upper = problem$upper,
    maxn = duan_control$max_evaluations,
    kstop = duan_control$kstop,
    pcento = duan_control$pcento,
    ngs = duan_control$complexes,
    npg = 2L * n + 1L,
    nps = n + 1L,
    nspl = 2L * n + 1L,
    mings = duan_control$complexes,
    peps = duan_control$parameter_epsilon,
    iniflg = 1
  )
}

soilhyp_duan_value <- function(problem) {
  set.seed(duan_control$seed)
  n <- length(problem$lower)

  silent_soilhyp(
    FUN = problem$fn,
    par = problem$initial,
    lower = problem$lower,
    upper = problem$upper,
    control = list(
      maxeval = duan_control$max_evaluations,
      ncomplex = duan_control$complexes,
      cce.iter = 2L * n + 1L,
      reltol = duan_control$pcento,
      tolsteps = duan_control$kstop,
      initsample = "latin"
    )
  )
}

safe_duan_value <- function(package, problem) {
  value <- tryCatch(
    switch(
      package,
      sceua = sceua_duan_value(problem),
      rtop = rtop_duan_value(problem),
      soilhyp = soilhyp_duan_value(problem)
    ),
    error = function(e) NA_real_
  )

  as.numeric(value)
}

# Number of timed runs per (problem, package). bench::mark runs each
# expression exactly this many times.
duan_iterations <- 5L

# Dependency-free cell visual: a fixed-width unicode bar for a scalar value
# scaled to a per-problem maximum. A positive sub-maximum value always gets
# at least one filled cell so the fastest run still shows a sliver.
bar_text <- function(value, max_value, width = 10L,
                     fill = "\u2588", empty = "\u2591") {
  if (is.na(value) || is.na(max_value) || max_value == 0) return("")
  n <- round(width * value / max_value)
  if (value > 0 && value < max_value) n <- max(n, 1L)
  n <- min(max(n, 0L), width)
  paste0(strrep(fill, n), strrep(empty, width - n))
}

fmt_time <- function(seconds) {
  if (is.na(seconds)) return("\u2014")
  if (seconds >= 1) return(sprintf("%.2f s", seconds))
  if (seconds >= 1e-3) return(sprintf("%.2f ms", seconds * 1e3))
  sprintf("%.0f \u00b5s", seconds * 1e6)
}

# Compute the best value for every (problem, package) pair. `safe_duan_value`
# returns NA when a package errors, so a failed run still has a row.
packages <- c("sceua", "rtop", "soilhyp")
duan_values <- do.call(rbind, lapply(names(duan_problems), function(name) {
  problem <- duan_problems[[name]]
  values <- vapply(packages, safe_duan_value, numeric(1), problem = problem)
  data.frame(
    problem = name,
    package = packages,
    value = values,
    abs_error = abs(values - problem$optimum),
    stringsAsFactors = FALSE
  )
}))

# Time every pair. `check = FALSE` disables bench's result-equality check; the
# tables below compare values to the optimum explicitly. A pair that errors is
# recorded with `NA` median time rather than aborting the vignette.
benchmark_duan_row <- function(name, package, problem) {
  exprs <- list(
    sceua = quote(sceua_duan_value(problem)),
    rtop = quote(rtop_duan_value(problem)),
    soilhyp = quote(soilhyp_duan_value(problem))
  )
  set.seed(duan_control$seed)
  res <- tryCatch(
    bench::mark(
      eval(exprs[[package]]),
      iterations = duan_iterations,
      relative = FALSE,
      check = FALSE
    ),
    error = function(e) NULL
  )
  median_time <- if (is.null(res)) NA_real_ else as.numeric(res$median[[1]])
  data.frame(
    problem = name,
    package = package,
    median_time = median_time,
    stringsAsFactors = FALSE
  )
}

duan_timing <- do.call(rbind, lapply(names(duan_problems), function(name) {
  problem <- duan_problems[[name]]
  do.call(rbind, lapply(packages, function(pkg) {
    benchmark_duan_row(name, pkg, problem)
  }))
}))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.

duan_summary <- merge(
  duan_values,
  duan_timing,
  by = c("problem", "package"),
  sort = FALSE
)
duan_summary <- duan_summary[order(
  match(duan_summary$problem, names(duan_problems)),
  match(duan_summary$package, packages)
), ]
duan_summary$optimum <- vapply(
  duan_summary$problem,
  function(n) duan_problems[[n]]$optimum,
  numeric(1)
)

fmt_value <- function(x) ifelse(is.na(x), "\u2014", sprintf("%.6g", x))
fmt_err <- function(x) ifelse(is.na(x), "\u2014", sprintf("%.3e", x))

# Per-problem table. The Timing bar is scaled to the slowest run within the
# problem, so a full bar marks the slowest package and shorter bars mark
# faster ones. A package that errors shows em dashes.
duan_problem_table <- function(name) {
  rows <- duan_summary[duan_summary$problem == name, , drop = FALSE]
  max_time <- if (all(is.na(rows$median_time))) {
    NA_real_
  } else {
    max(rows$median_time, na.rm = TRUE)
  }
  timing_bar <- vapply(
    rows$median_time, bar_text, character(1), max_value = max_time
  )

  data.frame(
    Package = rows$package,
    `Best value` = fmt_value(rows$value),
    Optimum = sprintf("%.6g", rows$optimum[1]),
    MAE = fmt_err(rows$abs_error),
    `Median time` = vapply(rows$median_time, fmt_time, character(1)),
    Timing = ifelse(timing_bar == "", "\u2014", timing_bar),
    check.names = FALSE,
    stringsAsFactors = FALSE
  )
}

duan_problem_caption <- function(name) {
  p <- duan_problems[[name]]
  sprintf(
    "%s: bounds [%s], optimum %g, %d parameters",
    name,
    paste(sprintf("[%g, %g]", p$lower, p$upper), collapse = ", "),
    p$optimum,
    length(p$lower)
  )
}
for (name in names(duan_problems)) {
  cat("###", name, "\n\n")
  cat(knitr::kable(
    duan_problem_table(name),
    align = c("l", "r", "r", "r", "r", "l"),
    caption = duan_problem_caption(name)
  ), sep = "\n")
  cat("\n\n")
}

goldstein_price

goldstein_price: bounds [[-2, 2], [-2, 2]], optimum 3, 2 parameters
Package Best value Optimum MAE Median time Timing
sceua 3 3 2.665e-15 3.01 ms █░░░░░░░░░
rtop 3 3 7.816e-14 989.85 ms ██████████
soilhyp 3 3 7.816e-14 149.47 ms ██░░░░░░░░

rosenbrock

rosenbrock: bounds [[-5, 5], [-5, 5]], optimum 0, 2 parameters
Package Best value Optimum MAE Median time Timing
sceua 1.44686e-18 0 1.447e-18 3.13 ms █░░░░░░░░░
rtop 0
soilhyp 0 0 0.000e+00 225.06 ms ██████████

six_hump_camelback

six_hump_camelback: bounds [[-5, 5], [-2, 8]], optimum -1.03163, 2 parameters
Package Best value Optimum MAE Median time Timing
sceua -1.03163 -1.03163 4.441e-16 2.84 ms █░░░░░░░░░
rtop -1.03163 -1.03163 1.023e-12 77.16 ms ███████░░░
soilhyp -1.03163 -1.03163 4.441e-16 118.42 ms ██████████

rastrigin_duan

rastrigin_duan: bounds [[-1, 1], [-1, 1]], optimum -2, 2 parameters
Package Best value Optimum MAE Median time Timing
sceua -2 -2 0.000e+00 3.04 ms █░░░░░░░░░
rtop -1.8789 -2 1.211e-01 126.72 ms ██████████
soilhyp -1.8789 -2 1.211e-01 59.85 ms █████░░░░░

griewank_duan

griewank_duan: bounds [[-600, 600], [-600, 600], [-600, 600], [-600, 600], [-600, 600], [-600, 600], [-600, 600], [-600, 600], [-600, 600], [-600, 600]], optimum 0, 10 parameters
Package Best value Optimum MAE Median time Timing
sceua 3.73966e-10 0 3.740e-10 23.14 ms █░░░░░░░░░
rtop 1.11022e-14 0 1.110e-14 2.12 s ██████████
soilhyp 0.0393384 0 3.934e-02 1.10 s █████░░░░░

shekel

shekel: bounds [[0, 10], [0, 10], [0, 10], [0, 10]], optimum -10.5364, 4 parameters
Package Best value Optimum MAE Median time Timing
sceua -10.5364 -10.5364 8.510e-09 75.35 ms █░░░░░░░░░
rtop -10.5364 -10.5364 8.527e-09 361.32 ms ███████░░░
soilhyp -10.5364 -10.5364 8.508e-09 509.09 ms ██████████

hartman

hartman: bounds [[0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1]], optimum -3.32237, 6 parameters
Package Best value Optimum MAE Median time Timing
sceua -3.32237 -3.32237 4.663e-14 83.83 ms █░░░░░░░░░
rtop -3.32237 -3.32237 3.868e-13 625.10 ms ███████░░░
soilhyp -3.20316 -3.32237 1.192e-01 904.93 ms ██████████