diff --git a/DESCRIPTION b/DESCRIPTION index 396b48ff6..9f141ab67 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: performance Title: Assessment of Regression Models Performance -Version: 0.19.0.7 +Version: 0.19.0.8 Authors@R: c(person(given = "Daniel", family = "Lüdecke", @@ -97,6 +97,7 @@ Suggests: brms, car, carData, + cluster, CompQuadForm, correlation (>= 0.8.8), cplm, @@ -130,6 +131,7 @@ Suggests: loo, MASS, Matrix, + matchednull (>= 0.2.2), mclogit, mclust, metadat, diff --git a/NAMESPACE b/NAMESPACE index af282acb9..1b65e8721 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -383,6 +383,7 @@ S3method(print,r2_mlm) S3method(print,r2_nakagawa) S3method(print,r2_nakagawa_by_group) S3method(print,r2_pseudo) +S3method(print,test_clusters) S3method(print,test_likelihoodratio) S3method(print,test_performance) S3method(print_html,check_group_variation) @@ -701,6 +702,7 @@ export(r2_zeroinflated) export(rmse) export(simulate_residuals) export(test_bf) +export(test_clusters) export(test_likelihoodratio) export(test_lrt) export(test_performance) diff --git a/NEWS.md b/NEWS.md index fbb3021c4..2906a3185 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,13 @@ ## Changes +* New function `test_clusters()` uses the `matchednull` package to compare a + selected cluster count with the counts obtained from matched-null data that + keep the observed margins and approximately the observed correlations. It + reports one of three outcomes (no detected excess, departure from the + reference model, below the reference range) and validates the return values + of the clustering function. + * `model_performance()` for mixed models now fits the null model only once when both R2 and ICC are requested, instead of once for each index. This applies to non-Gaussian models, for which both indices need the null model. @@ -202,6 +209,27 @@ in order of their degrees of freedom. `test_vuong()` and `test_wald()` now give a clear error for *lavaan* models instead of the same recursion (#427). +## Changes + +* `check_model()` gets a `show_smooth` argument. With `show_smooth = FALSE`, + the Linearity and Homogeneity of Variance panels draw neither the LOESS + smoother nor the cell means, and the Influential Observations panel draws no + LOESS smoother. This needs *see* newer than 0.14.2. Older versions ignore the + setting. + +* `check_model()` and `check_predictions()` now work for multivariate models + from package *brms*. Use the `response` argument to name the response to + check. Without `response`, both functions give an error that lists the names + of the responses (#908). + +* `check_model()` now shows the plot of influential observations for models + from package *glmmTMB* with a gaussian, poisson, binomial, Gamma, or `nbinom2` + family, and without zero-inflation or a dispersion formula. The plot uses hat + values and a one-step approximation of Cook's distance, so the model is not + refit. For other *glmmTMB* models, the plot is skipped, and with + `verbose = TRUE` a message names the reason. For the same models, + `plot(check_outliers())` shows this plot with the Cook's distances from the + refits (#95). * `check_collinearity()` uses a new method for the confidence intervals of VIF and tolerance. The method inverts the noncentral F distribution of the auxiliary regression of each term, as `MBESS::ci.R2()` does for fixed diff --git a/R/test_clusters.R b/R/test_clusters.R new file mode 100644 index 000000000..346e6189c --- /dev/null +++ b/R/test_clusters.R @@ -0,0 +1,326 @@ +#' Test a cluster count against matched-null data +#' +#' `test_clusters()` runs a cluster-count selector on the observed data and on +#' `iterations` matched-null twins of those data, and compares the observed +#' count with the counts the twins produce. The twins keep the observed values +#' of every variable exactly and reproduce the observed correlations +#' approximately, but contain no cluster structure by construction. The test +#' reports one of three outcomes: no detected excess, a departure from the +#' reference model, or a count below the reference range. The comparison is +#' implemented with [`matchednull::matched_null_test()`]. It asks a different +#' question from [`check_clusterstructure()`], which evaluates clustering +#' tendency against a spatial-uniformity null. +#' +#' @param x A numeric matrix or data frame with no missing or infinite values. +#' @param cluster_function A function that takes a numeric matrix and returns a +#' single finite number, typically the selected number of clusters. By +#' default, `mclust::Mclust()` selects the number of mixture components by +#' BIC. The function is called once on the observed data and once on every +#' twin, and its return value is checked each time (see 'Details'). +#' @param iterations Number of matched-null twins to evaluate. +#' @param n_max Maximum number of clusters considered by the default +#' `mclust::Mclust()` pipeline. Ignored when `cluster_function` is supplied. +#' @param standardize Logical. If `TRUE`, variables are standardized before the +#' observed and matched-null cluster counts are computed. +#' @param ... Additional arguments passed to +#' [`matchednull::matched_null_test()`], such as `copula`, `df`, `probs`, +#' `ridge`, or `parallel`. +#' +#' @return An object of class `"test_clusters"` and `"matched_null_test"`: a +#' list with the observed statistic (`real`), the twin statistics (`null`), +#' the reference interval (`interval`), the Monte Carlo exceedance +#' probability (`p_exceed`), the interval verdict (`within`), the number of +#' twins (`R`), and the copula family (`copula`, with `df` for the t copula). +#' The attribute `"outcome"` is `"within"`, `"above"`, or `"below"`: the +#' position of the observed statistic relative to the reference interval. +#' The attributes `"standardize"` and `"n_max"` record the preprocessing and +#' the maximum number of components of the default pipeline (`n_max` is +#' `NULL` when `cluster_function` is supplied). +#' +#' @details +#' The default cluster-count pipeline fits Gaussian mixture models with one to +#' `n_max` components using `mclust::Mclust()` and returns the BIC-selected +#' number of components. A user-supplied `cluster_function` receives the same +#' preprocessed data matrix as the default pipeline. Its return value is +#' checked on the observed data and on every twin: it must be a single finite +#' number, and any other value (`NA`, a character string, a factor, a logical +#' value, a vector of length two, `NULL`) is an error. The check runs here, on +#' the raw return value, so that the error names `cluster_function` and what it +#' returned. +#' +#' ## Reference model +#' +#' Each twin is drawn from a Gaussian copula whose latent correlation matrix +#' is the observed correlation matrix. Every column of a twin contains exactly +#' the observed values of that variable, rearranged in the rank order of the +#' copula draw. The margins are therefore reproduced exactly, but the +#' correlations only approximately: non-normal margins change the Pearson +#' correlations of the output relative to the latent correlation matrix. With +#' `copula = "t"`, the draw comes from a t copula instead, a sensitivity +#' variant with heavier-tailed dependence. Rerunning with `copula = "t"` shows +#' whether the outcome changes under that reference model; it does not by +#' itself establish or rule out clusters. +#' +#' ## Outcomes +#' +#' The observed count is compared with the interval between the `probs` +#' quantiles of the twin counts (by default the 2.5% and 97.5% quantiles). +#' +#' - A count above the interval is a departure from the specified reference +#' model. The test does not settle what kind of departure it is, and the +#' result is not evidence that clusters exist: typeless data with joint +#' asymmetry, for example, can exceed the reference model as well. +#' - A count inside the interval is no detected excess for the statistic that +#' was compared, the cluster count. It is not evidence that there are no +#' clusters, and it does not establish that the joint distribution belongs +#' to the reference family. +#' - A count below the interval is a diagnostic signal, not a verdict. It +#' calls for checking the pipeline and the reference before any +#' interpretation: whether the fits converged, whether the twins reproduce +#' the intended margins and correlations, and whether data and twins were +#' compared at the same sample size with the same statistic. +#' +#' `p_exceed` is a Monte Carlo exceedance probability under a reference model +#' fitted to the same data. It equals `(1 + k) / (iterations + 1)`, where `k` +#' is the number of twins whose count is at least the observed count. It is +#' not an exact finite-sample p-value. +#' +#' ## Before running the test +#' +#' Inspect the margins for pronounced multimodality first, for example with +#' [`check_multimodal()`] on single variables. Groups that differ only in +#' their means are detected at moderate separations. At large separations the +#' separation shows in the margins themselves. The twins keep the margins +#' exactly, so the pipeline can select as many clusters in the twins (which +#' contain no groups) as in the data, and the comparison may then report no +#' detected excess. This behaviour was observed in the simulation +#' constructions of Meng (2026); it is not asserted as a general rule. Whether +#' the joint distribution has clustering tendency is a different question +#' again; see [`check_clusterstructure()`], which uses a spatial-uniformity +#' null. +#' +#' @examplesIf all(insight::check_if_installed(c("matchednull", "mclust", "cluster"), quietly = TRUE)) +#' \donttest{ +#' set.seed(42) +#' test_clusters(iris[, 1:4], iterations = 19, n_max = 4) +#' +#' # Any pipeline that returns one number can be tested, for example k-means +#' # with the number of clusters chosen by the mean silhouette width. +#' select_k <- function(data) { +#' ks <- 2:5 +#' sil <- vapply(ks, function(k) { +#' cl <- stats::kmeans(data, centers = k, nstart = 5)$cluster +#' mean(cluster::silhouette(cl, stats::dist(data))[, "sil_width"]) +#' }, numeric(1)) +#' ks[which.max(sil)] +#' } +#' test_clusters(iris[, 1:4], cluster_function = select_k, iterations = 19) +#' } +#' +#' @references +#' - Meng, M. (2026). matchednull: Matched-null tests for cluster-count claims. +#' R package version 0.2.2. +#' +#' - Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse +#' Test of Categorical Versus Continuous Personality Structure. +#' Preregistration, Open Science Framework. \doi{10.17605/OSF.IO/2EKCG} +#' +#' @seealso [`matchednull::matched_null_test()`], +#' [`check_clusterstructure()`], [`check_multimodal()`] +#' @export +test_clusters <- function( + x, + cluster_function = NULL, + iterations = 200, + n_max = 10, + standardize = TRUE, + ... +) { + insight::check_if_installed("matchednull", minimum_version = "0.2.2") + + if (!is.logical(standardize) || length(standardize) != 1L || is.na(standardize)) { + insight::format_error("`standardize` must be `TRUE` or `FALSE`.") + } + if (!is.null(cluster_function) && !is.function(cluster_function)) { + insight::format_error("`cluster_function` must be a function or `NULL`.") + } + uses_default <- is.null(cluster_function) + if (!.is_positive_integer(iterations)) { + insight::format_error("`iterations` must be a positive integer.") + } + if (uses_default && !.is_positive_integer(n_max)) { + insight::format_error("`n_max` must be a positive integer.") + } + + x <- .validate_test_clusters_data(x) + x <- .standardize_test_clusters_data(x, standardize) + + if (uses_default) { + insight::check_if_installed("mclust") + # `mclust::Mclust()` rewrites its call to `mclustBIC` and evaluates it in + # the caller's frame, so the symbol must be visible from this function + # when mclust is not attached. Do not remove. + mclustBIC <- mclust::mclustBIC + max_components <- min(as.integer(n_max), nrow(x) - 1L) + cluster_function <- function(data) { + mclust::Mclust( + data, + G = seq_len(max_components), + verbose = FALSE + )$G + } + } + + out <- matchednull::matched_null_test( + x, + cluster_fn = .checked_cluster_function(cluster_function), + R = as.integer(iterations), + ... + ) + + attr(out, "standardize") <- standardize + attr(out, "n_max") <- if (uses_default) max_components else NULL + attr(out, "outcome") <- .test_clusters_outcome(out$real, out$interval) + class(out) <- c("test_clusters", class(out)) + out +} + + +#' @export +print.test_clusters <- function(x, digits = 3, ...) { + outcome <- .test_clusters_outcome(x$real, x$interval) + outcome_text <- switch( + outcome, + within = "No detected excess: the count lies within the reference interval.", + above = paste0( + "Departure from the reference model: the count exceeds the interval.\n", + "The test does not settle what kind of departure." + ), + below = "Below the reference range: a diagnostic signal, not a verdict." + ) + outcome_color <- switch(outcome, within = "green", above = "yellow", below = "yellow") + + reference <- if (is.null(x$copula) || x$copula == "gaussian") { + "Gaussian copula" + } else { + sprintf( + "t copula (df = %s)", + insight::format_value(x$df, digits = digits, protect_integers = TRUE) + ) + } + + interval <- sprintf( + "[%s]", + toString(insight::format_value(x$interval, digits = digits, protect_integers = TRUE)) + ) + if (!is.null(names(x$interval))) { + interval <- sprintf( + "%s (%s quantiles of the twins)", + interval, + paste(names(x$interval), collapse = " and ") + ) + } + + insight::print_color("# Matched-null test of the cluster count\n\n", "blue") + # the exceedance counts twins whose statistic is at least the observed one + cat(sprintf( + " Observed statistic: %s\n", + insight::format_value(x$real, digits = digits, protect_integers = TRUE) + )) + cat(sprintf(" Reference interval: %s\n", interval)) + cat(sprintf( + " MC exceedance (twins >= observed): %s\n", + insight::format_value(x$p_exceed, digits = digits) + )) + cat(sprintf( + " Reference model: %s, %s matched-null twins\n", + reference, + insight::format_value(x$R, protect_integers = TRUE) + )) + cat("\n") + insight::print_color(paste0(outcome_text, "\n"), outcome_color) + invisible(x) +} + + +# matchednull calls the clustering function once on the observed data and once +# per twin. The wrapper checks the raw return value itself, before any +# coercion, so that the error speaks of `cluster_function`. +.checked_cluster_function <- function(cluster_function) { + function(data) { + value <- cluster_function(data) + if (!is.numeric(value) || length(value) != 1L || !is.finite(value)) { + insight::format_error(sprintf( + "`cluster_function` must return a single finite number, but returned %s.", + .describe_cluster_value(value) + )) + } + unname(as.numeric(value)) + } +} + + +.describe_cluster_value <- function(value) { + if (is.numeric(value) && length(value) == 1L) { + # a number that is not finite: NA, NaN, Inf or -Inf + return(paste0("`", format(value), "`")) + } + sprintf("%s of length %d", class(value)[1], length(value)) +} + + +.test_clusters_outcome <- function(real, interval) { + if (real > interval[2]) { + "above" + } else if (real < interval[1]) { + "below" + } else { + "within" + } +} + + +.validate_test_clusters_data <- function(x) { + if (!is.matrix(x) && !is.data.frame(x)) { + insight::format_error("`x` must be a numeric matrix or data frame.") + } + if (is.data.frame(x) && !all(vapply(x, is.numeric, logical(1)))) { + insight::format_error("All columns in `x` must be numeric.") + } + + x <- as.matrix(x) + if (!is.numeric(x)) { + insight::format_error("`x` must contain only numeric values.") + } + if (nrow(x) < 2L || ncol(x) < 1L) { + insight::format_error("`x` must contain at least two rows and one column.") + } + if (anyNA(x) || !all(is.finite(x))) { + insight::format_error("`x` must not contain missing or infinite values.") + } + x +} + + +.standardize_test_clusters_data <- function(x, standardize) { + if (!standardize) { + return(x) + } + + column_sd <- apply(x, 2L, stats::sd) + if (!all(is.finite(column_sd)) || any(column_sd == 0)) { + insight::format_error( + "`x` cannot contain constant columns when `standardize = TRUE`." + ) + } + scale(x) +} + + +.is_positive_integer <- function(x) { + is.numeric(x) && + length(x) == 1L && + !is.na(x) && + x >= 1L && + x %% 1 == 0 +} diff --git a/inst/WORDLIST b/inst/WORDLIST index 02b0eed33..187a34423 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -156,6 +156,7 @@ Matuschek McElreath McKelvey Mehrvarz +Meng Merkle Methoden Michalos @@ -328,6 +329,7 @@ lm lme lmrob lmtest +matchednull mattansb metafor mfx @@ -342,6 +344,7 @@ modelfit modelling models’ multicollinearity +multimodality multimodel multiresponse multivariable @@ -382,6 +385,7 @@ subscale subscales theoreritcal tidymodels +typeless unadjusted und underfitted diff --git a/man/test_clusters.Rd b/man/test_clusters.Rd new file mode 100644 index 000000000..b01683071 --- /dev/null +++ b/man/test_clusters.Rd @@ -0,0 +1,158 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/test_clusters.R +\name{test_clusters} +\alias{test_clusters} +\title{Test a cluster count against matched-null data} +\usage{ +test_clusters( + x, + cluster_function = NULL, + iterations = 200, + n_max = 10, + standardize = TRUE, + ... +) +} +\arguments{ +\item{x}{A numeric matrix or data frame with no missing or infinite values.} + +\item{cluster_function}{A function that takes a numeric matrix and returns a +single finite number, typically the selected number of clusters. By +default, \code{mclust::Mclust()} selects the number of mixture components by +BIC. The function is called once on the observed data and once on every +twin, and its return value is checked each time (see 'Details').} + +\item{iterations}{Number of matched-null twins to evaluate.} + +\item{n_max}{Maximum number of clusters considered by the default +\code{mclust::Mclust()} pipeline. Ignored when \code{cluster_function} is supplied.} + +\item{standardize}{Logical. If \code{TRUE}, variables are standardized before the +observed and matched-null cluster counts are computed.} + +\item{...}{Additional arguments passed to +\code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}, such as \code{copula}, \code{df}, \code{probs}, +\code{ridge}, or \code{parallel}.} +} +\value{ +An object of class \code{"test_clusters"} and \code{"matched_null_test"}: a +list with the observed statistic (\code{real}), the twin statistics (\code{null}), +the reference interval (\code{interval}), the Monte Carlo exceedance +probability (\code{p_exceed}), the interval verdict (\code{within}), the number of +twins (\code{R}), and the copula family (\code{copula}, with \code{df} for the t copula). +The attribute \code{"outcome"} is \code{"within"}, \code{"above"}, or \code{"below"}: the +position of the observed statistic relative to the reference interval. +The attributes \code{"standardize"} and \code{"n_max"} record the preprocessing and +the maximum number of components of the default pipeline (\code{n_max} is +\code{NULL} when \code{cluster_function} is supplied). +} +\description{ +\code{test_clusters()} runs a cluster-count selector on the observed data and on +\code{iterations} matched-null twins of those data, and compares the observed +count with the counts the twins produce. The twins keep the observed values +of every variable exactly and reproduce the observed correlations +approximately, but contain no cluster structure by construction. The test +reports one of three outcomes: no detected excess, a departure from the +reference model, or a count below the reference range. The comparison is +implemented with \code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}. It asks a different +question from \code{\link[=check_clusterstructure]{check_clusterstructure()}}, which evaluates clustering +tendency against a spatial-uniformity null. +} +\details{ +The default cluster-count pipeline fits Gaussian mixture models with one to +\code{n_max} components using \code{mclust::Mclust()} and returns the BIC-selected +number of components. A user-supplied \code{cluster_function} receives the same +preprocessed data matrix as the default pipeline. Its return value is +checked on the observed data and on every twin: it must be a single finite +number, and any other value (\code{NA}, a character string, a factor, a logical +value, a vector of length two, \code{NULL}) is an error. The check runs here, on +the raw return value, so that the error names \code{cluster_function} and what it +returned. +\subsection{Reference model}{ + +Each twin is drawn from a Gaussian copula whose latent correlation matrix +is the observed correlation matrix. Every column of a twin contains exactly +the observed values of that variable, rearranged in the rank order of the +copula draw. The margins are therefore reproduced exactly, but the +correlations only approximately: non-normal margins change the Pearson +correlations of the output relative to the latent correlation matrix. With +\code{copula = "t"}, the draw comes from a t copula instead, a sensitivity +variant with heavier-tailed dependence. Rerunning with \code{copula = "t"} shows +whether the outcome changes under that reference model; it does not by +itself establish or rule out clusters. +} + +\subsection{Outcomes}{ + +The observed count is compared with the interval between the \code{probs} +quantiles of the twin counts (by default the 2.5\% and 97.5\% quantiles). +\itemize{ +\item A count above the interval is a departure from the specified reference +model. The test does not settle what kind of departure it is, and the +result is not evidence that clusters exist: typeless data with joint +asymmetry, for example, can exceed the reference model as well. +\item A count inside the interval is no detected excess for the statistic that +was compared, the cluster count. It is not evidence that there are no +clusters, and it does not establish that the joint distribution belongs +to the reference family. +\item A count below the interval is a diagnostic signal, not a verdict. It +calls for checking the pipeline and the reference before any +interpretation: whether the fits converged, whether the twins reproduce +the intended margins and correlations, and whether data and twins were +compared at the same sample size with the same statistic. +} + +\code{p_exceed} is a Monte Carlo exceedance probability under a reference model +fitted to the same data. It equals \code{(1 + k) / (iterations + 1)}, where \code{k} +is the number of twins whose count is at least the observed count. It is +not an exact finite-sample p-value. +} + +\subsection{Before running the test}{ + +Inspect the margins for pronounced multimodality first, for example with +\code{\link[=check_multimodal]{check_multimodal()}} on single variables. Groups that differ only in +their means are detected at moderate separations. At large separations the +separation shows in the margins themselves. The twins keep the margins +exactly, so the pipeline can select as many clusters in the twins (which +contain no groups) as in the data, and the comparison may then report no +detected excess. This behaviour was observed in the simulation +constructions of Meng (2026); it is not asserted as a general rule. Whether +the joint distribution has clustering tendency is a different question +again; see \code{\link[=check_clusterstructure]{check_clusterstructure()}}, which uses a spatial-uniformity +null. +} +} +\examples{ +\dontshow{if (all(insight::check_if_installed(c("matchednull", "mclust", "cluster"), quietly = TRUE))) withAutoprint(\{ # examplesIf} +\donttest{ +set.seed(42) +test_clusters(iris[, 1:4], iterations = 19, n_max = 4) + +# Any pipeline that returns one number can be tested, for example k-means +# with the number of clusters chosen by the mean silhouette width. +select_k <- function(data) { + ks <- 2:5 + sil <- vapply(ks, function(k) { + cl <- stats::kmeans(data, centers = k, nstart = 5)$cluster + mean(cluster::silhouette(cl, stats::dist(data))[, "sil_width"]) + }, numeric(1)) + ks[which.max(sil)] +} +test_clusters(iris[, 1:4], cluster_function = select_k, iterations = 19) +} +\dontshow{\}) # examplesIf} +} +\references{ +\itemize{ +\item Meng, M. (2026). matchednull: Matched-null tests for cluster-count claims. +R package version 0.2.2. \url{https://CRAN.R-project.org/package=matchednull} +\item Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse +Test of Categorical Versus Continuous Personality Structure. +Preregistration, Open Science Framework. \doi{10.17605/OSF.IO/2EKCG} +} +} +\seealso{ +\code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}, +\code{\link[=check_clusterstructure]{check_clusterstructure()}}, \code{\link[=check_multimodal]{check_multimodal()}} +} diff --git a/tests/testthat/test-test_clusters.R b/tests/testthat/test-test_clusters.R new file mode 100644 index 000000000..3aed9fbae --- /dev/null +++ b/tests/testthat/test-test_clusters.R @@ -0,0 +1,273 @@ +test_that("test_clusters supports a custom cluster function", { + skip_if_not_installed("matchednull", minimum_version = "0.2.2") + + set.seed(12) + x <- matrix(rnorm(120), ncol = 3) + out <- test_clusters( + x, + cluster_function = function(data) 2, + iterations = 9 + ) + + expect_s3_class(out, "test_clusters") + expect_s3_class(out, "matched_null_test") + expect_identical(out$real, 2) + expect_length(out$null, 9) + expect_true(out$within) + expect_identical(attr(out, "outcome"), "within") + expect_true(attr(out, "standardize")) + expect_null(attr(out, "n_max")) + expect_output(print(out), "Matched-null test of the cluster count") + expect_output(print(out), "MC exceedance (twins >= observed):", fixed = TRUE) + printed <- capture.output(print(out)) + p_line <- grep("MC exceedance", printed, value = TRUE, fixed = TRUE) + p_shown <- as.numeric(sub(".*:\\s+", "", p_line)) + expect_equal(p_shown, (1 + sum(out$null >= out$real)) / (out$R + 1), tolerance = 5e-4) + expect_output(print(out), "Gaussian copula, 9 matched-null twins", fixed = TRUE) + expect_output( + print(out), + "No detected excess: the count lies within the reference interval.", + fixed = TRUE + ) + + out_t <- test_clusters( + x, + cluster_function = function(data) 2, + iterations = 3, + copula = "t", + df = 4 + ) + expect_identical(out_t$copula, "t") + expect_identical(out_t$df, 4) + expect_output(print(out_t), "t copula (df = 4), 3 matched-null twins", fixed = TRUE) +}) + + +test_that("test_clusters reports counts above and below the reference range", { + skip_if_not_installed("matchednull", minimum_version = "0.2.2") + + set.seed(5) + x <- matrix(rnorm(120), ncol = 3) + # matchednull evaluates the observed data first and the twins afterwards + first_then <- function(first, rest) { + calls <- 0 + function(data) { + calls <<- calls + 1 + if (calls == 1) first else rest + } + } + + above <- test_clusters(x, cluster_function = first_then(5, 2), iterations = 9) + expect_identical(above$real, 5) + expect_false(above$within) + expect_identical(attr(above, "outcome"), "above") + expect_output( + print(above), + "Departure from the reference model: the count exceeds the interval.", + fixed = TRUE + ) + expect_output( + print(above), + "The test does not settle what kind of departure.", + fixed = TRUE + ) + + below <- test_clusters(x, cluster_function = first_then(1, 2), iterations = 9) + expect_identical(below$real, 1) + expect_false(below$within) + expect_identical(attr(below, "outcome"), "below") + expect_output( + print(below), + "Below the reference range: a diagnostic signal, not a verdict.", + fixed = TRUE + ) +}) + + +test_that("test_clusters is reproducible from a seed", { + skip_if_not_installed("matchednull", minimum_version = "0.2.2") + + set.seed(3) + x <- matrix(rnorm(180), ncol = 3) + cluster_summary <- function(data) sum(data[1:5, 1]) + + set.seed(11) + out1 <- test_clusters(x, cluster_summary, iterations = 9) + set.seed(11) + out2 <- test_clusters(x, cluster_summary, iterations = 9) + + expect_identical(out1$null, out2$null) + expect_gt(stats::var(out1$null), 0) +}) + + +test_that("default mclust pipeline distinguishes null and positive controls", { + skip_if_not_installed("matchednull", minimum_version = "0.2.2") + skip_if_not_installed("mclust") + + set.seed(42) + x_null <- matrix(rnorm(150 * 4), 150, 4) %*% + chol(diag(4) * 0.5 + 0.5) + set.seed(7) + null_result <- test_clusters(x_null, iterations = 9, n_max = 4) + expect_true(null_result$within) + expect_identical(attr(null_result, "outcome"), "within") + expect_identical(attr(null_result, "n_max"), 4L) + + set.seed(42) + group <- sample.int(2, 150, replace = TRUE) + x_positive <- matrix(rnorm(150 * 4), 150, 4) + positive_correlation <- chol(matrix(c(1, 0.9, 0.9, 1), 2, 2)) + negative_correlation <- chol(matrix(c(1, -0.9, -0.9, 1), 2, 2)) + x_positive[group == 1, 1:2] <- x_positive[group == 1, 1:2] %*% positive_correlation + x_positive[group == 1, 3:4] <- x_positive[group == 1, 3:4] %*% positive_correlation + x_positive[group == 2, 1:2] <- x_positive[group == 2, 1:2] %*% negative_correlation + x_positive[group == 2, 3:4] <- x_positive[group == 2, 3:4] %*% negative_correlation + + set.seed(7) + positive_result <- test_clusters(x_positive, iterations = 9, n_max = 4) + expect_gt(positive_result$real, positive_result$interval[2]) + expect_identical(attr(positive_result, "outcome"), "above") +}) + + +test_that("default mclust pipeline works in a clean session without attaching mclust", { + # `mclust::Mclust()` evaluates `mclustBIC` in the caller's frame; this only + # fails when mclust is not attached, so it has to be checked in a fresh + # process against the installed package. + skip_on_cran() + skip_if_not_installed("matchednull", minimum_version = "0.2.2") + skip_if_not_installed("mclust") + + # the child needs every library the parent sees, not only the first one: + # dependencies such as insight often live in a different library path + libs <- deparse1(.libPaths(), collapse = "") + code <- paste( + sprintf(".libPaths(c(%s, .libPaths()));", libs), + "loaded <- tryCatch(", + " {", + " loadNamespace('performance');", + " TRUE", + " },", + " error = function(e) gsub('[\\r\\n]+', ' ', conditionMessage(e))", + ");", + "if (!isTRUE(loaded)) {", + " cat(paste0('LOADFAIL=', loaded, '\\n'))", + "} else {", + " version <- as.character(utils::packageVersion('performance'));", + " if (!'test_clusters' %in% getNamespaceExports('performance')) {", + " cat(paste0('NOEXPORT;VERSION=', version, '\\n'))", + " } else {", + " set.seed(1);", + " x <- matrix(rnorm(180), 60, 3) %*% chol(diag(3) * 0.5 + 0.5);", + " out <- performance::test_clusters(x, iterations = 3, n_max = 3);", + " cat(paste0(", + " 'ATTACHED=', 'package:mclust' %in% search(),", + " ';REAL=', out$real, ';VERSION=', version, '\\n'", + " ))", + " }", + "}" + ) + output <- system2( + file.path(R.home("bin"), "Rscript"), + c("--vanilla", "-e", shQuote(code)), + stdout = TRUE, + stderr = TRUE + ) + result <- grep("^(LOADFAIL=|NOEXPORT;|ATTACHED=)", output, value = TRUE) + reported <- paste(output, collapse = "\n") + + if (length(result) != 1L) { + # neither a usable result nor a reason to skip: this must not pass quietly + fail(paste0("Unexpected output from the clean session:\n", reported)) + } else if (startsWith(result, "NOEXPORT;")) { + # an older performance is installed, so the fix cannot be checked here + skip(paste0( + "the installed performance (version ", + sub("^NOEXPORT;VERSION=", "", result), + ") does not export test_clusters()" + )) + } else if (startsWith(result, "LOADFAIL=")) { + fail(paste0( + "The clean session could not load performance:\n", + reported + )) + } else { + expect_match(result, "^ATTACHED=FALSE;REAL=") + real <- suppressWarnings(as.numeric( + sub("^ATTACHED=FALSE;REAL=([^;]+);VERSION=.*$", "\\1", result) + )) + expect_true(is.finite(real)) + } +}) + + +test_that("test_clusters validates the raw return value of cluster_function", { + skip_if_not_installed("matchednull", minimum_version = "0.2.2") + + set.seed(8) + x <- matrix(rnorm(60), ncol = 2) + returns <- function(value) function(data) value + # error messages are wrapped to the console width, so match across line breaks + not_finite <- "single\\s+finite\\s+number" + + expect_error(test_clusters(x, returns(NA_real_), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(Inf), iterations = 3), not_finite) + expect_error(test_clusters(x, returns("2"), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(factor(2)), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(TRUE), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(c(2, 3)), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(NULL), iterations = 3), not_finite) + expect_error( + test_clusters(x, returns("2"), iterations = 3), + "character\\s+of\\s+length\\s+1" + ) + expect_error( + test_clusters(x, returns(c(2, 3)), iterations = 3), + "numeric\\s+of\\s+length\\s+2" + ) + expect_error( + test_clusters(x, returns(NULL), iterations = 3), + "NULL\\s+of\\s+length\\s+0" + ) + expect_error(test_clusters(x, returns(NA_real_), iterations = 3), "`NA`") + + # the check also runs on every twin + calls <- 0 + na_on_twins <- function(data) { + calls <<- calls + 1 + if (calls == 1) 2 else NA_real_ + } + expect_error(test_clusters(x, na_on_twins, iterations = 3), not_finite) + + # errors raised inside cluster_function propagate unchanged + calls <- 0 + fails_on_twin <- function(data) { + calls <<- calls + 1 + if (calls > 1) { + stop("twin failure", call. = FALSE) + } + 2 + } + expect_error(test_clusters(x, fails_on_twin, iterations = 3), "twin failure") + + # a length-one integer or 1x1 matrix is accepted and coerced to a plain number + out <- test_clusters(x, returns(2L), iterations = 3) + expect_identical(out$real, 2) + out <- test_clusters(x, returns(matrix(2)), iterations = 3) + expect_identical(out$null, c(2, 2, 2)) +}) + + +test_that("test_clusters validates its inputs", { + skip_if_not_installed("matchednull", minimum_version = "0.2.2") + + x <- matrix(rnorm(40), ncol = 2) + expect_error(test_clusters(1:10), "matrix or data frame") + expect_error(test_clusters(data.frame(x = 1:3, y = letters[1:3])), "numeric") + expect_error(test_clusters(cbind(x, NA_real_)), "missing or infinite") + expect_error(test_clusters(cbind(x, 1)), "constant columns") + expect_error(test_clusters(x, cluster_function = 1), "must be a function") + expect_error(test_clusters(x, iterations = 1.5), "positive integer") + expect_error(test_clusters(x, n_max = 0), "positive integer") +})