From 4770559b6d273a181c9a2051c585c9eea10b2c08 Mon Sep 17 00:00:00 2001 From: Miura Meng Date: Sat, 22 Aug 2026 05:59:24 -0400 Subject: [PATCH 1/7] Add matched-null cluster test --- DESCRIPTION | 1 + NAMESPACE | 1 + NEWS.md | 4 + R/test_clusters.R | 153 ++++++++++++++++++++++++++++ man/test_clusters.Rd | 81 +++++++++++++++ tests/testthat/test-test_clusters.R | 89 ++++++++++++++++ 6 files changed, 329 insertions(+) create mode 100644 R/test_clusters.R create mode 100644 man/test_clusters.Rd create mode 100644 tests/testthat/test-test_clusters.R diff --git a/DESCRIPTION b/DESCRIPTION index 77bf2fd58..e9e4634ea 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -126,6 +126,7 @@ Suggests: loo, MASS, Matrix, + matchednull (>= 0.2.1), mclogit, mclust, metadat, diff --git a/NAMESPACE b/NAMESPACE index 3f6718881..6b60377c3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -690,6 +690,7 @@ export(r2_zeroinflated) export(rmse) export(simulate_residuals) export(test_bf) +export(test_clusters) export(test_likelihoodratio) export(test_lrt) export(test_performance) diff --git a/NEWS.md b/NEWS.md index bda1be394..e90648ce6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,10 @@ ## Changes +* New function `test_clusters()` uses the `matchednull` package to test whether + a selected cluster count exceeds what matched-null data with the same margins + and correlations would produce. + * `check_group_variation()` now returns a numeric effect size of the grouping variable's predictive association strength. diff --git a/R/test_clusters.R b/R/test_clusters.R new file mode 100644 index 000000000..74697c194 --- /dev/null +++ b/R/test_clusters.R @@ -0,0 +1,153 @@ +#' Test a clustering result against matched nulls +#' +#' `test_clusters()` compares the number of clusters selected from observed data +#' with the numbers selected from matched-null versions of those data. The null +#' data preserve the observed marginal distributions and correlation structure +#' but contain no cluster structure by construction. This complements +#' [`check_clusterstructure()`], which evaluates clustering tendency against a +#' spatial-uniformity null. The matched-null comparison is implemented with +#' [`matchednull::matched_null_test()`]. +#' +#' @param x A numeric matrix or data frame with no missing or infinite values. +#' @param cluster_function A function that takes a numeric matrix and returns one +#' number. By default, `mclust::Mclust()` selects the number of mixture +#' components by BIC. +#' @param iterations Number of matched-null data sets to evaluate. +#' @param n_max Maximum number of clusters considered by the default +#' `mclust::Mclust()` pipeline. Ignored when `cluster_function` is supplied. +#' @param standardize Logical. If `TRUE`, variables are standardized before the +#' observed and matched-null cluster counts are computed. +#' @param ... Additional arguments passed to +#' [`matchednull::matched_null_test()`], such as `copula`, `df`, `probs`, +#' `ridge`, or `parallel`. +#' +#' @return An object of class `"test_clusters"` and `"matched_null_test"`. It +#' contains the observed statistic (`real`), matched-null statistics (`null`), +#' null interval (`interval`), one-sided Monte Carlo p-value (`p_exceed`), and +#' interval verdict (`within`). +#' +#' @details +#' The default cluster-count pipeline fits Gaussian mixture models for one to +#' `n_max` components and returns the BIC-selected count. A custom +#' `cluster_function` receives the same preprocessed data as the default +#' pipeline and must return one non-missing number. +#' +#' The Gaussian matched null tests for structure beyond the observed margins +#' and correlations; rejection alone does not establish the existence of +#' discrete types. When tail dependence is plausible, rerun the test with +#' `copula = "t"` as a sensitivity analysis. +#' +#' @examplesIf requireNamespace("matchednull", quietly = TRUE) && requireNamespace("mclust", quietly = TRUE) +#' \donttest{ +#' set.seed(42) +#' test_clusters(iris[, 1:4], iterations = 19, n_max = 4) +#' +#' # Any scalar-returning clustering pipeline can be tested. +#' pick_two <- function(data) 2 +#' test_clusters(iris[, 1:4], cluster_function = pick_two, iterations = 19) +#' } +#' +#' @references +#' Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse +#' Test of Categorical Versus Continuous Personality Structure. Manuscript +#' under review. \doi{10.17605/OSF.IO/2EKCG} +#' +#' @seealso [`matchednull::matched_null_test()`], +#' [`check_clusterstructure()`] +#' @export +test_clusters <- function( + x, + cluster_function = NULL, + iterations = 200, + n_max = 10, + standardize = TRUE, + ... +) { + insight::check_if_installed("matchednull", minimum_version = "0.2.1") + + if (!is.logical(standardize) || length(standardize) != 1L || is.na(standardize)) { + insight::format_error("`standardize` must be `TRUE` or `FALSE`.") + } + if (!is.null(cluster_function) && !is.function(cluster_function)) { + insight::format_error("`cluster_function` must be a function or `NULL`.") + } + uses_default <- is.null(cluster_function) + if (!.is_positive_integer(iterations)) { + insight::format_error("`iterations` must be a positive integer.") + } + if (uses_default && !.is_positive_integer(n_max)) { + insight::format_error("`n_max` must be a positive integer.") + } + + x <- .validate_test_clusters_data(x) + x <- .standardize_test_clusters_data(x, standardize) + + if (uses_default) { + insight::check_if_installed("mclust") + mclustBIC <- mclust::mclustBIC + max_components <- min(as.integer(n_max), nrow(x) - 1L) + cluster_function <- function(data) { + mclust::Mclust( + data, + G = seq_len(max_components), + verbose = FALSE + )$G + } + } + + out <- matchednull::matched_null_test( + x, + cluster_fn = cluster_function, + R = as.integer(iterations), + ... + ) + + attr(out, "standardize") <- standardize + attr(out, "n_max") <- if (uses_default) max_components else NULL + class(out) <- c("test_clusters", class(out)) + out +} + + +.validate_test_clusters_data <- function(x) { + if (!is.matrix(x) && !is.data.frame(x)) { + insight::format_error("`x` must be a numeric matrix or data frame.") + } + if (is.data.frame(x) && !all(vapply(x, is.numeric, logical(1)))) { + insight::format_error("All columns in `x` must be numeric.") + } + + x <- as.matrix(x) + if (!is.numeric(x)) { + insight::format_error("`x` must contain only numeric values.") + } + if (nrow(x) < 2L || ncol(x) < 1L) { + insight::format_error("`x` must contain at least two rows and one column.") + } + if (anyNA(x) || !all(is.finite(x))) { + insight::format_error("`x` must not contain missing or infinite values.") + } + x +} + + +.standardize_test_clusters_data <- function(x, standardize) { + if (!standardize) { + return(x) + } + + column_sd <- apply(x, 2L, stats::sd) + if (!all(is.finite(column_sd)) || any(column_sd == 0)) { + insight::format_error("`x` cannot contain constant columns when `standardize = TRUE`.") + } + scale(x) +} + + +.is_positive_integer <- function(x) { + is.numeric(x) && + length(x) == 1L && + !is.na(x) && + x >= 1L && + x %% 1 == 0 +} diff --git a/man/test_clusters.Rd b/man/test_clusters.Rd new file mode 100644 index 000000000..abf5d2e56 --- /dev/null +++ b/man/test_clusters.Rd @@ -0,0 +1,81 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/test_clusters.R +\name{test_clusters} +\alias{test_clusters} +\title{Test a clustering result against matched nulls} +\usage{ +test_clusters( + x, + cluster_function = NULL, + iterations = 200, + n_max = 10, + standardize = TRUE, + ... +) +} +\arguments{ +\item{x}{A numeric matrix or data frame with no missing or infinite values.} + +\item{cluster_function}{A function that takes a numeric matrix and returns one +number. By default, \code{mclust::Mclust()} selects the number of mixture +components by BIC.} + +\item{iterations}{Number of matched-null data sets to evaluate.} + +\item{n_max}{Maximum number of clusters considered by the default +\code{mclust::Mclust()} pipeline. Ignored when \code{cluster_function} is supplied.} + +\item{standardize}{Logical. If \code{TRUE}, variables are standardized before the +observed and matched-null cluster counts are computed.} + +\item{...}{Additional arguments passed to +\code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}, such as \code{copula}, \code{df}, \code{probs}, +\code{ridge}, or \code{parallel}.} +} +\value{ +An object of class \code{"test_clusters"} and \code{"matched_null_test"}. It +contains the observed statistic (\code{real}), matched-null statistics (\code{null}), +null interval (\code{interval}), one-sided Monte Carlo p-value (\code{p_exceed}), and +interval verdict (\code{within}). +} +\description{ +\code{test_clusters()} compares the number of clusters selected from observed data +with the numbers selected from matched-null versions of those data. The null +data preserve the observed marginal distributions and correlation structure +but contain no cluster structure by construction. This complements +\code{\link[=check_clusterstructure]{check_clusterstructure()}}, which evaluates clustering tendency against a +spatial-uniformity null. The matched-null comparison is implemented with +\code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}. +} +\details{ +The default cluster-count pipeline fits Gaussian mixture models for one to +\code{n_max} components and returns the BIC-selected count. A custom +\code{cluster_function} receives the same preprocessed data as the default +pipeline and must return one non-missing number. + +The Gaussian matched null tests for structure beyond the observed margins +and correlations; rejection alone does not establish the existence of +discrete types. When tail dependence is plausible, rerun the test with +\code{copula = "t"} as a sensitivity analysis. +} +\examples{ +\dontshow{if (requireNamespace("matchednull", quietly = TRUE) && requireNamespace("mclust", quietly = TRUE)) withAutoprint(\{ # examplesIf} +\donttest{ +set.seed(42) +test_clusters(iris[, 1:4], iterations = 19, n_max = 4) + +# Any scalar-returning clustering pipeline can be tested. +pick_two <- function(data) 2 +test_clusters(iris[, 1:4], cluster_function = pick_two, iterations = 19) +} +\dontshow{\}) # examplesIf} +} +\references{ +Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse +Test of Categorical Versus Continuous Personality Structure. Manuscript +under review. \doi{10.17605/OSF.IO/2EKCG} +} +\seealso{ +\code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}, +\code{\link[=check_clusterstructure]{check_clusterstructure()}} +} diff --git a/tests/testthat/test-test_clusters.R b/tests/testthat/test-test_clusters.R new file mode 100644 index 000000000..926ea8cbe --- /dev/null +++ b/tests/testthat/test-test_clusters.R @@ -0,0 +1,89 @@ +test_that("test_clusters supports a custom cluster function", { + skip_if_not_installed("matchednull", minimum_version = "0.2.1") + + set.seed(12) + x <- matrix(rnorm(120), ncol = 3) + out <- test_clusters( + x, + cluster_function = function(data) 2, + iterations = 9 + ) + + expect_s3_class(out, "test_clusters") + expect_s3_class(out, "matched_null_test") + expect_identical(out$real, 2) + expect_length(out$null, 9) + expect_true(out$within) + expect_true(attr(out, "standardize")) + expect_null(attr(out, "n_max")) + expect_output(print(out), "Matched-null test") + + out_t <- test_clusters( + x, + cluster_function = function(data) 2, + iterations = 3, + copula = "t", + df = 4 + ) + expect_identical(out_t$copula, "t") + expect_identical(out_t$df, 4) +}) + + +test_that("test_clusters is reproducible from a seed", { + skip_if_not_installed("matchednull", minimum_version = "0.2.1") + + set.seed(3) + x <- matrix(rnorm(180), ncol = 3) + cluster_summary <- function(data) sum(data[1:5, 1]) + + set.seed(11) + out1 <- test_clusters(x, cluster_summary, iterations = 9) + set.seed(11) + out2 <- test_clusters(x, cluster_summary, iterations = 9) + + expect_identical(out1$null, out2$null) + expect_gt(stats::var(out1$null), 0) +}) + + +test_that("default mclust pipeline distinguishes null and positive controls", { + skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("mclust") + + set.seed(42) + x_null <- matrix(rnorm(150 * 4), 150, 4) %*% + chol(diag(4) * 0.5 + 0.5) + set.seed(7) + null_result <- test_clusters(x_null, iterations = 9, n_max = 4) + expect_true(null_result$within) + expect_identical(attr(null_result, "n_max"), 4L) + + set.seed(42) + group <- sample.int(2, 150, replace = TRUE) + x_positive <- matrix(rnorm(150 * 4), 150, 4) + positive_correlation <- chol(matrix(c(1, 0.9, 0.9, 1), 2, 2)) + negative_correlation <- chol(matrix(c(1, -0.9, -0.9, 1), 2, 2)) + x_positive[group == 1, 1:2] <- x_positive[group == 1, 1:2] %*% positive_correlation + x_positive[group == 1, 3:4] <- x_positive[group == 1, 3:4] %*% positive_correlation + x_positive[group == 2, 1:2] <- x_positive[group == 2, 1:2] %*% negative_correlation + x_positive[group == 2, 3:4] <- x_positive[group == 2, 3:4] %*% negative_correlation + + set.seed(7) + positive_result <- test_clusters(x_positive, iterations = 9, n_max = 4) + expect_gt(positive_result$real, positive_result$interval[2]) +}) + + +test_that("test_clusters validates its inputs", { + skip_if_not_installed("matchednull", minimum_version = "0.2.1") + + x <- matrix(rnorm(40), ncol = 2) + expect_error(test_clusters(1:10), "matrix or data frame") + expect_error(test_clusters(data.frame(x = 1:3, y = letters[1:3])), "numeric") + expect_error(test_clusters(cbind(x, NA_real_)), "missing or infinite") + expect_error(test_clusters(cbind(x, 1)), "constant columns") + expect_error(test_clusters(x, cluster_function = 1), "must be a function") + expect_error(test_clusters(x, iterations = 1.5), "positive integer") + expect_error(test_clusters(x, n_max = 0), "positive integer") +}) From f08d6d5c36e6e9a7b3751709f9fd205ec9d17cab Mon Sep 17 00:00:00 2001 From: Miura Meng Date: Sat, 22 Aug 2026 06:04:03 -0400 Subject: [PATCH 2/7] Apply repository formatting --- R/test_clusters.R | 4 +++- inst/WORDLIST | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/R/test_clusters.R b/R/test_clusters.R index 74697c194..a964b96ce 100644 --- a/R/test_clusters.R +++ b/R/test_clusters.R @@ -138,7 +138,9 @@ test_clusters <- function( column_sd <- apply(x, 2L, stats::sd) if (!all(is.finite(column_sd)) || any(column_sd == 0)) { - insight::format_error("`x` cannot contain constant columns when `standardize = TRUE`.") + insight::format_error( + "`x` cannot contain constant columns when `standardize = TRUE`." + ) } scale(x) } diff --git a/inst/WORDLIST b/inst/WORDLIST index 847ab8bbe..c13d5596f 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -155,6 +155,7 @@ Matuschek McElreath McKelvey Mehrvarz +Meng Merkle Methoden Michalos From c9bb0d29a03af939194c8377470e351e4f276468 Mon Sep 17 00:00:00 2001 From: Miura Meng Date: Tue, 25 Aug 2026 10:43:22 -0400 Subject: [PATCH 3/7] Bump development version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 954da9052..db7182b4d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: performance Title: Assessment of Regression Models Performance -Version: 0.17.1.10 +Version: 0.17.1.11 Authors@R: c(person(given = "Daniel", family = "Lüdecke", From 0dc405af35fde647c8acdfd2acbdbe299280ab12 Mon Sep 17 00:00:00 2001 From: Miura Meng <251798069+haomeng797-ship-it@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:05:43 -0400 Subject: [PATCH 4/7] Validate cluster counts, add print method, clarify docs --- DESCRIPTION | 3 +- NAMESPACE | 1 + NEWS.md | 15 +- R/test_clusters.R | 239 ++++++++++++++++++++++++---- man/test_clusters.Rd | 142 +++++++++++++---- tests/testthat/test-test_clusters.R | 187 +++++++++++++++++++++- 6 files changed, 516 insertions(+), 71 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 95e80ae78..6161b7120 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: performance Title: Assessment of Regression Models Performance -Version: 0.18.2 +Version: 0.18.2.1 Authors@R: c(person(given = "Daniel", family = "Lüdecke", @@ -95,6 +95,7 @@ Suggests: brms, car, carData, + cluster, CompQuadForm, correlation (>= 0.8.8), cplm, diff --git a/NAMESPACE b/NAMESPACE index 8e086f19f..47466b134 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -383,6 +383,7 @@ S3method(print,r2_mlm) S3method(print,r2_nakagawa) S3method(print,r2_nakagawa_by_group) S3method(print,r2_pseudo) +S3method(print,test_clusters) S3method(print,test_likelihoodratio) S3method(print,test_performance) S3method(print_html,check_group_variation) diff --git a/NEWS.md b/NEWS.md index 9c04b83b9..31ad0e2d7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,14 @@ +# performance (devel) + +## Changes + +* New function `test_clusters()` uses the `matchednull` package to compare a + selected cluster count with the counts obtained from matched-null data that + keep the observed margins and approximately the observed correlations. It + reports one of three outcomes (no detected excess, departure from the + reference model, below the reference range) and validates the return values + of the clustering function. + # performance 0.18.2 ## Bug fixes @@ -29,10 +40,6 @@ ## Changes -* New function `test_clusters()` uses the `matchednull` package to test whether - a selected cluster count exceeds what matched-null data with the same margins - and correlations would produce. - * New functions `performance_mor()`, `performance_poor()` and `performance_ior()` to calculate median odds ratios, proportions of opposed odds ratios, and interval odds ratios for logistic multilevel regression. diff --git a/R/test_clusters.R b/R/test_clusters.R index a964b96ce..f5fd3f071 100644 --- a/R/test_clusters.R +++ b/R/test_clusters.R @@ -1,18 +1,23 @@ -#' Test a clustering result against matched nulls +#' Test a cluster count against matched-null data #' -#' `test_clusters()` compares the number of clusters selected from observed data -#' with the numbers selected from matched-null versions of those data. The null -#' data preserve the observed marginal distributions and correlation structure -#' but contain no cluster structure by construction. This complements -#' [`check_clusterstructure()`], which evaluates clustering tendency against a -#' spatial-uniformity null. The matched-null comparison is implemented with -#' [`matchednull::matched_null_test()`]. +#' `test_clusters()` runs a cluster-count selector on the observed data and on +#' `iterations` matched-null twins of those data, and compares the observed +#' count with the counts the twins produce. The twins keep the observed values +#' of every variable exactly and reproduce the observed correlations +#' approximately, but contain no cluster structure by construction. The test +#' reports one of three outcomes: no detected excess, a departure from the +#' reference model, or a count below the reference range. The comparison is +#' implemented with [`matchednull::matched_null_test()`]. It asks a different +#' question from [`check_clusterstructure()`], which evaluates clustering +#' tendency against a spatial-uniformity null. #' #' @param x A numeric matrix or data frame with no missing or infinite values. -#' @param cluster_function A function that takes a numeric matrix and returns one -#' number. By default, `mclust::Mclust()` selects the number of mixture -#' components by BIC. -#' @param iterations Number of matched-null data sets to evaluate. +#' @param cluster_function A function that takes a numeric matrix and returns a +#' single finite number, typically the selected number of clusters. By +#' default, `mclust::Mclust()` selects the number of mixture components by +#' BIC. The function is called once on the observed data and once on every +#' twin, and its return value is checked each time (see 'Details'). +#' @param iterations Number of matched-null twins to evaluate. #' @param n_max Maximum number of clusters considered by the default #' `mclust::Mclust()` pipeline. Ignored when `cluster_function` is supplied. #' @param standardize Logical. If `TRUE`, variables are standardized before the @@ -21,39 +26,108 @@ #' [`matchednull::matched_null_test()`], such as `copula`, `df`, `probs`, #' `ridge`, or `parallel`. #' -#' @return An object of class `"test_clusters"` and `"matched_null_test"`. It -#' contains the observed statistic (`real`), matched-null statistics (`null`), -#' null interval (`interval`), one-sided Monte Carlo p-value (`p_exceed`), and -#' interval verdict (`within`). +#' @return An object of class `"test_clusters"` and `"matched_null_test"`: a +#' list with the observed statistic (`real`), the twin statistics (`null`), +#' the reference interval (`interval`), the Monte Carlo exceedance +#' probability (`p_exceed`), the interval verdict (`within`), the number of +#' twins (`R`), and the copula family (`copula`, with `df` for the t copula). +#' The attribute `"outcome"` is `"within"`, `"above"`, or `"below"`: the +#' position of the observed statistic relative to the reference interval. +#' The attributes `"standardize"` and `"n_max"` record the preprocessing and +#' the maximum number of components of the default pipeline (`n_max` is +#' `NULL` when `cluster_function` is supplied). #' #' @details -#' The default cluster-count pipeline fits Gaussian mixture models for one to -#' `n_max` components and returns the BIC-selected count. A custom -#' `cluster_function` receives the same preprocessed data as the default -#' pipeline and must return one non-missing number. +#' The default cluster-count pipeline fits Gaussian mixture models with one to +#' `n_max` components using `mclust::Mclust()` and returns the BIC-selected +#' number of components. A user-supplied `cluster_function` receives the same +#' preprocessed data matrix as the default pipeline. Its return value is +#' checked on the observed data and on every twin: it must be a single finite +#' number, and any other value (`NA`, a character string, a factor, a logical +#' value, a vector of length two, `NULL`) is an error. The check runs here so +#' that invalid values are rejected whatever matchednull version is installed: +#' matchednull 0.2.1 dropped missing twin values from the exceedance +#' probability without notice, and later versions stop with an error of their +#' own. #' -#' The Gaussian matched null tests for structure beyond the observed margins -#' and correlations; rejection alone does not establish the existence of -#' discrete types. When tail dependence is plausible, rerun the test with -#' `copula = "t"` as a sensitivity analysis. +#' ## Reference model #' -#' @examplesIf requireNamespace("matchednull", quietly = TRUE) && requireNamespace("mclust", quietly = TRUE) +#' Each twin is drawn from a Gaussian copula whose latent correlation matrix +#' is the observed correlation matrix. Every column of a twin contains exactly +#' the observed values of that variable, rearranged in the rank order of the +#' copula draw. The margins are therefore reproduced exactly, but the +#' correlations only approximately: non-normal margins change the Pearson +#' correlations of the output relative to the latent correlation matrix. With +#' `copula = "t"`, the draw comes from a t copula instead, a sensitivity +#' variant with heavier-tailed dependence. Rerunning with `copula = "t"` shows +#' whether the outcome changes under that reference model; it does not by +#' itself establish or rule out clusters. +#' +#' ## Outcomes +#' +#' The observed count is compared with the interval between the `probs` +#' quantiles of the twin counts (by default the 2.5% and 97.5% quantiles). +#' +#' - A count above the interval is a departure from the specified reference +#' model. The test does not settle what kind of departure it is, and the +#' result is not evidence that clusters exist: typeless data with joint +#' asymmetry, for example, can exceed the reference model as well. +#' - A count inside the interval is no detected excess for the statistic that +#' was compared, the cluster count. It is not evidence that there are no +#' clusters, and it does not establish that the joint distribution belongs +#' to the reference family. +#' - A count below the interval is a diagnostic signal, not a verdict. It +#' calls for checking the pipeline and the reference before any +#' interpretation: whether the fits converged, whether the twins reproduce +#' the intended margins and correlations, and whether data and twins were +#' compared at the same sample size with the same statistic. +#' +#' `p_exceed` is a Monte Carlo exceedance probability under a reference model +#' fitted to the same data. It equals `(1 + k) / (iterations + 1)`, where `k` +#' is the number of twins whose count is at least the observed count. It is +#' not an exact finite-sample p-value. +#' +#' ## Before running the test +#' +#' Inspect the margins for pronounced multimodality first, for example with +#' [`check_multimodal()`] on single variables. Groups that differ only in +#' their means are detected at moderate separations. At large separations the +#' separation is expressed in the margins themselves, and the twins preserve +#' the margins exactly, so the reference model can reproduce such groups and +#' the comparison may then report no detected excess. This behaviour was +#' observed in the simulation constructions of Meng (2026); it is not asserted +#' as a general rule. Whether the joint distribution has clustering tendency is a different +#' question again; see [`check_clusterstructure()`], which uses a +#' spatial-uniformity null. +#' +#' @examplesIf all(insight::check_if_installed(c("matchednull", "mclust", "cluster"), quietly = TRUE)) #' \donttest{ #' set.seed(42) #' test_clusters(iris[, 1:4], iterations = 19, n_max = 4) #' -#' # Any scalar-returning clustering pipeline can be tested. -#' pick_two <- function(data) 2 -#' test_clusters(iris[, 1:4], cluster_function = pick_two, iterations = 19) +#' # Any pipeline that returns one number can be tested, for example k-means +#' # with the number of clusters chosen by the mean silhouette width. +#' select_k <- function(data) { +#' ks <- 2:5 +#' sil <- vapply(ks, function(k) { +#' cl <- stats::kmeans(data, centers = k, nstart = 5)$cluster +#' mean(cluster::silhouette(cl, stats::dist(data))[, "sil_width"]) +#' }, numeric(1)) +#' ks[which.max(sil)] +#' } +#' test_clusters(iris[, 1:4], cluster_function = select_k, iterations = 19) #' } #' #' @references -#' Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse -#' Test of Categorical Versus Continuous Personality Structure. Manuscript -#' under review. \doi{10.17605/OSF.IO/2EKCG} +#' - Meng, M. (2026). matchednull: Matched-null tests for cluster-count claims. +#' R package version 0.2.1. +#' +#' - Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse +#' Test of Categorical Versus Continuous Personality Structure. +#' Preregistration, Open Science Framework. \doi{10.17605/OSF.IO/2EKCG} #' #' @seealso [`matchednull::matched_null_test()`], -#' [`check_clusterstructure()`] +#' [`check_clusterstructure()`], [`check_multimodal()`] #' @export test_clusters <- function( x, @@ -84,6 +158,9 @@ test_clusters <- function( if (uses_default) { insight::check_if_installed("mclust") + # `mclust::Mclust()` rewrites its call to `mclustBIC` and evaluates it in + # the caller's frame, so the symbol must be visible from this function + # when mclust is not attached. Do not remove. mclustBIC <- mclust::mclustBIC max_components <- min(as.integer(n_max), nrow(x) - 1L) cluster_function <- function(data) { @@ -97,18 +174,114 @@ test_clusters <- function( out <- matchednull::matched_null_test( x, - cluster_fn = cluster_function, + cluster_fn = .checked_cluster_function(cluster_function), R = as.integer(iterations), ... ) attr(out, "standardize") <- standardize attr(out, "n_max") <- if (uses_default) max_components else NULL + attr(out, "outcome") <- .test_clusters_outcome(out$real, out$interval) class(out) <- c("test_clusters", class(out)) out } +#' @export +print.test_clusters <- function(x, digits = 3, ...) { + outcome <- .test_clusters_outcome(x$real, x$interval) + outcome_text <- switch( + outcome, + within = "No detected excess: the count lies within the reference interval.", + above = paste0( + "Departure from the reference model: the count exceeds the interval.\n", + "The test does not settle what kind of departure." + ), + below = "Below the reference range: a diagnostic signal, not a verdict." + ) + outcome_color <- switch(outcome, within = "green", above = "yellow", below = "yellow") + + reference <- if (is.null(x$copula) || x$copula == "gaussian") { + "Gaussian copula" + } else { + sprintf( + "t copula (df = %s)", + insight::format_value(x$df, digits = digits, protect_integers = TRUE) + ) + } + + interval <- sprintf( + "[%s]", + toString(insight::format_value(x$interval, digits = digits, protect_integers = TRUE)) + ) + if (!is.null(names(x$interval))) { + interval <- sprintf( + "%s (%s quantiles of the twins)", + interval, + paste(names(x$interval), collapse = " and ") + ) + } + + insight::print_color("# Matched-null test of the cluster count\n\n", "blue") + # the exceedance counts twins whose statistic is at least the observed one + cat(sprintf( + " Observed statistic: %s\n", + insight::format_value(x$real, digits = digits, protect_integers = TRUE) + )) + cat(sprintf(" Reference interval: %s\n", interval)) + cat(sprintf( + " MC exceedance (twins >= observed): %s\n", + insight::format_value(x$p_exceed, digits = digits) + )) + cat(sprintf( + " Reference model: %s, %s matched-null twins\n", + reference, + insight::format_value(x$R, protect_integers = TRUE) + )) + cat("\n") + insight::print_color(paste0(outcome_text, "\n"), outcome_color) + invisible(x) +} + + +# matchednull calls the clustering function once on the observed data and once +# per twin. matchednull 0.2.1 dropped missing twin values from the exceedance +# probability without notice, so the wrapper checks the raw return value +# itself, before any coercion, whatever version is installed. +.checked_cluster_function <- function(cluster_function) { + function(data) { + value <- cluster_function(data) + if (!is.numeric(value) || length(value) != 1L || !is.finite(value)) { + insight::format_error(sprintf( + "`cluster_function` must return a single finite number, but returned %s.", + .describe_cluster_value(value) + )) + } + unname(as.numeric(value)) + } +} + + +.describe_cluster_value <- function(value) { + if (is.numeric(value) && length(value) == 1L) { + # a number that is not finite: NA, NaN, Inf or -Inf + return(paste0("`", format(value), "`")) + } + sprintf("%s of length %d", class(value)[1], length(value)) +} + + +.test_clusters_outcome <- function(real, interval) { + if (real > interval[2]) { + "above" + } else if (real < interval[1]) { + "below" + } else { + "within" + } +} + + .validate_test_clusters_data <- function(x) { if (!is.matrix(x) && !is.data.frame(x)) { insight::format_error("`x` must be a numeric matrix or data frame.") diff --git a/man/test_clusters.Rd b/man/test_clusters.Rd index abf5d2e56..a43487407 100644 --- a/man/test_clusters.Rd +++ b/man/test_clusters.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/test_clusters.R \name{test_clusters} \alias{test_clusters} -\title{Test a clustering result against matched nulls} +\title{Test a cluster count against matched-null data} \usage{ test_clusters( x, @@ -16,11 +16,13 @@ test_clusters( \arguments{ \item{x}{A numeric matrix or data frame with no missing or infinite values.} -\item{cluster_function}{A function that takes a numeric matrix and returns one -number. By default, \code{mclust::Mclust()} selects the number of mixture -components by BIC.} +\item{cluster_function}{A function that takes a numeric matrix and returns a +single finite number, typically the selected number of clusters. By +default, \code{mclust::Mclust()} selects the number of mixture components by +BIC. The function is called once on the observed data and once on every +twin, and its return value is checked each time (see 'Details').} -\item{iterations}{Number of matched-null data sets to evaluate.} +\item{iterations}{Number of matched-null twins to evaluate.} \item{n_max}{Maximum number of clusters considered by the default \code{mclust::Mclust()} pipeline. Ignored when \code{cluster_function} is supplied.} @@ -33,49 +35,125 @@ observed and matched-null cluster counts are computed.} \code{ridge}, or \code{parallel}.} } \value{ -An object of class \code{"test_clusters"} and \code{"matched_null_test"}. It -contains the observed statistic (\code{real}), matched-null statistics (\code{null}), -null interval (\code{interval}), one-sided Monte Carlo p-value (\code{p_exceed}), and -interval verdict (\code{within}). +An object of class \code{"test_clusters"} and \code{"matched_null_test"}: a +list with the observed statistic (\code{real}), the twin statistics (\code{null}), +the reference interval (\code{interval}), the Monte Carlo exceedance +probability (\code{p_exceed}), the interval verdict (\code{within}), the number of +twins (\code{R}), and the copula family (\code{copula}, with \code{df} for the t copula). +The attribute \code{"outcome"} is \code{"within"}, \code{"above"}, or \code{"below"}: the +position of the observed statistic relative to the reference interval. +The attributes \code{"standardize"} and \code{"n_max"} record the preprocessing and +the maximum number of components of the default pipeline (\code{n_max} is +\code{NULL} when \code{cluster_function} is supplied). } \description{ -\code{test_clusters()} compares the number of clusters selected from observed data -with the numbers selected from matched-null versions of those data. The null -data preserve the observed marginal distributions and correlation structure -but contain no cluster structure by construction. This complements -\code{\link[=check_clusterstructure]{check_clusterstructure()}}, which evaluates clustering tendency against a -spatial-uniformity null. The matched-null comparison is implemented with -\code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}. +\code{test_clusters()} runs a cluster-count selector on the observed data and on +\code{iterations} matched-null twins of those data, and compares the observed +count with the counts the twins produce. The twins keep the observed values +of every variable exactly and reproduce the observed correlations +approximately, but contain no cluster structure by construction. The test +reports one of three outcomes: no detected excess, a departure from the +reference model, or a count below the reference range. The comparison is +implemented with \code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}. It asks a different +question from \code{\link[=check_clusterstructure]{check_clusterstructure()}}, which evaluates clustering +tendency against a spatial-uniformity null. } \details{ -The default cluster-count pipeline fits Gaussian mixture models for one to -\code{n_max} components and returns the BIC-selected count. A custom -\code{cluster_function} receives the same preprocessed data as the default -pipeline and must return one non-missing number. +The default cluster-count pipeline fits Gaussian mixture models with one to +\code{n_max} components using \code{mclust::Mclust()} and returns the BIC-selected +number of components. A user-supplied \code{cluster_function} receives the same +preprocessed data matrix as the default pipeline. Its return value is +checked on the observed data and on every twin: it must be a single finite +number, and any other value (\code{NA}, a character string, a factor, a logical +value, a vector of length two, \code{NULL}) is an error. The check runs here so +that invalid values are rejected whatever matchednull version is installed: +matchednull 0.2.1 dropped missing twin values from the exceedance +probability without notice, and later versions stop with an error of their +own. +\subsection{Reference model}{ -The Gaussian matched null tests for structure beyond the observed margins -and correlations; rejection alone does not establish the existence of -discrete types. When tail dependence is plausible, rerun the test with -\code{copula = "t"} as a sensitivity analysis. +Each twin is drawn from a Gaussian copula whose latent correlation matrix +is the observed correlation matrix. Every column of a twin contains exactly +the observed values of that variable, rearranged in the rank order of the +copula draw. The margins are therefore reproduced exactly, but the +correlations only approximately: non-normal margins change the Pearson +correlations of the output relative to the latent correlation matrix. With +\code{copula = "t"}, the draw comes from a t copula instead, a sensitivity +variant with heavier-tailed dependence. Rerunning with \code{copula = "t"} shows +whether the outcome changes under that reference model; it does not by +itself establish or rule out clusters. +} + +\subsection{Outcomes}{ + +The observed count is compared with the interval between the \code{probs} +quantiles of the twin counts (by default the 2.5\% and 97.5\% quantiles). +\itemize{ +\item A count above the interval is a departure from the specified reference +model. The test does not settle what kind of departure it is, and the +result is not evidence that clusters exist: typeless data with joint +asymmetry, for example, can exceed the reference model as well. +\item A count inside the interval is no detected excess for the statistic that +was compared, the cluster count. It is not evidence that there are no +clusters, and it does not establish that the joint distribution belongs +to the reference family. +\item A count below the interval is a diagnostic signal, not a verdict. It +calls for checking the pipeline and the reference before any +interpretation: whether the fits converged, whether the twins reproduce +the intended margins and correlations, and whether data and twins were +compared at the same sample size with the same statistic. +} + +\code{p_exceed} is a Monte Carlo exceedance probability under a reference model +fitted to the same data. It equals \code{(1 + k) / (iterations + 1)}, where \code{k} +is the number of twins whose count is at least the observed count. It is +not an exact finite-sample p-value. +} + +\subsection{Before running the test}{ + +Inspect the margins for pronounced multimodality first, for example with +\code{\link[=check_multimodal]{check_multimodal()}} on single variables. Groups that differ only in +their means are detected at moderate separations. At large separations the +separation is expressed in the margins themselves, and the twins preserve +the margins exactly, so the reference model can reproduce such groups and +the comparison may then report no detected excess. This behaviour was +observed in the simulation constructions of Meng (2026); it is not asserted +as a general rule. Whether the joint distribution has clustering tendency is a different +question again; see \code{\link[=check_clusterstructure]{check_clusterstructure()}}, which uses a +spatial-uniformity null. +} } \examples{ -\dontshow{if (requireNamespace("matchednull", quietly = TRUE) && requireNamespace("mclust", quietly = TRUE)) withAutoprint(\{ # examplesIf} +\dontshow{if (all(insight::check_if_installed(c("matchednull", "mclust", "cluster"), quietly = TRUE))) withAutoprint(\{ # examplesIf} \donttest{ set.seed(42) test_clusters(iris[, 1:4], iterations = 19, n_max = 4) -# Any scalar-returning clustering pipeline can be tested. -pick_two <- function(data) 2 -test_clusters(iris[, 1:4], cluster_function = pick_two, iterations = 19) +# Any pipeline that returns one number can be tested, for example k-means +# with the number of clusters chosen by the mean silhouette width. +select_k <- function(data) { + ks <- 2:5 + sil <- vapply(ks, function(k) { + cl <- stats::kmeans(data, centers = k, nstart = 5)$cluster + mean(cluster::silhouette(cl, stats::dist(data))[, "sil_width"]) + }, numeric(1)) + ks[which.max(sil)] +} +test_clusters(iris[, 1:4], cluster_function = select_k, iterations = 19) } \dontshow{\}) # examplesIf} } \references{ -Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse -Test of Categorical Versus Continuous Personality Structure. Manuscript -under review. \doi{10.17605/OSF.IO/2EKCG} +\itemize{ +\item Meng, M. (2026). matchednull: Matched-null tests for cluster-count claims. +R package version 0.2.1. \url{https://CRAN.R-project.org/package=matchednull} +\item Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse +Test of Categorical Versus Continuous Personality Structure. +Preregistration, Open Science Framework. \doi{10.17605/OSF.IO/2EKCG} +} } \seealso{ \code{\link[matchednull:matched_null_test]{matchednull::matched_null_test()}}, -\code{\link[=check_clusterstructure]{check_clusterstructure()}} +\code{\link[=check_clusterstructure]{check_clusterstructure()}}, \code{\link[=check_multimodal]{check_multimodal()}} } diff --git a/tests/testthat/test-test_clusters.R b/tests/testthat/test-test_clusters.R index 926ea8cbe..542c55be8 100644 --- a/tests/testthat/test-test_clusters.R +++ b/tests/testthat/test-test_clusters.R @@ -14,9 +14,21 @@ test_that("test_clusters supports a custom cluster function", { expect_identical(out$real, 2) expect_length(out$null, 9) expect_true(out$within) + expect_identical(attr(out, "outcome"), "within") expect_true(attr(out, "standardize")) expect_null(attr(out, "n_max")) - expect_output(print(out), "Matched-null test") + expect_output(print(out), "Matched-null test of the cluster count") + expect_output(print(out), "MC exceedance (twins >= observed):", fixed = TRUE) + printed <- capture.output(print(out)) + p_line <- grep("MC exceedance", printed, value = TRUE, fixed = TRUE) + p_shown <- as.numeric(sub(".*:\\s+", "", p_line)) + expect_equal(p_shown, (1 + sum(out$null >= out$real)) / (out$R + 1), tolerance = 5e-4) + expect_output(print(out), "Gaussian copula, 9 matched-null twins", fixed = TRUE) + expect_output( + print(out), + "No detected excess: the count lies within the reference interval.", + fixed = TRUE + ) out_t <- test_clusters( x, @@ -27,6 +39,48 @@ test_that("test_clusters supports a custom cluster function", { ) expect_identical(out_t$copula, "t") expect_identical(out_t$df, 4) + expect_output(print(out_t), "t copula (df = 4), 3 matched-null twins", fixed = TRUE) +}) + + +test_that("test_clusters reports counts above and below the reference range", { + skip_if_not_installed("matchednull", minimum_version = "0.2.1") + + set.seed(5) + x <- matrix(rnorm(120), ncol = 3) + # matchednull evaluates the observed data first and the twins afterwards + first_then <- function(first, rest) { + calls <- 0 + function(data) { + calls <<- calls + 1 + if (calls == 1) first else rest + } + } + + above <- test_clusters(x, cluster_function = first_then(5, 2), iterations = 9) + expect_identical(above$real, 5) + expect_false(above$within) + expect_identical(attr(above, "outcome"), "above") + expect_output( + print(above), + "Departure from the reference model: the count exceeds the interval.", + fixed = TRUE + ) + expect_output( + print(above), + "The test does not settle what kind of departure.", + fixed = TRUE + ) + + below <- test_clusters(x, cluster_function = first_then(1, 2), iterations = 9) + expect_identical(below$real, 1) + expect_false(below$within) + expect_identical(attr(below, "outcome"), "below") + expect_output( + print(below), + "Below the reference range: a diagnostic signal, not a verdict.", + fixed = TRUE + ) }) @@ -57,6 +111,7 @@ test_that("default mclust pipeline distinguishes null and positive controls", { set.seed(7) null_result <- test_clusters(x_null, iterations = 9, n_max = 4) expect_true(null_result$within) + expect_identical(attr(null_result, "outcome"), "within") expect_identical(attr(null_result, "n_max"), 4L) set.seed(42) @@ -72,6 +127,136 @@ test_that("default mclust pipeline distinguishes null and positive controls", { set.seed(7) positive_result <- test_clusters(x_positive, iterations = 9, n_max = 4) expect_gt(positive_result$real, positive_result$interval[2]) + expect_identical(attr(positive_result, "outcome"), "above") +}) + + +test_that("default mclust pipeline works in a clean session without attaching mclust", { + # `mclust::Mclust()` evaluates `mclustBIC` in the caller's frame; this only + # fails when mclust is not attached, so it has to be checked in a fresh + # process against the installed package. + skip_on_cran() + skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("mclust") + + # the child needs every library the parent sees, not only the first one: + # dependencies such as insight often live in a different library path + libs <- deparse1(.libPaths(), collapse = "") + code <- paste( + sprintf(".libPaths(c(%s, .libPaths()));", libs), + "loaded <- tryCatch(", + " {", + " loadNamespace('performance');", + " TRUE", + " },", + " error = function(e) gsub('[\\r\\n]+', ' ', conditionMessage(e))", + ");", + "if (!isTRUE(loaded)) {", + " cat(paste0('LOADFAIL=', loaded, '\\n'))", + "} else {", + " version <- as.character(utils::packageVersion('performance'));", + " if (!'test_clusters' %in% getNamespaceExports('performance')) {", + " cat(paste0('NOEXPORT;VERSION=', version, '\\n'))", + " } else {", + " set.seed(1);", + " x <- matrix(rnorm(180), 60, 3) %*% chol(diag(3) * 0.5 + 0.5);", + " out <- performance::test_clusters(x, iterations = 3, n_max = 3);", + " cat(paste0(", + " 'ATTACHED=', 'package:mclust' %in% search(),", + " ';REAL=', out$real, ';VERSION=', version, '\\n'", + " ))", + " }", + "}" + ) + output <- system2( + file.path(R.home("bin"), "Rscript"), + c("--vanilla", "-e", shQuote(code)), + stdout = TRUE, + stderr = TRUE + ) + result <- grep("^(LOADFAIL=|NOEXPORT;|ATTACHED=)", output, value = TRUE) + reported <- paste(output, collapse = "\n") + + if (length(result) != 1L) { + # neither a usable result nor a reason to skip: this must not pass quietly + fail(paste0("Unexpected output from the clean session:\n", reported)) + } else if (startsWith(result, "NOEXPORT;")) { + # an older performance is installed, so the fix cannot be checked here + skip(paste0( + "the installed performance (version ", + sub("^NOEXPORT;VERSION=", "", result), + ") does not export test_clusters()" + )) + } else if (startsWith(result, "LOADFAIL=")) { + fail(paste0( + "The clean session could not load performance:\n", + reported + )) + } else { + expect_match(result, "^ATTACHED=FALSE;REAL=") + real <- suppressWarnings(as.numeric( + sub("^ATTACHED=FALSE;REAL=([^;]+);VERSION=.*$", "\\1", result) + )) + expect_true(is.finite(real)) + } +}) + + +test_that("test_clusters validates the raw return value of cluster_function", { + skip_if_not_installed("matchednull", minimum_version = "0.2.1") + + set.seed(8) + x <- matrix(rnorm(60), ncol = 2) + returns <- function(value) function(data) value + # error messages are wrapped to the console width, so match across line breaks + not_finite <- "single\\s+finite\\s+number" + + expect_error(test_clusters(x, returns(NA_real_), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(Inf), iterations = 3), not_finite) + expect_error(test_clusters(x, returns("2"), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(factor(2)), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(TRUE), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(c(2, 3)), iterations = 3), not_finite) + expect_error(test_clusters(x, returns(NULL), iterations = 3), not_finite) + expect_error( + test_clusters(x, returns("2"), iterations = 3), + "character\\s+of\\s+length\\s+1" + ) + expect_error( + test_clusters(x, returns(c(2, 3)), iterations = 3), + "numeric\\s+of\\s+length\\s+2" + ) + expect_error( + test_clusters(x, returns(NULL), iterations = 3), + "NULL\\s+of\\s+length\\s+0" + ) + expect_error(test_clusters(x, returns(NA_real_), iterations = 3), "`NA`") + + # the check also runs on every twin, where matchednull would otherwise drop + # a missing value from the exceedance probability + calls <- 0 + na_on_twins <- function(data) { + calls <<- calls + 1 + if (calls == 1) 2 else NA_real_ + } + expect_error(test_clusters(x, na_on_twins, iterations = 3), not_finite) + + # errors raised inside cluster_function propagate unchanged + calls <- 0 + fails_on_twin <- function(data) { + calls <<- calls + 1 + if (calls > 1) { + stop("twin failure", call. = FALSE) + } + 2 + } + expect_error(test_clusters(x, fails_on_twin, iterations = 3), "twin failure") + + # a length-one integer or 1x1 matrix is accepted and coerced to a plain number + out <- test_clusters(x, returns(2L), iterations = 3) + expect_identical(out$real, 2) + out <- test_clusters(x, returns(matrix(2)), iterations = 3) + expect_identical(out$null, c(2, 2, 2)) }) From e43e18bfef81c61b92c1f7c6871604dca5960777 Mon Sep 17 00:00:00 2001 From: Miura Meng <251798069+haomeng797-ship-it@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:36:06 -0400 Subject: [PATCH 5/7] Add new words to WORDLIST --- inst/WORDLIST | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inst/WORDLIST b/inst/WORDLIST index c13d5596f..2d65bd461 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -327,6 +327,7 @@ lm lme lmrob lmtest +matchednull mattansb metafor mfx @@ -341,6 +342,7 @@ modelfit modelling models’ multicollinearity +multimodality multimodel multiresponse multivariable @@ -379,6 +381,7 @@ subscale subscales theoreritcal tidymodels +typeless unadjusted und underfitted From bd8859ca4955afdffa1c1a70972fbe10b576cffb Mon Sep 17 00:00:00 2001 From: Miura Meng <251798069+haomeng797-ship-it@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:08:01 -0400 Subject: [PATCH 6/7] Clarify what the twins reproduce in test_clusters docs --- R/test_clusters.R | 15 ++++++++------- man/test_clusters.Rd | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/R/test_clusters.R b/R/test_clusters.R index f5fd3f071..a4df5c2bf 100644 --- a/R/test_clusters.R +++ b/R/test_clusters.R @@ -92,13 +92,14 @@ #' Inspect the margins for pronounced multimodality first, for example with #' [`check_multimodal()`] on single variables. Groups that differ only in #' their means are detected at moderate separations. At large separations the -#' separation is expressed in the margins themselves, and the twins preserve -#' the margins exactly, so the reference model can reproduce such groups and -#' the comparison may then report no detected excess. This behaviour was -#' observed in the simulation constructions of Meng (2026); it is not asserted -#' as a general rule. Whether the joint distribution has clustering tendency is a different -#' question again; see [`check_clusterstructure()`], which uses a -#' spatial-uniformity null. +#' separation shows in the margins themselves. The twins keep the margins +#' exactly, so the pipeline can select as many clusters in the twins (which +#' contain no groups) as in the data, and the comparison may then report no +#' detected excess. This behaviour was observed in the simulation +#' constructions of Meng (2026); it is not asserted as a general rule. Whether +#' the joint distribution has clustering tendency is a different question +#' again; see [`check_clusterstructure()`], which uses a spatial-uniformity +#' null. #' #' @examplesIf all(insight::check_if_installed(c("matchednull", "mclust", "cluster"), quietly = TRUE)) #' \donttest{ diff --git a/man/test_clusters.Rd b/man/test_clusters.Rd index a43487407..16f9f9cdf 100644 --- a/man/test_clusters.Rd +++ b/man/test_clusters.Rd @@ -115,13 +115,14 @@ not an exact finite-sample p-value. Inspect the margins for pronounced multimodality first, for example with \code{\link[=check_multimodal]{check_multimodal()}} on single variables. Groups that differ only in their means are detected at moderate separations. At large separations the -separation is expressed in the margins themselves, and the twins preserve -the margins exactly, so the reference model can reproduce such groups and -the comparison may then report no detected excess. This behaviour was -observed in the simulation constructions of Meng (2026); it is not asserted -as a general rule. Whether the joint distribution has clustering tendency is a different -question again; see \code{\link[=check_clusterstructure]{check_clusterstructure()}}, which uses a -spatial-uniformity null. +separation shows in the margins themselves. The twins keep the margins +exactly, so the pipeline can select as many clusters in the twins (which +contain no groups) as in the data, and the comparison may then report no +detected excess. This behaviour was observed in the simulation +constructions of Meng (2026); it is not asserted as a general rule. Whether +the joint distribution has clustering tendency is a different question +again; see \code{\link[=check_clusterstructure]{check_clusterstructure()}}, which uses a spatial-uniformity +null. } } \examples{ From 9104570e8d5c23b24310693207bb07c489632888 Mon Sep 17 00:00:00 2001 From: Miura Meng <251798069+haomeng797-ship-it@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:33:26 -0500 Subject: [PATCH 7/7] require matchednull 0.2.2 now that it's on CRAN --- DESCRIPTION | 2 +- R/test_clusters.R | 17 +++++++---------- man/test_clusters.Rd | 10 ++++------ tests/testthat/test-test_clusters.R | 17 ++++++++--------- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index f025408bf..9f141ab67 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -131,7 +131,7 @@ Suggests: loo, MASS, Matrix, - matchednull (>= 0.2.1), + matchednull (>= 0.2.2), mclogit, mclust, metadat, diff --git a/R/test_clusters.R b/R/test_clusters.R index a4df5c2bf..346e6189c 100644 --- a/R/test_clusters.R +++ b/R/test_clusters.R @@ -44,11 +44,9 @@ #' preprocessed data matrix as the default pipeline. Its return value is #' checked on the observed data and on every twin: it must be a single finite #' number, and any other value (`NA`, a character string, a factor, a logical -#' value, a vector of length two, `NULL`) is an error. The check runs here so -#' that invalid values are rejected whatever matchednull version is installed: -#' matchednull 0.2.1 dropped missing twin values from the exceedance -#' probability without notice, and later versions stop with an error of their -#' own. +#' value, a vector of length two, `NULL`) is an error. The check runs here, on +#' the raw return value, so that the error names `cluster_function` and what it +#' returned. #' #' ## Reference model #' @@ -121,7 +119,7 @@ #' #' @references #' - Meng, M. (2026). matchednull: Matched-null tests for cluster-count claims. -#' R package version 0.2.1. +#' R package version 0.2.2. #' #' - Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse #' Test of Categorical Versus Continuous Personality Structure. @@ -138,7 +136,7 @@ test_clusters <- function( standardize = TRUE, ... ) { - insight::check_if_installed("matchednull", minimum_version = "0.2.1") + insight::check_if_installed("matchednull", minimum_version = "0.2.2") if (!is.logical(standardize) || length(standardize) != 1L || is.na(standardize)) { insight::format_error("`standardize` must be `TRUE` or `FALSE`.") @@ -246,9 +244,8 @@ print.test_clusters <- function(x, digits = 3, ...) { # matchednull calls the clustering function once on the observed data and once -# per twin. matchednull 0.2.1 dropped missing twin values from the exceedance -# probability without notice, so the wrapper checks the raw return value -# itself, before any coercion, whatever version is installed. +# per twin. The wrapper checks the raw return value itself, before any +# coercion, so that the error speaks of `cluster_function`. .checked_cluster_function <- function(cluster_function) { function(data) { value <- cluster_function(data) diff --git a/man/test_clusters.Rd b/man/test_clusters.Rd index 16f9f9cdf..b01683071 100644 --- a/man/test_clusters.Rd +++ b/man/test_clusters.Rd @@ -65,11 +65,9 @@ number of components. A user-supplied \code{cluster_function} receives the same preprocessed data matrix as the default pipeline. Its return value is checked on the observed data and on every twin: it must be a single finite number, and any other value (\code{NA}, a character string, a factor, a logical -value, a vector of length two, \code{NULL}) is an error. The check runs here so -that invalid values are rejected whatever matchednull version is installed: -matchednull 0.2.1 dropped missing twin values from the exceedance -probability without notice, and later versions stop with an error of their -own. +value, a vector of length two, \code{NULL}) is an error. The check runs here, on +the raw return value, so that the error names \code{cluster_function} and what it +returned. \subsection{Reference model}{ Each twin is drawn from a Gaussian copula whose latent correlation matrix @@ -148,7 +146,7 @@ test_clusters(iris[, 1:4], cluster_function = select_k, iterations = 19) \references{ \itemize{ \item Meng, M. (2026). matchednull: Matched-null tests for cluster-count claims. -R package version 0.2.1. \url{https://CRAN.R-project.org/package=matchednull} +R package version 0.2.2. \url{https://CRAN.R-project.org/package=matchednull} \item Meng, M. (2026). Types Without Taxa: A Covariance-Matched-Null Multiverse Test of Categorical Versus Continuous Personality Structure. Preregistration, Open Science Framework. \doi{10.17605/OSF.IO/2EKCG} diff --git a/tests/testthat/test-test_clusters.R b/tests/testthat/test-test_clusters.R index 542c55be8..3aed9fbae 100644 --- a/tests/testthat/test-test_clusters.R +++ b/tests/testthat/test-test_clusters.R @@ -1,5 +1,5 @@ test_that("test_clusters supports a custom cluster function", { - skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("matchednull", minimum_version = "0.2.2") set.seed(12) x <- matrix(rnorm(120), ncol = 3) @@ -44,7 +44,7 @@ test_that("test_clusters supports a custom cluster function", { test_that("test_clusters reports counts above and below the reference range", { - skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("matchednull", minimum_version = "0.2.2") set.seed(5) x <- matrix(rnorm(120), ncol = 3) @@ -85,7 +85,7 @@ test_that("test_clusters reports counts above and below the reference range", { test_that("test_clusters is reproducible from a seed", { - skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("matchednull", minimum_version = "0.2.2") set.seed(3) x <- matrix(rnorm(180), ncol = 3) @@ -102,7 +102,7 @@ test_that("test_clusters is reproducible from a seed", { test_that("default mclust pipeline distinguishes null and positive controls", { - skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("matchednull", minimum_version = "0.2.2") skip_if_not_installed("mclust") set.seed(42) @@ -136,7 +136,7 @@ test_that("default mclust pipeline works in a clean session without attaching mc # fails when mclust is not attached, so it has to be checked in a fresh # process against the installed package. skip_on_cran() - skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("matchednull", minimum_version = "0.2.2") skip_if_not_installed("mclust") # the child needs every library the parent sees, not only the first one: @@ -203,7 +203,7 @@ test_that("default mclust pipeline works in a clean session without attaching mc test_that("test_clusters validates the raw return value of cluster_function", { - skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("matchednull", minimum_version = "0.2.2") set.seed(8) x <- matrix(rnorm(60), ncol = 2) @@ -232,8 +232,7 @@ test_that("test_clusters validates the raw return value of cluster_function", { ) expect_error(test_clusters(x, returns(NA_real_), iterations = 3), "`NA`") - # the check also runs on every twin, where matchednull would otherwise drop - # a missing value from the exceedance probability + # the check also runs on every twin calls <- 0 na_on_twins <- function(data) { calls <<- calls + 1 @@ -261,7 +260,7 @@ test_that("test_clusters validates the raw return value of cluster_function", { test_that("test_clusters validates its inputs", { - skip_if_not_installed("matchednull", minimum_version = "0.2.1") + skip_if_not_installed("matchednull", minimum_version = "0.2.2") x <- matrix(rnorm(40), ncol = 2) expect_error(test_clusters(1:10), "matrix or data frame")