diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts index 1dc0f5be32..1cbf482ea2 100644 --- a/packages/cli/src/commands/check.test.ts +++ b/packages/cli/src/commands/check.test.ts @@ -12,6 +12,7 @@ vi.mock("../telemetry/events.js", () => ({ import { contrastRatio, parseColorRGBA } from "./contrast-bg.js"; import { createCheckCommand, parseCheckRange } from "./check.js"; +import { readConfig } from "../telemetry/config.js"; import { CheckDeadlineError, CheckStoppedError } from "../utils/checkDeadline.js"; import { trackCommandFailures } from "../utils/command-failure-tracking.js"; import type { CommandDef } from "citty"; @@ -1589,6 +1590,55 @@ describe("check pipeline", () => { expect(said).not.toContain("Browser session never ran"); }); + describe("remembers the newest check for hyperframes feedback", () => { + const run = (runPipeline: () => Promise) => + runViaCli( + createCheckCommand({ + resolveProject: () => PROJECT, + runPipeline: vi.fn(runPipeline), + prepareMedia: vi.fn(async () => []), + withMeta: (value) => value, + }), + { rawArgs: [] }, + ); + + it("keeps the error codes of a failed check", async () => { + const lint = lintWith( + "error", + "root_missing_composition_id", + "Root needs data-composition-id.", + ); + const { report } = await runScenario(fakeDriver(), {}, { lint }); + await run(async () => report); + + expect(readConfig().lastCheck).toMatchObject({ + exitCode: 1, + codes: { root_missing_composition_id: 1 }, + }); + }); + + it("records a check that threw as a failure, not the check before it", async () => { + await run(async () => (await runScenario(fakeDriver(), {}, {})).report); + await run(async () => { + throw new Error("browser died"); + }); + + expect(readConfig().lastCheck).toMatchObject({ exitCode: 1, codes: { runtime_error: 1 } }); + }); + + it("leaves a check that never finished as unfinished, with when it started", async () => { + const { report } = await runScenario(fakeDriver(), {}, {}); + await run(async () => report); + let finish!: (value: CheckReport) => void; + const running = run(() => new Promise((resolve) => (finish = resolve))); + await vi.waitFor(() => expect(readConfig().lastCheck?.exitCode).toBeUndefined()); + + expect(readConfig().lastCheck?.at).toMatch(/^\d{4}-\d\d-\d\dT/); + finish(report); + await running; + }); + }); + it("marks browserSkipped true when the linter itself crashes", async () => { const { deps } = dependencies(fakeDriver()); deps.lintProject = vi.fn(async () => { diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index 3a402382eb..ac2026b343 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -41,6 +41,8 @@ import { resolveLocalBrowserGpuMode } from "../browser/gpuPolicy.js"; import { preResolveHostileMediaProxies, type PreparedMediaFile } from "../utils/checkBrowser.js"; import { parseTimeoutMs } from "../utils/timeoutArg.js"; import { failureMessage, jsonFailure } from "../utils/jsonError.js"; +import { recordLastCheck } from "../telemetry/config.js"; +import { failureCodes } from "../utils/checkFailureCodes.js"; export const examples: Example[] = [ ["Run the full verification gate", "hyperframes check"], @@ -287,9 +289,15 @@ export function createCheckCommand( await runPrepareMedia(dependencies, project, options, asJson); return; } + recordLastCheck({ at: new Date().toISOString(), codes: {} }); const report = await withProjectLock(project.dir, "check", () => dependencies.runPipeline(project, options), ); + recordLastCheck({ + at: new Date().toISOString(), + exitCode: checkExitCode(report), + codes: failureCodes(report), + }); if (asJson) { console.log(JSON.stringify(dependencies.withMeta(report), null, 2)); } else { @@ -303,6 +311,11 @@ export function createCheckCommand( } setCommandExitCode(checkExitCode(report)); } catch (error) { + recordLastCheck({ + at: new Date().toISOString(), + exitCode: 1, + codes: { [error instanceof CliUsageError ? error.code : "runtime_error"]: 1 }, + }); reportCheckFailure(error, asJson, (value) => dependencies.withMeta(value)); } finally { reported.done(); diff --git a/packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts b/packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts index 069fb2310f..4a60962e61 100644 --- a/packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts +++ b/packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts @@ -22,6 +22,43 @@ describe("buildTelemetryJoinKeys", () => { expect(keys).toBe("fid=f tid=t renders=render-a,render-b!"); }); + it("names the last check's exit code and top error codes", () => { + const keys = buildTelemetryJoinKeys({ + feedbackId: "f", + anonymousId: "t", + lastCheck: { at: "", exitCode: 1, codes: { text_overlap: 4, contrast_low: 1 } }, + }); + expect(keys).toBe("fid=f tid=t check=exit1:text_overlap*4,contrast_low*1"); + }); + + it("says unfinished when the last check never reported an exit code", () => { + const keys = buildTelemetryJoinKeys({ + feedbackId: "f", + anonymousId: "t", + lastCheck: { at: "", codes: {} }, + }); + expect(keys).toBe("fid=f tid=t check=unfinished"); + }); + + it("says how many minutes ago the last check ran, so one from another project is spotted", () => { + const keys = buildTelemetryJoinKeys({ + feedbackId: "f", + anonymousId: "t", + lastCheck: { at: "2026-10-29T10:00:00.000Z", exitCode: 1, codes: { text_overlap: 4 } }, + now: Date.parse("2026-10-29T10:12:30.000Z"), + }); + expect(keys).toBe("fid=f tid=t check=exit1:text_overlap*4 check_age=13m"); + }); + + it("says only the exit code when the last check had no error codes", () => { + const keys = buildTelemetryJoinKeys({ + feedbackId: "f", + anonymousId: "t", + lastCheck: { at: "", exitCode: 0, codes: {} }, + }); + expect(keys).toBe("fid=f tid=t check=exit0"); + }); + it("stays within the backend env cap for a full ring of uuid render ids", () => { const uuid = "01234567-89ab-cdef-0123-456789abcdef"; const keys = buildTelemetryJoinKeys({ @@ -36,4 +73,20 @@ describe("buildTelemetryJoinKeys", () => { // submitFeedback caps env at 500 chars; the doctor summary consumes ~100. expect(keys.length).toBeLessThan(400); }); + + it("still fits the env cap with a full ring and three long error codes", () => { + const uuid = "01234567-89ab-cdef-0123-456789abcdef"; + const code = "x".repeat(60); + const keys = buildTelemetryJoinKeys({ + feedbackId: uuid, + anonymousId: uuid, + recentRenders: Array.from({ length: 5 }, () => ({ id: uuid, at: "", ok: true })), + lastCheck: { + at: "", + exitCode: 1, + codes: { [code + "a"]: 9, [code + "b"]: 8, [code + "c"]: 7 }, + }, + }); + expect(keys.length).toBeLessThan(500 - 100); + }); }); diff --git a/packages/cli/src/commands/feedback.ts b/packages/cli/src/commands/feedback.ts index ac4f166000..cf29c43c6e 100644 --- a/packages/cli/src/commands/feedback.ts +++ b/packages/cli/src/commands/feedback.ts @@ -12,7 +12,7 @@ import { } from "../telemetry/events.js"; import { shouldTrack } from "../telemetry/client.js"; import { getDoctorSummary } from "../telemetry/feedback.js"; -import { readConfig, type RecentRenderRecord } from "../telemetry/config.js"; +import { readConfig, type LastCheckRecord, type RecentRenderRecord } from "../telemetry/config.js"; import { publishProjectArchive } from "../utils/publishProject.js"; import { submitCatalogSearchMiss, submitFeedback } from "../utils/submitFeedback.js"; import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js"; @@ -57,11 +57,25 @@ export function buildTelemetryJoinKeys(input: { feedbackId: string; anonymousId: string; recentRenders?: RecentRenderRecord[]; + lastCheck?: LastCheckRecord; + now?: number; }): string { const parts = [`fid=${input.feedbackId}`, `tid=${input.anonymousId}`]; if (input.recentRenders?.length) { parts.push(`renders=${input.recentRenders.map((r) => `${r.id}${r.ok ? "" : "!"}`).join(",")}`); } + if (input.lastCheck) { + const codes = Object.entries(input.lastCheck.codes).map( + ([code, n]) => `${code.slice(0, 24)}*${n}`, + ); + const result = + input.lastCheck.exitCode === undefined ? "unfinished" : `exit${input.lastCheck.exitCode}`; + parts.push(`check=${result}${codes.length ? `:${codes.join(",")}` : ""}`); + const minutes = Math.round( + ((input.now ?? Date.now()) - Date.parse(input.lastCheck.at)) / 60_000, + ); + if (Number.isFinite(minutes) && minutes >= 0) parts.push(`check_age=${minutes}m`); + } return parts.join(" "); } @@ -298,6 +312,7 @@ export default defineCommand({ feedbackId, anonymousId: config.anonymousId, recentRenders: config.recentRenders, + lastCheck: config.lastCheck, }); const envWithJoinKeys = doctorSummary ? `${doctorSummary} ${joinKeys}` : joinKeys; diff --git a/packages/cli/src/telemetry/config.ts b/packages/cli/src/telemetry/config.ts index 0af74c5982..430a5e2d85 100644 --- a/packages/cli/src/telemetry/config.ts +++ b/packages/cli/src/telemetry/config.ts @@ -575,6 +575,23 @@ export interface HyperframesConfig { * telemetry rows of the renders it describes. */ recentRenders?: RecentRenderRecord[]; + /** The newest local `check`: exit code and its top error codes, which `hyperframes feedback` attaches. */ + lastCheck?: LastCheckRecord; +} + +/** The newest `check` in {@link HyperframesConfig.lastCheck}; codes and counts only, no content or paths. */ +export interface LastCheckRecord { + at: string; + /** Absent while the check has not finished: it is still running, or was stopped. */ + exitCode?: number; + codes: Record; +} + +/** Remember the newest check so a later `hyperframes feedback` can name why it failed. */ +export function recordLastCheck(lastCheck: LastCheckRecord): void { + const config = readConfigFresh(); + config.lastCheck = lastCheck; + writeConfig(config); } /** One entry in {@link HyperframesConfig.recentRenders}. */ diff --git a/packages/cli/src/utils/checkFailureCodes.ts b/packages/cli/src/utils/checkFailureCodes.ts new file mode 100644 index 0000000000..930e9bffd2 --- /dev/null +++ b/packages/cli/src/utils/checkFailureCodes.ts @@ -0,0 +1,24 @@ +import type { CheckReport } from "./checkTypes.js"; + +const MAX_CODES = 3; + +/** The error codes of a check report with how often each fired, most frequent first; codes only, never content. */ +export function failureCodes(report: CheckReport): Record { + const counts = new Map(); + for (const section of [ + report.lint, + report.runtime, + report.layout, + report.motion, + report.contrast, + report.craft, + report.audio, + ]) { + for (const finding of section.findings) { + if (finding.severity === "error") + counts.set(finding.code, (counts.get(finding.code) ?? 0) + 1); + } + } + const top = [...counts].sort((a, b) => b[1] - a[1]).slice(0, MAX_CODES); + return Object.fromEntries(top); +}