Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions packages/cli/src/commands/check.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ vi.mock("../telemetry/events.js", () => ({

import { contrastRatio, parseColorRGBA } from "./contrast-bg.js";
import { createCheckCommand, parseCheckRange } from "./check.js";
import { readConfig } from "../telemetry/config.js";
import { CheckDeadlineError, CheckStoppedError } from "../utils/checkDeadline.js";
import { trackCommandFailures } from "../utils/command-failure-tracking.js";
import type { CommandDef } from "citty";
Expand Down Expand Up @@ -1589,6 +1590,55 @@ describe("check pipeline", () => {
expect(said).not.toContain("Browser session never ran");
});

describe("remembers the newest check for hyperframes feedback", () => {
const run = (runPipeline: () => Promise<CheckReport>) =>
runViaCli(
createCheckCommand({
resolveProject: () => PROJECT,
runPipeline: vi.fn(runPipeline),
prepareMedia: vi.fn(async () => []),
withMeta: (value) => value,
}),
{ rawArgs: [] },
);

it("keeps the error codes of a failed check", async () => {
const lint = lintWith(
"error",
"root_missing_composition_id",
"Root needs data-composition-id.",
);
const { report } = await runScenario(fakeDriver(), {}, { lint });
await run(async () => report);

expect(readConfig().lastCheck).toMatchObject({
exitCode: 1,
codes: { root_missing_composition_id: 1 },
});
});

it("records a check that threw as a failure, not the check before it", async () => {
await run(async () => (await runScenario(fakeDriver(), {}, {})).report);
await run(async () => {
throw new Error("browser died");
});

expect(readConfig().lastCheck).toMatchObject({ exitCode: 1, codes: { runtime_error: 1 } });
});

it("leaves a check that never finished as unfinished, with when it started", async () => {
const { report } = await runScenario(fakeDriver(), {}, {});
await run(async () => report);
let finish!: (value: CheckReport) => void;
const running = run(() => new Promise<CheckReport>((resolve) => (finish = resolve)));
await vi.waitFor(() => expect(readConfig().lastCheck?.exitCode).toBeUndefined());

expect(readConfig().lastCheck?.at).toMatch(/^\d{4}-\d\d-\d\dT/);
finish(report);
await running;
});
});

it("marks browserSkipped true when the linter itself crashes", async () => {
const { deps } = dependencies(fakeDriver());
deps.lintProject = vi.fn(async () => {
Expand Down
13 changes: 13 additions & 0 deletions packages/cli/src/commands/check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ import { resolveLocalBrowserGpuMode } from "../browser/gpuPolicy.js";
import { preResolveHostileMediaProxies, type PreparedMediaFile } from "../utils/checkBrowser.js";
import { parseTimeoutMs } from "../utils/timeoutArg.js";
import { failureMessage, jsonFailure } from "../utils/jsonError.js";
import { recordLastCheck } from "../telemetry/config.js";
import { failureCodes } from "../utils/checkFailureCodes.js";

export const examples: Example[] = [
["Run the full verification gate", "hyperframes check"],
Expand Down Expand Up @@ -287,9 +289,15 @@ export function createCheckCommand(
await runPrepareMedia(dependencies, project, options, asJson);
return;
}
recordLastCheck({ at: new Date().toISOString(), codes: {} });
const report = await withProjectLock(project.dir, "check", () =>
dependencies.runPipeline(project, options),
);
recordLastCheck({
at: new Date().toISOString(),
exitCode: checkExitCode(report),
codes: failureCodes(report),
});
if (asJson) {
console.log(JSON.stringify(dependencies.withMeta(report), null, 2));
} else {
Expand All @@ -303,6 +311,11 @@ export function createCheckCommand(
}
setCommandExitCode(checkExitCode(report));
} catch (error) {
recordLastCheck({
at: new Date().toISOString(),
exitCode: 1,
codes: { [error instanceof CliUsageError ? error.code : "runtime_error"]: 1 },
});
reportCheckFailure(error, asJson, (value) => dependencies.withMeta(value));
} finally {
reported.done();
Expand Down
53 changes: 53 additions & 0 deletions packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,43 @@ describe("buildTelemetryJoinKeys", () => {
expect(keys).toBe("fid=f tid=t renders=render-a,render-b!");
});

it("names the last check's exit code and top error codes", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "f",
anonymousId: "t",
lastCheck: { at: "", exitCode: 1, codes: { text_overlap: 4, contrast_low: 1 } },
});
expect(keys).toBe("fid=f tid=t check=exit1:text_overlap*4,contrast_low*1");
});

it("says unfinished when the last check never reported an exit code", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "f",
anonymousId: "t",
lastCheck: { at: "", codes: {} },
});
expect(keys).toBe("fid=f tid=t check=unfinished");
});

it("says how many minutes ago the last check ran, so one from another project is spotted", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "f",
anonymousId: "t",
lastCheck: { at: "2026-10-29T10:00:00.000Z", exitCode: 1, codes: { text_overlap: 4 } },
now: Date.parse("2026-10-29T10:12:30.000Z"),
});
expect(keys).toBe("fid=f tid=t check=exit1:text_overlap*4 check_age=13m");
});

it("says only the exit code when the last check had no error codes", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "f",
anonymousId: "t",
lastCheck: { at: "", exitCode: 0, codes: {} },
});
expect(keys).toBe("fid=f tid=t check=exit0");
});

it("stays within the backend env cap for a full ring of uuid render ids", () => {
const uuid = "01234567-89ab-cdef-0123-456789abcdef";
const keys = buildTelemetryJoinKeys({
Expand All @@ -36,4 +73,20 @@ describe("buildTelemetryJoinKeys", () => {
// submitFeedback caps env at 500 chars; the doctor summary consumes ~100.
expect(keys.length).toBeLessThan(400);
});

it("still fits the env cap with a full ring and three long error codes", () => {
const uuid = "01234567-89ab-cdef-0123-456789abcdef";
const code = "x".repeat(60);
const keys = buildTelemetryJoinKeys({
feedbackId: uuid,
anonymousId: uuid,
recentRenders: Array.from({ length: 5 }, () => ({ id: uuid, at: "", ok: true })),
lastCheck: {
at: "",
exitCode: 1,
codes: { [code + "a"]: 9, [code + "b"]: 8, [code + "c"]: 7 },
},
});
expect(keys.length).toBeLessThan(500 - 100);
});
});
17 changes: 16 additions & 1 deletion packages/cli/src/commands/feedback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
} from "../telemetry/events.js";
import { shouldTrack } from "../telemetry/client.js";
import { getDoctorSummary } from "../telemetry/feedback.js";
import { readConfig, type RecentRenderRecord } from "../telemetry/config.js";
import { readConfig, type LastCheckRecord, type RecentRenderRecord } from "../telemetry/config.js";
import { publishProjectArchive } from "../utils/publishProject.js";
import { submitCatalogSearchMiss, submitFeedback } from "../utils/submitFeedback.js";
import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js";
Expand Down Expand Up @@ -57,11 +57,25 @@ export function buildTelemetryJoinKeys(input: {
feedbackId: string;
anonymousId: string;
recentRenders?: RecentRenderRecord[];
lastCheck?: LastCheckRecord;
now?: number;
}): string {
const parts = [`fid=${input.feedbackId}`, `tid=${input.anonymousId}`];
if (input.recentRenders?.length) {
parts.push(`renders=${input.recentRenders.map((r) => `${r.id}${r.ok ? "" : "!"}`).join(",")}`);
}
if (input.lastCheck) {
const codes = Object.entries(input.lastCheck.codes).map(
([code, n]) => `${code.slice(0, 24)}*${n}`,
);
const result =
input.lastCheck.exitCode === undefined ? "unfinished" : `exit${input.lastCheck.exitCode}`;
parts.push(`check=${result}${codes.length ? `:${codes.join(",")}` : ""}`);
const minutes = Math.round(
((input.now ?? Date.now()) - Date.parse(input.lastCheck.at)) / 60_000,
);
if (Number.isFinite(minutes) && minutes >= 0) parts.push(`check_age=${minutes}m`);
}
return parts.join(" ");
}

Expand Down Expand Up @@ -298,6 +312,7 @@ export default defineCommand({
feedbackId,
anonymousId: config.anonymousId,
recentRenders: config.recentRenders,
lastCheck: config.lastCheck,
});
const envWithJoinKeys = doctorSummary ? `${doctorSummary} ${joinKeys}` : joinKeys;

Expand Down
17 changes: 17 additions & 0 deletions packages/cli/src/telemetry/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,23 @@ export interface HyperframesConfig {
* telemetry rows of the renders it describes.
*/
recentRenders?: RecentRenderRecord[];
/** The newest local `check`: exit code and its top error codes, which `hyperframes feedback` attaches. */
lastCheck?: LastCheckRecord;
}

/** The newest `check` in {@link HyperframesConfig.lastCheck}; codes and counts only, no content or paths. */
export interface LastCheckRecord {
at: string;
/** Absent while the check has not finished: it is still running, or was stopped. */
exitCode?: number;
codes: Record<string, number>;
}

/** Remember the newest check so a later `hyperframes feedback` can name why it failed. */
export function recordLastCheck(lastCheck: LastCheckRecord): void {
const config = readConfigFresh();
config.lastCheck = lastCheck;
writeConfig(config);
}

/** One entry in {@link HyperframesConfig.recentRenders}. */
Expand Down
24 changes: 24 additions & 0 deletions packages/cli/src/utils/checkFailureCodes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import type { CheckReport } from "./checkTypes.js";

const MAX_CODES = 3;

/** The error codes of a check report with how often each fired, most frequent first; codes only, never content. */
export function failureCodes(report: CheckReport): Record<string, number> {
const counts = new Map<string, number>();
for (const section of [
report.lint,
report.runtime,
report.layout,
report.motion,
report.contrast,
report.craft,
report.audio,
]) {
for (const finding of section.findings) {
if (finding.severity === "error")
counts.set(finding.code, (counts.get(finding.code) ?? 0) + 1);
}
}
const top = [...counts].sort((a, b) => b[1] - a[1]).slice(0, MAX_CODES);
return Object.fromEntries(top);
}
Loading