fix: require rendered evidence for every target viewport

This commit is contained in:
Samuel Bushi
2026-09-24 12:37:05 +02:00
parent 90481d5e4a
commit c770b08eef
3 changed files with 26 additions and 3 deletions
@@ -133,6 +133,9 @@ node <installed-skill>/scripts/check-evidence.js <local-report.json>
The record contains:
- `contract`: the checkable product constraints or a local contract reference.
- `targetViewports`: a nonempty array of viewport strings from the contract,
such as `["390x844", "1440x900"]`. Each target requires a matching `viewport`
entry in `renderedEvidence`; one mobile capture cannot cover a desktop target.
- `accessibility` and `responsive`: objects with `status: "pass"` and a nonempty
`evidence` observation describing the checks performed.
- `contentStates`: one object for each of `loading`, `empty`, `error`, `partial`,
@@ -17,8 +17,12 @@ function checkEvidence(report) {
return [`${name} needs a passing observation${allowNotApplicable ? ' or a not-applicable reason' : ''}`];
};
const renders = Array.isArray(report.renderedEvidence) ? report.renderedEvidence : [];
const targets = Array.isArray(report.targetViewports) ? report.targetViewports : [];
return [
...(text(report.contract) ? [] : ['contract is required']),
...(targets.length && targets.every(text) ? [] : ['targetViewports must contain nonempty viewport strings']),
...targets.filter(text).flatMap(viewport => renders.some(item => object(item) && item.viewport === viewport)
? [] : [`renderedEvidence is missing target viewport ${viewport}`]),
...check('accessibility', report.accessibility),
...check('responsive', report.responsive),
...states.flatMap(state => check(`contentStates.${state}`, report.contentStates?.[state], true)),
+19 -3
View File
@@ -13,6 +13,7 @@ function report() {
const observation = { status: 'pass', evidence: 'Observed in the local test run.' };
return {
contract: 'Settings: account owner can save changes and recover from failure.',
targetViewports: ['390x844'],
accessibility: { ...observation },
responsive: { ...observation },
contentStates: Object.fromEntries(['loading', 'empty', 'error', 'partial', 'success', 'permission']
@@ -24,7 +25,7 @@ function report() {
test('requires observed accessibility, responsive behavior and every content state', () => {
const evidence = report();
assert.deepEqual(checkEvidence(evidence), []);
for (const key of ['contract', 'accessibility', 'responsive', 'contentStates', 'renderedEvidence']) {
for (const key of ['contract', 'targetViewports', 'accessibility', 'responsive', 'contentStates', 'renderedEvidence']) {
const missing = { ...evidence };
delete missing[key];
assert.ok(checkEvidence(missing).length, `accepted missing ${key}`);
@@ -40,6 +41,16 @@ test('requires observed accessibility, responsive behavior and every content sta
assert.ok(checkEvidence({ ...evidence, responsive: { status: 'pass', evidence: ' ' } }).length);
});
test('requires rendered evidence for every declared target viewport', () => {
const evidence = { ...report(), targetViewports: ['390x844', '1440x900'] };
assert.ok(checkEvidence(evidence).length, 'mobile alone must not satisfy a desktop target');
const desktop = { viewport: '1440x900', artifact: 'desktop.png', observation: 'Long labels fit.' };
assert.deepEqual(checkEvidence({ ...evidence, renderedEvidence: [...evidence.renderedEvidence, desktop] }), []);
for (const targetViewports of [[], '390x844', [' '], [null], ['390x844', 1440]]) {
assert.ok(checkEvidence({ ...report(), targetViewports }).length, 'invalid targets must fail');
}
});
test('only content states may be not applicable with a reason', () => {
const evidence = report();
const contentStates = { ...evidence.contentStates, permission: { status: 'not-applicable', reason: 'Public view without restricted actions.' } };
@@ -57,6 +68,7 @@ test('rejects malformed reports and rendered evidence without inspection details
test('CLI requires a nonempty local artifact and fails safely on invalid input', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ui-evidence-'));
const external = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ui-external-'));
const input = path.join(dir, 'report.json');
const run = (...args) => spawnSync(process.execPath, [script, ...args], { encoding: 'utf8' });
try {
@@ -66,10 +78,13 @@ test('CLI requires a nonempty local artifact and fails safely on invalid input',
assert.equal(run(input).status, 0);
fs.writeFileSync(path.join(dir, 'render.png'), '');
assert.equal(run(input).status, 1, 'empty artifact must fail');
fs.writeFileSync(path.join(dir, 'render.png'), 'nonempty local artifact');
fs.writeFileSync(path.join(external, 'render.png'), 'nonempty external artifact');
fs.symlinkSync(path.join(dir, 'render.png'), path.join(dir, 'linked.png'));
fs.symlinkSync(os.tmpdir(), path.join(dir, 'outside'));
fs.symlinkSync(path.join(external, 'render.png'), path.join(dir, 'outside'));
fs.symlinkSync(external, path.join(dir, 'external-dir'), 'junction');
for (const artifact of ['.', 'https://example.com/render.png', path.join(dir, 'render.png'),
'../render.png', 'C:\\render.png', 'linked.png', 'outside']) {
'../render.png', 'C:\\render.png', 'linked.png', 'outside', 'external-dir/render.png']) {
fs.writeFileSync(input, JSON.stringify({ ...report(), renderedEvidence: [{ viewport: '390x844', artifact, observation: 'Checked.' }] }));
assert.equal(run(input).status, 1, 'directories and remote artifacts must fail');
}
@@ -82,5 +97,6 @@ test('CLI requires a nonempty local artifact and fails safely on invalid input',
assert.equal(run().status, 1);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
fs.rmSync(external, { recursive: true, force: true });
}
});