Evidence retrieval with a measured baseline

Prerequisite & prediction

Will adding more keywords prove semantic generalization?

Complete implementation & test runner

Download all four files into one folder. Run node check.mjs. The checks exercise all three projects; no installation or API key is required.

// Node.js 18+ or modern browser. Fixed corpus and labels; no generative model.
export const documents=[
  {id:'refund',text:'Refund policy: ask within thirty days.',tags:['reimbursement','window']},
  {id:'delivery',text:'Shipping delivery usually takes five days.',tags:['parcel','arrival']},
  {id:'support',text:'Contact support for damaged items.',tags:['broken','help']}
];
export const cases=[{q:'refund policy',gold:'refund'},{q:'reimbursement window',gold:'refund'},{q:'parcel arrival',gold:'delivery'},{q:'broken help',gold:'support'},{q:'astronomy telescope',gold:null}];
const words=s=>new Set(s.toLowerCase().match(/[a-z]+/g)||[]);
export function retrieve(query, useTags=false) {
  const q=words(query);
  return documents.map(d=>({id:d.id,score:[...q].filter(w=>words(d.text+(useTags?' '+d.tags.join(' '):'')).has(w)).length}))
    .filter(d=>d.score>0).sort((a,b)=>b.score-a.score||a.id.localeCompare(b.id))[0]?.id||null;
}
export function evaluate(useTags) {
  const rows=cases.map(c=>({...c,predicted:retrieve(c.q,useTags)}));
  return {accuracy:rows.filter(r=>r.predicted===r.gold).length/rows.length,
    answerableRecall:rows.filter(r=>r.gold&&r.predicted===r.gold).length/rows.filter(r=>r.gold).length,
    correctAbstentions:rows.filter(r=>r.gold===null&&r.predicted===null).length,rows};
}
export function run(){return {baseline:evaluate(false),treatment:evaluate(true),
  limitation:'Five development examples. Tags were authored with these concepts in mind; this is not a held-out generalization claim.'}}

import assert from 'node:assert/strict';
import {run as attention,softmax,attend,full,cached} from './attention.mjs';
import {run as retrieval,retrieve} from './retrieval.mjs';
import {run as control,simulate} from './control.mjs';
assert.ok(attention().passes);assert.throws(()=>softmax([-Infinity,-Infinity]));assert.throws(()=>attend([1,2],[[1]],[[1]]));
assert.equal(retrieval().baseline.accuracy,.4);assert.equal(retrieval().treatment.accuracy,1);assert.equal(retrieve('unseen vocabulary'),null);
const result=control();assert.ok(result.baseline.maxForce<=5);assert.ok(result.delayed.maxForce<=5);assert.equal(simulate({steps:1}).trace[0].x,0);assert.equal(simulate({steps:1}).trace[0].v,.05);assert.throws(()=>simulate({mass:0}));

// Full and cached paths must reject inconsistent shapes, not just the single-query helper.
assert.throws(()=>full([[1,2]],[[1]],[[1]]),/Shape mismatch/);
assert.throws(()=>cached([[1,2]],[],[]),/sequence shape/);

console.log('PASS: 12 capstone checks. Toy arithmetic and invariants only; no GPU or hardware benchmark.');

Transfer task & ablation

Freeze tags, collect ten unseen paraphrases and five unrelated queries, then measure again without tuning on them.

Rubric

Preserve document IDs; report baseline/treatment rows; distinguish false matches from missed retrieval; provide a genuinely held-out evaluation.

Score each of the four criteria 0 (missing), 1 (with help), 2 (independent evidence). Passing requires 2 on every criterion. A self-check is not a certification.