122 lines
14 KiB
JavaScript
122 lines
14 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
|
|
export const correctnessCases=[
|
|
{id:'reset',re:'-0.5',im:'0',span:'3.4',bits:256,baseIter:750,adaptive:false,processMode:'validate',renderMode:'fast',activeDirect:true,certified:true,frontierCheck:true},
|
|
{id:'cusp-low',re:'0.25',im:'0',span:'0.85',bits:256,baseIter:900,adaptive:false,processMode:'validate',renderMode:'accurate',certified:true},
|
|
{id:'seahorse-low',re:'-0.743643887037151',im:'0.13182590420533',span:'0.1',bits:320,baseIter:1200,adaptive:false,processMode:'validate',renderMode:'accurate'},
|
|
{id:'moderate-1e5-fast',re:'-0.743643887037151',im:'0.13182590420533',span:'0.000034',bits:384,baseIter:350,adaptive:true,processMode:'standard',renderMode:'fast',frontierCheck:true},
|
|
{id:'moderate-1e5-accurate',re:'-0.743643887037151',im:'0.13182590420533',span:'0.000034',bits:384,baseIter:350,adaptive:true,processMode:'standard',renderMode:'accurate',frontierCheck:true},
|
|
{id:'moderate-1e8-fast',re:'-0.743643887037151',im:'0.13182590420533',span:'0.000000034',bits:384,baseIter:350,adaptive:true,processMode:'standard',renderMode:'fast',frontierCheck:true},
|
|
{id:'moderate-1e8-accurate',re:'-0.743643887037151',im:'0.13182590420533',span:'0.000000034',bits:384,baseIter:350,adaptive:true,processMode:'standard',renderMode:'accurate',frontierCheck:true},
|
|
{id:'seahorse-deep',re:'-0.743643887037151',im:'0.13182590420533',span:'0.00000000000034',bits:448,baseIter:2900,adaptive:false,processMode:'validate',renderMode:'accurate',activeDeep:true,frontierCheck:true},
|
|
{id:'interior-heavy',re:'0',im:'0',span:'0.5',bits:320,baseIter:1500,adaptive:false,processMode:'validate',renderMode:'accurate',certified:true},
|
|
{id:'real-axis-boundary',re:'-0.75',im:'0',span:'0.02',bits:384,baseIter:3000,adaptive:false,processMode:'validate',renderMode:'accurate',activeDeep:true,certified:true,frontierCheck:true},
|
|
{id:'misiurewicz-hard',re:'-0.10109636384562',im:'0.95628651080914',span:'0.00000002',bits:384,baseIter:3000,adaptive:false,processMode:'validate',renderMode:'accurate',activeDeep:true,frontierCheck:true},
|
|
{id:'reported-black-regression',re:'-29466147000382485924219538765424656674342940730553342389512209954887705938978413587',im:'5179018789925933872641942859702187326563010594882200888329237785122225251258352',span:'250637324516887125119843167990565159991251418774166207381437546036409',bits:273,baseIter:350,adaptive:true,processMode:'standard',renderMode:'accurate',fixed:true,frontierCheck:true}
|
|
];
|
|
|
|
export const performanceCases=[
|
|
{id:'accurate-320x220-750',viewport:{width:320,height:220},view:correctnessCases[2],iterations:750},
|
|
{id:'bla-654x690-2900',viewport:{width:654,height:690},view:correctnessCases[7],iterations:2900},
|
|
{id:'fast-1e14-800x600',viewport:{width:800,height:600},view:{id:'fast-1e14',re:'-0.743643887037151',im:'0.13182590420533',span:'0.000000000000034',bits:448,baseIter:350,adaptive:true,processMode:'standard',renderMode:'fast'},target:true,absoluteP50Ms:1000,absoluteP95Ms:1500},
|
|
{id:'accurate-1e14-800x600',viewport:{width:800,height:600},view:{id:'accurate-1e14',re:'-0.743643887037151',im:'0.13182590420533',span:'0.000000000000034',bits:448,baseIter:350,adaptive:true,processMode:'standard',renderMode:'accurate'},target:true,absoluteP50Ms:1500,absoluteP95Ms:2000},
|
|
{id:'hd-reset-1500',viewport:{width:1920,height:1080},view:correctnessCases[0],iterations:1500},
|
|
{id:'hd-boundary-3000',viewport:{width:1920,height:1080},view:correctnessCases[9],iterations:3000}
|
|
];
|
|
|
|
export const productionPerformanceThresholds={targetP50Reduction:.20,targetP95Reduction:.15,nonTargetP50Regression:.05,nonTargetP95Regression:.10};
|
|
|
|
export const interactionStabilityViews=[
|
|
{id:'near-a',re:'-0.743643887037151',im:'0.13182590420533',span:'0.0000000000034',bits:448,baseIter:350,adaptive:true,processMode:'standard',renderMode:'fast'},
|
|
{id:'near-b',re:'-0.743643887037151',im:'0.13182590420533',span:'0.000000000003030253189',bits:448,baseIter:350,adaptive:true,processMode:'standard',renderMode:'fast'},
|
|
{id:'near-a-repeat',re:'-0.743643887037151',im:'0.13182590420533',span:'0.0000000000034',bits:448,baseIter:350,adaptive:true,processMode:'standard',renderMode:'fast'}
|
|
];
|
|
|
|
function numericalReasonTotal(reasons={}){return (reasons.errorBound||0)+(reasons.escapeUncertain||0)+(reasons.referenceEnd||0)+(reasons.rebaseGap||0)+(reasons.range||0)}
|
|
function operationLimit(reasons={}){return reasons.operationLimit||0}
|
|
|
|
async function evaluateCase(page,testCase,options){
|
|
return page.evaluate(async({testCase,options})=>{
|
|
const t=globalThis.__MANDEL_TEST__;if(!t)throw new Error('__MANDEL_TEST__ is unavailable');
|
|
await t.setView(testCase);await t.waitForFinal({timeout:options.timeout});
|
|
let numericComplete=true,numericError='';try{await t.waitForNumericComplete({timeout:options.timeout})}catch(error){numericComplete=false;numericError=String(error&&error.message||error)}
|
|
const state=t.state(),hashes=await t.fieldHashes(),diagnostics=t.gpuDiagnostics();
|
|
let activeDirect=null,activeDeep=null,certified=null,frontier=null;
|
|
if(testCase.activeDirect)activeDirect=await t.runActiveDirectShadow({chunkSize:128});
|
|
if(testCase.activeDeep)activeDeep=await t.runActiveDeepShadow({width:options.shadowWidth,height:options.shadowHeight,chunkSize:128,strict:true});
|
|
if(testCase.certified)certified=await t.runCertifiedInteriorShadow({width:options.oracleWidth,height:options.oracleHeight,samples:options.oracleSamples});
|
|
if(testCase.frontierCheck)frontier=await t.compareCurrentToFrontierBaseline({strict:true});
|
|
return{state,hashes,diagnostics,numericComplete,numericError,activeDirect,activeDeep,certified,frontier};
|
|
},{testCase,options});
|
|
}
|
|
|
|
function auditCase(testCase,result,expected){
|
|
const reasons=result.state.unknownReasons||{},numeric=numericalReasonTotal(reasons),op=operationLimit(reasons),coverage=Math.max(0,(result.state.unresolved||0)-numeric-op),errors=(result.diagnostics?.compilation||[]).flatMap(x=>x.messages||[]).filter(x=>x.type==='error');
|
|
assert.equal(result.state.kernelVersion,'24.3.0-optimized-unknown-pipeline',testCase.id+' kernel version');
|
|
assert.equal(coverage,0,testCase.id+' coverage UNKNOWN');
|
|
assert.equal(errors.length,0,testCase.id+' WGSL compilation errors');
|
|
assert.equal(result.diagnostics?.uncapturedErrors?.length||0,0,testCase.id+' uncaptured GPU errors');
|
|
assert.equal(result.diagnostics?.lossReason||'','',testCase.id+' device loss');
|
|
assert.equal(result.numericComplete,true,testCase.id+' numeric completion: '+result.numericError);
|
|
assert.equal(numeric,0,testCase.id+' final numeric UNKNOWN');
|
|
assert.equal(result.state.historyReady,true,testCase.id+' completed history commit');
|
|
if(result.activeDirect)assert.equal(result.activeDirect.gatePassed,true,testCase.id+' Direct active mismatch');
|
|
if(result.activeDeep)assert.equal(result.activeDeep.gatePassed,true,testCase.id+' Deep active mismatch');
|
|
if(result.certified)assert.equal(result.certified.gatePassed,true,testCase.id+' certified interior oracle mismatch');
|
|
if(result.frontier)assert.equal(result.frontier.gatePassed,true,testCase.id+' production frontier mismatch');
|
|
assert.equal(result.state.frontierConverged,true,testCase.id+' frontier convergence');
|
|
assert.ok((result.state.frontierMaxDispatchWork||0)<=4000000,testCase.id+' frontier TDR work bound');
|
|
if(expected){assert.equal(result.hashes.metaSha256,expected.metaSha256,testCase.id+' meta hash');assert.equal(result.hashes.smoothSha256,expected.smoothSha256,testCase.id+' smooth hash')}
|
|
return{coverageUnknown:coverage,numericalUnknown:numeric,operationLimit:op};
|
|
}
|
|
|
|
export async function runCorrectnessCorpus(page,{timeout=180000,viewport={width:480,height:320},shadowWidth=160,shadowHeight=112,oracleWidth=320,oracleHeight=220,oracleSamples=289,expectedHashes={},expectProduction=false}={}){
|
|
if(page.setViewportSize)await page.setViewportSize(viewport);
|
|
const compile=await page.evaluate(()=>globalThis.__MANDEL_TEST__.compileActiveShadow()),records=[];
|
|
assert.equal(compile.productionEligible,expectProduction);
|
|
assert.equal(compile.candidateEnabled,true);
|
|
assert.equal(compile.pixelFrontier,true);
|
|
assert.equal(compile.frontierWorkLimit,4000000);
|
|
for(const testCase of correctnessCases){const started=Date.now(),result=await evaluateCase(page,testCase,{timeout,shadowWidth,shadowHeight,oracleWidth,oracleHeight,oracleSamples}),audit=auditCase(testCase,result,expectedHashes[testCase.id]);records.push({id:testCase.id,elapsedMs:Date.now()-started,audit,...result})}
|
|
const adapter=await page.evaluate(()=>globalThis.__MANDEL_TEST__.gpuDiagnostics()?.adapterInfo||{});
|
|
return{version:'24.3.0',kind:'correctness',date:new Date().toISOString(),expectProduction,compile,adapter,records};
|
|
}
|
|
|
|
function percentile(values,p){const sorted=[...values].sort((a,b)=>a-b);if(!sorted.length)return Infinity;const index=Math.min(sorted.length-1,Math.max(0,Math.ceil(sorted.length*p)-1));return sorted[index]}
|
|
function summarizeRuns(samples){const primary=samples.map(x=>Number(x.metric?.primaryMs??x.wallMs)),recovery=samples.map(x=>Number(x.metric?.numericQuiescentMs??x.metric?.historyCommitMs??x.metric?.finalMs??x.wallMs));return{runs:samples.length,primary:{p50:percentile(primary,.50),p95:percentile(primary,.95)},recovery:{p50:percentile(recovery,.50),p95:percentile(recovery,.95)}}}
|
|
export function evaluateProductionPerformance(candidate,baseline,{thresholds=productionPerformanceThresholds}={}){const failures=[],byId=new Map((baseline?.records||[]).map(x=>[x.id,x]));for(const record of candidate?.records||[]){const base=byId.get(record.id),summary=record.summary||summarizeRuns(record.runs||[]);if(record.absoluteP50Ms&&summary.recovery.p50>record.absoluteP50Ms)failures.push(record.id+' recovery p50 '+summary.recovery.p50+' > '+record.absoluteP50Ms);if(record.absoluteP95Ms&&summary.recovery.p95>record.absoluteP95Ms)failures.push(record.id+' recovery p95 '+summary.recovery.p95+' > '+record.absoluteP95Ms);if(!base){failures.push(record.id+' missing baseline');continue}const bs=base.summary||summarizeRuns(base.runs||[]),p50=(summary.recovery.p50-bs.recovery.p50)/Math.max(1,bs.recovery.p50),p95=(summary.recovery.p95-bs.recovery.p95)/Math.max(1,bs.recovery.p95);if(record.target){if(p50>-thresholds.targetP50Reduction)failures.push(record.id+' target p50 reduction '+p50);if(p95>-thresholds.targetP95Reduction)failures.push(record.id+' target p95 reduction '+p95)}else{if(p50>thresholds.nonTargetP50Regression)failures.push(record.id+' non-target p50 regression '+p50);if(p95>thresholds.nonTargetP95Regression)failures.push(record.id+' non-target p95 regression '+p95)}}return{pass:failures.length===0,failures}}
|
|
|
|
export async function runPerformanceCorpus(page,{runs=10,timeout=240000,includeLarge=false}={}){
|
|
const cases=includeLarge?performanceCases:performanceCases.slice(0,2),records=[];
|
|
for(const scenario of cases){if(page.setViewportSize)await page.setViewportSize(scenario.viewport);const view=scenario.iterations?{...scenario.view,baseIter:scenario.iterations,adaptive:false,processMode:'standard'}:{...scenario.view},samples=[];for(let run=0;run<runs+1;run++){const started=Date.now();await page.evaluate(async({view,timeout})=>{const t=globalThis.__MANDEL_TEST__;await t.setView(view);await t.waitForNumericComplete({timeout})},{view,timeout});const state=await page.evaluate(()=>globalThis.__MANDEL_TEST__.state()),metric=state.generationMetrics.at(-1),diagnostics=await page.evaluate(()=>globalThis.__MANDEL_TEST__.gpuDiagnostics());assert.equal(state.numericalFailures,0,scenario.id+' numerical failures');assert.equal(diagnostics?.uncapturedErrors?.length||0,0,scenario.id+' uncaptured GPU errors');assert.equal(diagnostics?.lossReason||'','',scenario.id+' device loss');if(run>0)samples.push({wallMs:Date.now()-started,metric})}records.push({id:scenario.id,viewport:scenario.viewport,target:!!scenario.target,absoluteP50Ms:scenario.absoluteP50Ms||0,absoluteP95Ms:scenario.absoluteP95Ms||0,runs:samples,summary:summarizeRuns(samples)})}
|
|
const exports=await page.evaluate(async()=>({ss1:await globalThis.__MANDEL_TEST__.smokeExportTile({w:128,h:96,strict:true,ss:1}),ss2:await globalThis.__MANDEL_TEST__.smokeExportTile({w:128,h:96,strict:true,ss:2})}));
|
|
assert.equal(exports.ss1.length,exports.ss1.expected);assert.equal(exports.ss2.length,exports.ss2.expected);
|
|
const adapter=await page.evaluate(()=>globalThis.__MANDEL_TEST__.gpuDiagnostics()?.adapterInfo||{});
|
|
return{version:'24.3.0',kind:'performance',date:new Date().toISOString(),adapter,records,exports};
|
|
}
|
|
|
|
export async function runInteractionStabilityCorpus(page,{timeout=240000,expectProduction=false}={}){
|
|
if(page.setViewportSize)await page.setViewportSize({width:800,height:600});
|
|
const policy=await page.evaluate(()=>globalThis.__MANDEL_TEST__.stabilityPolicy());
|
|
assert.equal(policy.adaptiveIterationQuantum,64);
|
|
assert.equal(policy.blaFastProductionOptIn,false);
|
|
assert.equal(policy.blaFastDisabled,true);
|
|
assert.equal(policy.pixelFrontierProduction,expectProduction);
|
|
assert.equal(policy.pixelFrontierCandidate,true);
|
|
assert.equal(policy.pixelFrontierEnabled,true);
|
|
assert.equal(policy.pixelFrontierWorkLimit,4000000);
|
|
const records=[];
|
|
for(const view of interactionStabilityViews){
|
|
const result=await page.evaluate(async({view,timeout})=>{const t=globalThis.__MANDEL_TEST__;await t.setView(view);await t.waitForNumericComplete({timeout});return{state:t.state(),hashes:await t.fieldHashes(),policy:t.stabilityPolicy()}},{view,timeout});
|
|
assert.notEqual(result.state.backend,'fast-bla',view.id+' timing-selected backend');
|
|
assert.equal(result.state.numericalFailures,0,view.id+' numerical failures');
|
|
records.push({id:view.id,...result});
|
|
}
|
|
assert.equal(records[0].state.iter,records[1].state.iter,'nearby zoom iteration envelope');
|
|
assert.equal(records[0].hashes.metaSha256,records[2].hashes.metaSha256,'repeat meta hash');
|
|
assert.equal(records[0].hashes.smoothSha256,records[2].hashes.smoothSha256,'repeat smooth hash');
|
|
const adapter=await page.evaluate(()=>globalThis.__MANDEL_TEST__.gpuDiagnostics()?.adapterInfo||{});
|
|
return{version:'24.3.0',kind:'interaction-stability',date:new Date().toISOString(),expectProduction,policy,adapter,records};
|
|
}
|
|
|
|
export async function runWebGpuCorpus(page,options={}){const correctness=await runCorrectnessCorpus(page,options.correctness),stability=await runInteractionStabilityCorpus(page,options.stability),performance=await runPerformanceCorpus(page,options.performance);return{correctness,stability,performance}}
|