quantized/base93-v3/verify_decoders.js

// Independent whole-file decoder verification against Python's little-endian hashes.
const fs=require('fs'),crypto=require('crypto'),assert=require('assert');
const {decodeBase93}=require('./decode_base93');
const text=fs.readFileSync(process.argv[2]||__dirname+'/weights_base93.txt','ascii');
const expected=JSON.parse(fs.readFileSync(__dirname+'/decoded_tensor_hashes.json','utf8'));
assert.strictEqual(JSON.parse(JSON.stringify(text)),text);
assert(Buffer.byteLength(JSON.stringify(text),'utf8')<5000000);
const {tensors}=decodeBase93(text);
assert.strictEqual(Object.keys(tensors).length,Object.keys(expected).length);
for(const [name,t] of Object.entries(tensors)) {
  const bytes=Buffer.alloc(t.values.length*(t.kind==='i'?8:4));
  for(let i=0;i<t.values.length;i++) {
    if(t.kind==='i')bytes.writeBigInt64LE(t.values[i],8*i);
    else bytes.writeFloatLE(t.values[i],4*i);
  }
  assert.strictEqual(crypto.createHash('sha256').update(bytes).digest('hex'),expected[name],name);
}
console.log(JSON.stringify({tensors:Object.keys(tensors).length,python_javascript_bitwise_match:true,json_string_bytes:Buffer.byteLength(JSON.stringify(text)),sha256:crypto.createHash('sha256').update(text,'ascii').digest('hex')}));