tools/whistle-eval/build-data.mjs


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

// Rebuilds the fp32 ONNX external-data files from the compact whistle.pack.
// This is the "small weights" path: instead of committing ~145 MB of .onnx.data
// to git (or downloading it), keep the 17 MB pack and expand it on first run.
import fs from 'fs';
import { parsePack, buildModelData } from './vendor/js/pack.js';

const root = new URL('./vendor/', import.meta.url);
const packBytes = fs.readFileSync(new URL('whistle.pack', root));
const pack = parsePack(
  packBytes.buffer.slice(packBytes.byteOffset, packBytes.byteOffset + packBytes.byteLength));

for (const name of ['encoder', 'decoder']) {
  const target = new URL(`onnx/${name}.onnx.data`, root);
  const data = buildModelData(pack, name, (p) => {
    process.stdout.write(`\r${name}: ${(p * 100).toFixed(0)}%   `);
  });
  fs.writeFileSync(target, data);
  process.stdout.write(`\r${name}: wrote ${data.length} bytes\n`);
}