net.ts
ts8 kB266 linesIn cnn-visualizer.zip
cnn-visualizer.zipsrcnnnet.ts
// A tiny CNN, written out by hand so every activation and gradient can be drawn.
//
// input 28×28 → conv 3×3, 6 filters → 26×26×6 → ReLU → max pool 2×2 → 13×13×6
// → flatten 1014 → dense 16 (ReLU) → dense 10 → softmax
//
// 60 + 6 + 16,224 + 16 + 160 + 10 = 16,476 parameters.
export const S = 28; // input side
export const K = 3; // kernel side
export const F = 6; // number of filters
export const C = S - K + 1; // conv output side: 26
export const P = C / 2; // pooled side: 13
export const CC = C * C; // 676 pixels per feature map
export const PP = P * P; // 169 pixels per pooled map
export const FLAT = F * PP; // 1014
export const H = 16; // hidden neurons
export const O = 10; // classes
export type Params = {
k: Float32Array; // F×3×3 kernels
kb: Float32Array; // F conv biases
w1: Float32Array; // H×FLAT, row j holds the weights into hidden neuron j
b1: Float32Array; // H
w2: Float32Array; // O×H
b2: Float32Array; // O
};
// Gradients have exactly the same shapes as the parameters.
export type Grads = Params;
export type Trace = {
x: Float32Array; // input, 0..1
label: number;
conv: Float32Array; // F×26×26, before ReLU
relu: Float32Array; // F×26×26
pool: Float32Array; // F×13×13
arg: Int32Array; // for each pooled pixel, which of the 676 pixels in its map was the max
hPre: Float32Array;
h: Float32Array;
z: Float32Array; // logits
p: Float32Array; // softmax probabilities
loss: number;
// backward pass, named after what they are the gradient of
dz: Float32Array; // = p − onehot
dh: Float32Array;
dhPre: Float32Array;
dPool: Float32Array;
dRelu: Float32Array; // sparse: only the max positions receive gradient
dConv: Float32Array;
g: Grads; // this sample's weight gradients
};
export function zeroParams(): Params {
return {
k: new Float32Array(F * K * K),
kb: new Float32Array(F),
w1: new Float32Array(H * FLAT),
b1: new Float32Array(H),
w2: new Float32Array(O * H),
b2: new Float32Array(O),
};
}
export function cloneParams(p: Params): Params {
return { k: p.k.slice(), kb: p.kb.slice(), w1: p.w1.slice(), b1: p.b1.slice(), w2: p.w2.slice(), b2: p.b2.slice() };
}
const KEYS = ["k", "kb", "w1", "b1", "w2", "b2"] as const;
export function copyParams(from: Params, to: Params): void {
for (const key of KEYS) to[key].set(from[key]);
}
function mulberry32(seed: number) {
return () => {
seed |= 0;
seed = (seed + 0x6d2b79f5) | 0;
let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
// He initialisation: weights ~ N(0, 2 / fan_in), biases 0.
export function initParams(seed = Date.now()): Params {
const rand = mulberry32(seed);
const gauss = () => Math.sqrt(-2 * Math.log(rand() + 1e-12)) * Math.cos(2 * Math.PI * rand());
const p = zeroParams();
const fill = (a: Float32Array, fanIn: number) => {
const std = Math.sqrt(2 / fanIn);
for (let i = 0; i < a.length; i++) a[i] = gauss() * std;
};
fill(p.k, K * K);
fill(p.w1, FLAT);
fill(p.w2, H);
return p;
}
export function newTrace(): Trace {
return {
x: new Float32Array(S * S),
label: 0,
conv: new Float32Array(F * CC),
relu: new Float32Array(F * CC),
pool: new Float32Array(FLAT),
arg: new Int32Array(FLAT),
hPre: new Float32Array(H),
h: new Float32Array(H),
z: new Float32Array(O),
p: new Float32Array(O),
loss: 0,
dz: new Float32Array(O),
dh: new Float32Array(H),
dhPre: new Float32Array(H),
dPool: new Float32Array(FLAT),
dRelu: new Float32Array(F * CC),
dConv: new Float32Array(F * CC),
g: zeroParams(),
};
}
export function forward(p: Params, t: Trace): void {
const { x, conv, relu, pool, arg, hPre, h, z, p: prob } = t;
// convolution: slide each 3×3 kernel over the image, multiply and sum
for (let f = 0; f < F; f++) {
for (let r = 0; r < C; r++) {
for (let c = 0; c < C; c++) {
let s = p.kb[f];
for (let i = 0; i < K; i++) {
for (let j = 0; j < K; j++) s += x[(r + i) * S + c + j] * p.k[f * 9 + i * K + j];
}
const o = f * CC + r * C + c;
conv[o] = s;
relu[o] = s > 0 ? s : 0;
}
}
}
// 2×2 max pooling, remembering where each max came from (needed for backprop)
for (let f = 0; f < F; f++) {
for (let r = 0; r < P; r++) {
for (let c = 0; c < P; c++) {
let best = -Infinity;
let at = 0;
for (let i = 0; i < 2; i++) {
for (let j = 0; j < 2; j++) {
const q = (2 * r + i) * C + 2 * c + j;
const v = relu[f * CC + q];
if (v > best) {
best = v;
at = q;
}
}
}
const o = f * PP + r * P + c;
pool[o] = best;
arg[o] = at;
}
}
}
for (let j = 0; j < H; j++) {
let s = p.b1[j];
const row = j * FLAT;
for (let i = 0; i < FLAT; i++) s += p.w1[row + i] * pool[i];
hPre[j] = s;
h[j] = s > 0 ? s : 0;
}
let max = -Infinity;
for (let k = 0; k < O; k++) {
let s = p.b2[k];
for (let j = 0; j < H; j++) s += p.w2[k * H + j] * h[j];
z[k] = s;
if (s > max) max = s;
}
let sum = 0;
for (let k = 0; k < O; k++) {
prob[k] = Math.exp(z[k] - max);
sum += prob[k];
}
for (let k = 0; k < O; k++) prob[k] /= sum;
t.loss = -Math.log(Math.max(prob[t.label], 1e-12));
}
// Fills every gradient in the trace. Call forward first.
export function backward(p: Params, t: Trace): void {
const { x, conv, pool, arg, hPre, h, dz, dh, dhPre, dPool, dRelu, dConv, g } = t;
// softmax + cross-entropy: δ = p − t
for (let k = 0; k < O; k++) dz[k] = t.p[k] - (k === t.label ? 1 : 0);
for (let k = 0; k < O; k++) {
g.b2[k] = dz[k];
for (let j = 0; j < H; j++) g.w2[k * H + j] = dz[k] * h[j];
}
for (let j = 0; j < H; j++) {
let s = 0;
for (let k = 0; k < O; k++) s += dz[k] * p.w2[k * H + j];
dh[j] = s;
dhPre[j] = hPre[j] > 0 ? s : 0;
}
dPool.fill(0);
for (let j = 0; j < H; j++) {
const d = dhPre[j];
const row = j * FLAT;
g.b1[j] = d;
for (let i = 0; i < FLAT; i++) {
g.w1[row + i] = d * pool[i];
dPool[i] += d * p.w1[row + i];
}
}
// max pooling sends each gradient back only to the pixel that won
dRelu.fill(0);
for (let f = 0; f < F; f++) {
for (let q = 0; q < PP; q++) dRelu[f * CC + arg[f * PP + q]] += dPool[f * PP + q];
}
for (let i = 0; i < F * CC; i++) dConv[i] = conv[i] > 0 ? dRelu[i] : 0;
// a kernel weight is shared by all 676 positions, so its gradient is a sum over them
for (let f = 0; f < F; f++) {
let bsum = 0;
for (let i = 0; i < K; i++) {
for (let j = 0; j < K; j++) {
let s = 0;
for (let r = 0; r < C; r++) {
for (let c = 0; c < C; c++) s += dConv[f * CC + r * C + c] * x[(r + i) * S + c + j];
}
g.k[f * 9 + i * K + j] = s;
}
}
for (let q = 0; q < CC; q++) bsum += dConv[f * CC + q];
g.kb[f] = bsum;
}
}
export function addGrads(acc: Grads, g: Grads): void {
for (const key of KEYS) {
const a = acc[key];
const b = g[key];
for (let i = 0; i < a.length; i++) a[i] += b[i];
}
}
export function zeroGrads(g: Grads): void {
for (const key of KEYS) g[key].fill(0);
}
// w ← w − η · ∂E/∂w, with the gradient averaged over `n` samples
export function sgdStep(p: Params, g: Grads, lr: number, n = 1): void {
const s = lr / n;
for (const key of KEYS) {
const a = p[key];
const b = g[key];
for (let i = 0; i < a.length; i++) a[i] -= s * b[i];
}
}
export function predict(t: Trace): number {
let best = 0;
for (let k = 1; k < O; k++) if (t.p[k] > t.p[best]) best = k;
return best;
}