Try first

A network is training badly. Before you build anything, list what you would want to see on screen to work out why.

You have met every failure this section covers. Your list should have four or five things on it.

What to show and what each thing catches

training and validation loss   overfitting, underfitting, when to stop
gradient size per layer        vanishing, exploding, dead layers
weight size per layer          runaway weights, whether decay is working
dead unit fraction             relu units lost
the decision boundary          what the model actually does

Five panels, and each one maps to a specific lesson: 3096, 3092, 3098, 3101 and 3075. Together they cover nearly everything that goes wrong.

Layout




 
const dash = document.getElementById("dash");
const dctx = dash.getContext("2d");

function panel(x, y, w, h, title) {
  dctx.strokeStyle = "#ccc";
  dctx.strokeRect(x, y, w, h);
  dctx.fillStyle = "#333";
  dctx.font = "12px system-ui";
  dctx.fillText(title, x + 6, y + 14);
  return { x, y, w, h };
}

function plotSeries(p, series, colors, logScale = false) {
  const all = series.flat().filter(Number.isFinite);
  if (!all.length) return;
  let lo = Math.min(...all), hi = Math.max(...all);
  if (logScale) { lo = Math.log10(lo + 1e-15); hi = Math.log10(hi + 1e-15); }
  if (hi === lo) hi = lo + 1;
  series.forEach((s, k) => {
    dctx.strokeStyle = colors[k];
    dctx.beginPath();
    s.forEach((v, i) => {
      const t = logScale ? Math.log10(v + 1e-15) : v;
      const px = p.x + 6 + (i / Math.max(1, s.length - 1)) * (p.w - 12);
      const py = p.y + p.h - 8 - ((t - lo) / (hi - lo)) * (p.h - 28);
      i ? dctx.lineTo(px, py) : dctx.moveTo(px, py);
    });
    dctx.stroke();
  });
}

panel draws a labeled box. plotSeries draws one or more lines inside it, rescaling to whatever range the data currently covers. The log option matters for gradients, which span many orders of magnitude and are unreadable on a linear scale.

The training loop that feeds it

const hist = { train: [], val: [], grad: [], wnorm: [], dead: [] };

function record(net, tr, va) {
  hist.train.push(netLoss(net, tr));
  hist.val.push(netLoss(net, va));
  const g = backwardAll(net, tr);
  hist.grad.push(norm(g.dW1.flat()));
  hist.wnorm.push(norm(net.W1.flat()));
  hist.dead.push(deadFraction(net, tr));
}

function draw(net, tr, va) {
  dctx.clearRect(0, 0, dash.width, dash.height);

  const p1 = panel(10, 10, 340, 190, "loss   blue train   red validation");
  plotSeries(p1, [hist.train, hist.val], ["#38a", "#c33"]);

  const p2 = panel(370, 10, 340, 190, "gradient size, log scale");
  plotSeries(p2, [hist.grad], ["#4a4"], true);

  const p3 = panel(10, 210, 340, 190, "weight size");
  plotSeries(p3, [hist.wnorm], ["#84c"]);

  const p4 = panel(370, 210, 340, 190, "dead unit fraction");
  plotSeries(p4, [hist.dead], ["#e80"]);

  const i = hist.train.length - 1;
  document.getElementById("stats").textContent =
    "epoch " + i +
    "   train " + hist.train[i].toFixed(4) +
    "   val " + hist.val[i].toFixed(4) +
    "   gap " + (hist.val[i] - hist.train[i]).toFixed(4) +
    "   |w| " + hist.wnorm[i].toFixed(2) +
    "   dead " + (100 * hist.dead[i]).toFixed(0) + "%";
}

function runTraining(net, tr, va, epochs = 400, lr = 0.5, batch = 8) {
  let e = 0;
  const tick = () => {
    for (let k = 0; k < 5 && e < epochs; k++, e++) {
      for (const b of batches(shuffled(tr), batch)) step(net, backwardAll(net, b), lr);
      record(net, tr, va);
    }
    draw(net, tr, va);
    if (e < epochs) requestAnimationFrame(tick);
  };
  tick();
}

requestAnimationFrame again, for the reason given in lesson 3075: a tight loop would freeze the page and show you only the last frame.

Four things to go and cause on purpose

The dashboard is only useful if you know what its failures look like. Produce each of these deliberately.

Overfitting. Sixteen hidden units, twelve rows. The two loss curves start together, then the red one flattens and turns up while the blue one keeps falling. The gap in the readout grows. That turn is where early stopping would fire.

Exploding gradients. Set the learning rate to 50. The green line climbs steeply and then the panel empties as the numbers become NaN.

Vanishing gradients. Six sigmoid layers. The green line sits near 1e-8 and both loss curves lie flat. Nothing is happening at all.

Dead units. relu at a learning rate of 5. The orange line jumps up in the first few epochs and never comes back down, and the loss stalls at whatever the surviving units can manage.

What to watch

Recording costs a full forward and backward pass over the training set, so measuring every epoch on a large dataset is a serious slowdown. Record every tenth epoch, or on a fixed subsample. Watch that this does not become the reason training is slow.

Exercises

  1. Add a fifth panel showing the decision boundary, using decisionMap from lesson 3075.
  2. Add early stopping: keep the weights from the best validation epoch and restore them at the end.
  3. Show gradient size per layer as separate lines rather than one total.
  4. Add a control for the learning rate that works mid-run, and try halving it when the loss stalls.
PreviousNext