Backprop Through Depth

type Vector = number[];
type Matrix = number[][]; // rows = neurons, cols = inputs

function relu(x: number): number {
  return Math.max(0, x);
}

function reluDeriv(x: number): number {
  return x > 0 ? 1 : 0;
}

/** Square footage is thousands; we scale inputs so training does not explode. */
function scale(features: Vector): Vector {
  return [features[0] / 1000, features[1], features[2] / 10];
}

/** Mulberry32 — fixed seed so this lesson’s numbers are reproducible. */
function mulberry32(seed: number): () => number {
  return () => {
    seed |= 0;
    seed = (seed + 0x6d2b79f5) | 0;
    let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

class Layer {
  weights: Matrix;
  biases: Vector;
  activation: "relu" | "linear";

  constructor(
    inputSize: number,
    outputSize: number,
    activation: "relu" | "linear",
    rnd: () => number,
  ) {
    this.activation = activation;
    this.weights = Array.from({ length: outputSize }, () =>
      Array.from({ length: inputSize }, () => rnd() * 0.5 - 0.25),
    );
    this.biases = Array(outputSize).fill(0);
  }

  forward(inputs: Vector): { outputs: Vector; preActivations: Vector } {
    const preActivations: Vector = [];
    const outputs: Vector = [];
    for (let j = 0; j < this.weights.length; j++) {
      let sum = this.biases[j];
      for (let i = 0; i < inputs.length; i++) {
        sum += inputs[i] * this.weights[j][i];
      }
      preActivations.push(sum);
      outputs.push(this.activation === "relu" ? relu(sum) : sum);
    }
    return { outputs, preActivations };
  }
}

/** DeepHouseNet: 3 → 4 ReLU → 4 ReLU → 1 linear. */
class DeepHouseNet {
  h1: Layer;
  h2: Layer;
  out: Layer;

  constructor(seed = 37) {
    const rnd = mulberry32(seed);
    this.h1 = new Layer(3, 4, "relu", rnd);
    this.h2 = new Layer(4, 4, "relu", rnd);
    this.out = new Layer(4, 1, "linear", rnd);
  }

  forward(features: Vector): number {
    const x = scale(features);
    const a = this.h1.forward(x);
    const b = this.h2.forward(a.outputs);
    const o = this.out.forward(b.outputs);
    return o.outputs[0];
  }
}

/** One house, one downhill step through every layer. */
function trainOne(
  net: DeepHouseNet,
  features: Vector,
  target: number,
  lr: number,
): void {
  const x = scale(features); // Square footage is thousands; we scale inputs so training does not explode.
  // Forward — keep pre-activations for ReLU′
  const h1 = net.h1.forward(x);
  const h2 = net.h2.forward(h1.outputs);
  const o = net.out.forward(h2.outputs);
  const pred = o.outputs[0];
  const error = target - pred;

  // 1. Update output (linear)
  for (let j = 0; j < net.out.weights.length; j++) {
    for (let i = 0; i < h2.outputs.length; i++) {
      net.out.weights[j][i] += error * h2.outputs[i] * lr;
    }
    net.out.biases[j] += error * lr;
  }

  // 2. Chain blame into h2 × ReLU′
  const h2Error: Vector = Array(h2.outputs.length).fill(0);
  for (let i = 0; i < h2.outputs.length; i++) {
    let sum = 0;
    for (let j = 0; j < net.out.weights.length; j++) {
      sum += net.out.weights[j][i] * error;
    }
    h2Error[i] = sum * reluDeriv(h2.preActivations[i]);
  }

  for (let j = 0; j < net.h2.weights.length; j++) {
    for (let i = 0; i < h1.outputs.length; i++) {
      net.h2.weights[j][i] += h2Error[j] * h1.outputs[i] * lr;
    }
    net.h2.biases[j] += h2Error[j] * lr;
  }

  // 3. Chain blame into h1 × ReLU′
  const h1Error: Vector = Array(h1.outputs.length).fill(0);
  for (let i = 0; i < h1.outputs.length; i++) {
    let sum = 0;
    for (let j = 0; j < net.h2.weights.length; j++) {
      sum += net.h2.weights[j][i] * h2Error[j];
    }
    h1Error[i] = sum * reluDeriv(h1.preActivations[i]);
  }

  for (let j = 0; j < net.h1.weights.length; j++) {
    for (let i = 0; i < x.length; i++) {
      net.h1.weights[j][i] += h1Error[j] * x[i] * lr;
    }
    net.h1.biases[j] += h1Error[j] * lr;
  }
}

const net = new DeepHouseNet();
const features: Vector = [1500, 3, 10];
const target = 295_000;
const lr = 1e-9;

console.log("before", net.forward(features).toFixed(4)); // ≈ 0.0060
trainOne(net, features, target, lr);
console.log("after ", net.forward(features).toFixed(4)); // ≈ 0.0064
// One step: tiny nudge — expected. Off ReLU neurons get ReLU′ = 0 — no blame.