Testing and Delivering Device Software

// Unit tests vs integration tests for one device endpoint: PATCH /api/v1/settings.
import express from "express";
import request from "supertest";

// ---- The code under test ----
const RULES = {
  name: { type: "string", maxLength: 32 },
  volume: { type: "integer", min: 0, max: 100 },
  brightness: { type: "integer", min: 0, max: 100 },
};
function validatePatch(patch) {
  const errors = [];
  for (const [key, value] of Object.entries(patch ?? {})) {
    const rule = RULES[key];
    if (!rule) errors.push(`unknown setting '${key}'`);
    else if (rule.type === "integer" && !Number.isInteger(value)) errors.push(`${key} must be an integer`);
    else if (rule.type === "integer" && (value < rule.min || value > rule.max)) errors.push(`${key} must be ${rule.min}–${rule.max}`);
    else if (rule.type === "string" && (typeof value !== "string" || value.length > rule.maxLength)) errors.push(`${key} must be a string of at most ${rule.maxLength}`);
  }
  return errors;
}
const applyPatch = (current, patch) => ({ ...current, ...patch });

function createApp({ fixed }) {
  const store = { settings: { name: "Huddle 4", volume: 50, brightness: 80 } };
  const app = express();
  // The bug: express.json() only parses Content-Type application/json by default.
  app.use(express.json(fixed ? { type: ["application/json", "application/merge-patch+json"] } : {}));
  app.get("/api/v1/settings", (req, res) => res.json(store.settings));
  app.patch("/api/v1/settings", (req, res) => {
    const errors = validatePatch(req.body);
    if (errors.length) return res.status(422).type("application/problem+json").json({ title: "Invalid settings", status: 422, errors });
    store.settings = applyPatch(store.settings, req.body);
    res.json(store.settings);
  });
  return app;
}

// ---- A tiny test runner (prints the same thing every run, unlike timing reporters) ----
async function run(title, tests) {
  console.log(title);
  let failed = 0;
  for (const [name, fn] of tests) {
    try { await fn(); console.log(`  ✓ ${name}`); }
    catch (err) { failed++; console.log(`  ✗ ${name}: ${err.message}`); }
  }
  console.log(`  ${tests.length - failed} passed, ${failed} failed`);
}
const expect = (actual, expected, what) => {
  if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`${what}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
};

const unit = [
  ["accepts a valid volume", () => expect(validatePatch({ volume: 40 }), [], "errors")],
  ["rejects volume 140", () => expect(validatePatch({ volume: 140 }), ["volume must be 0–100"], "errors")],
  ["rejects a misspelt key", () => expect(validatePatch({ volum: 40 }), ["unknown setting 'volum'"], "errors")],
  ["merges a patch", () => expect(applyPatch({ volume: 50, brightness: 80 }, { volume: 40 }), { volume: 40, brightness: 80 }, "settings")],
];

const integration = (app) => [
  ["GET returns JSON settings", async () => {
    const res = await request(app).get("/api/v1/settings");
    expect([res.status, res.type, res.body.volume], [200, "application/json", 50], "status, type, volume");
  }],
  ["PATCH volume 140 → 422 problem details", async () => {
    const res = await request(app).patch("/api/v1/settings").send({ volume: 140 });
    expect([res.status, res.type, res.body.errors], [422, "application/problem+json", ["volume must be 0–100"]], "status, type, errors");
  }],
  ["PATCH as merge-patch+json changes the volume", async () => {
    const res = await request(app).patch("/api/v1/settings").set("Content-Type", "application/merge-patch+json").send(JSON.stringify({ volume: 40 }));
    const after = await request(app).get("/api/v1/settings");
    expect([res.status, after.body.volume], [200, 40], "status, volume after");
  }],
];

await run("Unit tests (functions called directly)", unit);
await run("Integration tests (real Express app over HTTP, buggy)", integration(createApp({ fixed: false })));
await run("Integration tests (real Express app over HTTP, fixed)", integration(createApp({ fixed: true })));

Unit tests (functions called directly)
  ✓ accepts a valid volume
  ✓ rejects volume 140
  ✓ rejects a misspelt key
  ✓ merges a patch
  4 passed, 0 failed
Integration tests (real Express app over HTTP, buggy)
  ✓ GET returns JSON settings
  ✓ PATCH volume 140 → 422 problem details
  ✗ PATCH as merge-patch+json changes the volume: status, volume after: expected [200,40], got [200,50]
  2 passed, 1 failed
Integration tests (real Express app over HTTP, fixed)
  ✓ GET returns JSON settings
  ✓ PATCH volume 140 → 422 problem details
  ✓ PATCH as merge-patch+json changes the volume
  3 passed, 0 failed

// One simulated device, used by an API test (supertest) and a UI-logic test (real fetch over HTTP).
import express from "express";
import request from "supertest";
import { createServer } from "node:http";

function createFakeClock() {
  let now = 0;
  const timers = [];
  return {
    now: () => now,
    setTimeout: (fn, ms) => timers.push({ at: now + ms, fn }),
    advance(ms) {
      now += ms;
      for (const t of timers.filter((t) => t.at <= now).sort((a, b) => a.at - b.at)) { timers.splice(timers.indexOf(t), 1); t.fn(); }
    },
  };
}

// The simulator: same routes and status codes as the real device, a state machine, a fake clock, fault controls.
function createSimulatedDevice({ clock, bootMs = 45_000, firmware = "5.0.1" }) {
  let state = "ready", bootedAt = 0, nextOp = 1;
  const app = express();
  app.use((req, res, next) => (state === "down" ? req.socket.destroy() : next())); // unreachable while rebooting
  app.get("/api/v1/status", (req, res) => res.json({ state, firmware, uptimeS: (clock.now() - bootedAt) / 1000 }));
  app.post("/api/v1/reboot", (req, res) => {
    res.status(202).location(`/api/v1/operations/${nextOp++}`).json({ status: "accepted" });
    res.on("finish", () => sim.powerCycle()); // reply first, then go down, like a real device
  });
  const sim = {
    app,
    powerCycle() {
      state = "down";
      clock.setTimeout(() => { state = "ready"; bootedAt = clock.now(); }, bootMs);
    },
  };
  return sim;
}

const t = (clock) => `t=${String(clock.now() / 1000).padStart(2)}s`;

// ---- API test: reboot returns 202, the device drops off, then comes back with uptime reset ----
{
  const clock = createFakeClock();
  const device = createSimulatedDevice({ clock });
  const status = () => request(device.app).get("/api/v1/status").then((r) => `${r.status} ${JSON.stringify(r.body)}`, (e) => `error ${e.code}`);
  clock.advance(600_000);
  console.log(`API test  ${t(clock)}   GET status → ${await status()}`);
  const r = await request(device.app).post("/api/v1/reboot");
  console.log(`API test  ${t(clock)}   POST reboot → ${r.status}, Location ${r.headers.location}`);
  clock.advance(44_000);
  console.log(`API test  ${t(clock)}   GET status → ${await status()}`);
  clock.advance(1_000);
  console.log(`API test  ${t(clock)}   GET status → ${await status()}`);
}

// ---- UI test: the admin UI's connection banner, polling every 5 s, against the same simulator ----
function createConnectionBanner({ baseUrl }) {
  let failures = 0, rebootRequested = false, banner = "Online";
  return {
    get banner() { return banner; },
    async requestReboot() { await fetch(`${baseUrl}/api/v1/reboot`, { method: "POST" }); rebootRequested = true; },
    async poll() {
      try {
        const s = await (await fetch(`${baseUrl}/api/v1/status`)).json();
        banner = failures ? `Back online (firmware ${s.firmware})` : "Online";
        failures = 0; rebootRequested = false;
      } catch {
        failures++;
        banner = rebootRequested ? `Restarting… reconnecting (attempt ${failures})` : failures >= 2 ? "Device unreachable" : "Online";
      }
    },
  };
}
for (const scenario of ["user reboot", "power loss"]) {
  const clock = createFakeClock();
  const device = createSimulatedDevice({ clock, bootMs: 20_000 });
  const server = createServer(device.app);
  await new Promise((r) => server.listen(0, "127.0.0.1", r));
  const ui = createConnectionBanner({ baseUrl: `http://127.0.0.1:${server.address().port}` });
  if (scenario === "user reboot") await ui.requestReboot(); else device.powerCycle();
  const seen = [];
  for (let i = 0; i < 6; i++) {
    clock.advance(5_000);
    await ui.poll();
    seen.push(`${t(clock)} ${ui.banner}`);
  }
  console.log(`UI test (${scenario}):\n  ${seen.join("\n  ")}`);
  server.closeAllConnections();
  await new Promise((r) => server.close(r));
}

API test  t=600s   GET status → 200 {"state":"ready","firmware":"5.0.1","uptimeS":600}
API test  t=600s   POST reboot → 202, Location /api/v1/operations/1
API test  t=644s   GET status → error ECONNRESET
API test  t=645s   GET status → 200 {"state":"ready","firmware":"5.0.1","uptimeS":0}
UI test (user reboot):
  t= 5s Restarting… reconnecting (attempt 1)
  t=10s Restarting… reconnecting (attempt 2)
  t=15s Restarting… reconnecting (attempt 3)
  t=20s Back online (firmware 5.0.1)
  t=25s Online
  t=30s Online
UI test (power loss):
  t= 5s Online
  t=10s Device unreachable
  t=15s Device unreachable
  t=20s Back online (firmware 5.0.1)
  t=25s Online
  t=30s Online

// A hardware-in-the-loop style matrix: cut power after every step of a firmware update, then boot and see what runs.
// The device model follows the A/B scheme: two slots, a "bootable" flag, a "successful" flag and a boot-try counter.
const newDevice = (slotB) => ({
  active: "A",
  slots: { A: { image: "v1", bootable: true, successful: true, tries: 0 }, B: slotB ? { image: "v1", bootable: true, successful: true, tries: 0 } : { image: null, bootable: false } },
});
const other = (s) => (s === "A" ? "B" : "A");

// Three updaters. Each is a list of steps; the rig may cut power after any of them.
const updaters = {
  "in-place": [
    ["download", () => {}],
    ["write 50%", (d) => { d.slots[d.active].image = "partial"; }],
    ["write 100%", (d) => { d.slots[d.active].image = "v2"; }],
  ],
  "A/B": [
    ["download", () => {}],
    ["mark B unbootable", (d) => { d.slots.B.bootable = false; }],
    ["write B 50%", (d) => { d.slots.B.image = "partial"; }],
    ["write B 100%", (d) => { d.slots.B.image = "v2"; }],
    ["set B active", (d) => { Object.assign(d.slots.B, { bootable: true, successful: false, tries: 3 }); d.active = "B"; }],
  ],
  "A/B, successful too early": [
    ["download", () => {}],
    ["mark B unbootable", (d) => { d.slots.B.bootable = false; }],
    ["write B 50%", (d) => { d.slots.B.image = "partial"; }],
    ["write B 100%", (d) => { d.slots.B.image = "v2"; }],
    ["set B active", (d) => { Object.assign(d.slots.B, { bootable: true, successful: true, tries: 0 }); d.active = "B"; }],
  ],
};

// Bootloader + first boot: a slot that isn't marked successful uses up a try per boot; the health check
// (update_verifier's role on Android, plus our own app checks) marks it successful only if the new build works.
function boot(d, v2Works) {
  for (let attempt = 0; attempt < 6; attempt++) {
    const slot = d.slots[d.active];
    if (!slot.bootable) { d.active = other(d.active); continue; }
    if (!slot.successful) { if (slot.tries === 0) { slot.bootable = false; continue; } slot.tries--; }
    const healthy = slot.image === "v1" || (slot.image === "v2" && v2Works);
    if (healthy) { slot.successful = true; return `runs ${slot.image}`; }
    // partial image or broken build: the device crashes and reboots
  }
  return d.slots.A.image === "partial" || d.slots.B.image === "partial" ? "BRICKED" : "BOOT LOOP";
}

for (const [name, steps] of Object.entries(updaters)) {
  console.log(`${name.padEnd(30)}v2 good    v2 bad`);
  for (let cut = 1; cut <= steps.length; cut++) {
    const results = [true, false].map((v2Works) => {
      const d = newDevice(name !== "in-place"); // an in-place device has a single system slot
      steps.slice(0, cut).forEach(([, step]) => step(d)); // run the update up to the cut, then power off
      return boot(d, v2Works).padEnd(10);
    });
    const when = cut === steps.length ? "no cut (update completes)" : `cut after ${steps[cut - 1][0]}`;
    console.log(`  ${when.padEnd(28)}${results.join(" ")}`.trimEnd());
  }
}

in-place                      v2 good    v2 bad
  cut after download          runs v1    runs v1
  cut after write 50%         BRICKED    BRICKED
  no cut (update completes)   runs v2    BOOT LOOP
A/B                           v2 good    v2 bad
  cut after download          runs v1    runs v1
  cut after mark B unbootable runs v1    runs v1
  cut after write B 50%       runs v1    runs v1
  cut after write B 100%      runs v1    runs v1
  no cut (update completes)   runs v2    runs v1
A/B, successful too early     v2 good    v2 bad
  cut after download          runs v1    runs v1
  cut after mark B unbootable runs v1    runs v1
  cut after write B 50%       runs v1    runs v1
  cut after write B 100%      runs v1    runs v1
  no cut (update completes)   runs v2    BOOT LOOP

// A staged-rollout planner for a device fleet: stable buckets, soak per stage, a health gate against a control group.
import { createHash } from "node:crypto";

function mulberry32(seed) { // seeded PRNG: same fleet and same failures every run
  return () => {
    seed = (seed + 0x6d2b79f5) | 0;
    let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}
// Stable bucket 0–9999 per (rollout, device): a device in the 5% stage stays in for 20%, 50% and 100%.
const bucket = (rolloutId, serial) => createHash("sha256").update(`${rolloutId}:${serial}`).digest().readUInt32BE(0) % 10_000;

const rand = mulberry32(2026);
const MODELS = ["room-bar", "codec", "touch-panel"]; // 60%, 30% and 10% of the fleet
const fleet = Array.from({ length: 40_000 }, (_, i) => {
  const r = rand();
  return { serial: `SN${String(i).padStart(6, "0")}`, model: MODELS[r < 0.6 ? 0 : r < 0.9 ? 1 : 2] };
});

const STAGES = [{ pct: 1, soakH: 24 }, { pct: 5, soakH: 48 }, { pct: 20, soakH: 48 }, { pct: 50, soakH: 72 }, { pct: 100 }];
const GATE = { minFailures: 5, minExcessPct: 0.5, minZ: 3, minSegment: 50 };
const BASE_FAILURE = 0.004; // chance a device on the old build has a failure (crash, failed call, reboot) in a soak

// Health gate: is the cohort's failure rate clearly worse than the control group's?
function judge(cohort, control) {
  const p1 = cohort.failed / cohort.n, p0 = control.failed / control.n;
  const z = (p1 - p0) / Math.sqrt((p0 * (1 - p0)) / cohort.n);
  const bad = cohort.failed >= GATE.minFailures && (p1 - p0) * 100 >= GATE.minExcessPct && z >= GATE.minZ;
  return { bad, text: `${cohort.failed}/${cohort.n} = ${(100 * p1).toFixed(2)}% vs control ${(100 * p0).toFixed(2)}%` };
}

function rollOut(version, failureChance) {
  console.log(`Release ${version}`);
  let hours = 0;
  for (const [i, stage] of STAGES.entries()) {
    const inStage = (d) => bucket(version, d.serial) < stage.pct * 100;
    const cohort = fleet.filter(inStage);
    if (!stage.soakH) { console.log(`  stage ${i + 1} (${stage.pct}%): ${cohort.length} devices, rollout complete after ${hours / 24} days`); return; }
    hours += stage.soakH;
    // Soak: about 92% of updated devices report back in time; every device may fail with its build's chance.
    const report = (d, chance) => ({ ...d, reported: rand() < 0.92, failed: rand() < chance });
    const updated = cohort.map((d) => report(d, failureChance(d.model))).filter((d) => d.reported);
    const control = fleet.filter((d) => !inStage(d)).map((d) => report(d, BASE_FAILURE)).filter((d) => d.reported);
    const count = (list) => ({ n: list.length, failed: list.filter((d) => d.failed).length });
    const overall = judge(count(updated), count(control));
    const segments = MODELS.map((m) => [m, count(updated.filter((d) => d.model === m)), count(control.filter((d) => d.model === m))]);
    console.log(`  stage ${i + 1} (${stage.pct}%): ${cohort.length} updated, ${updated.length} reported`);
    console.log(`      all models   ${overall.text} ${overall.bad ? "✗" : "✓"}`);
    let failing = 0;
    for (const [m, c, k] of segments) {
      const j = c.n >= GATE.minSegment ? judge(c, k) : null; // too few devices to judge this model yet
      if (j?.bad) failing++;
      if (!j || j.bad) console.log(`      ${m.padEnd(12)} ${j ? j.text + " ✗" : `${c.n} reported, too few to judge`}`);
    }
    if (overall.bad || failing) {
      console.log(`  HALT on day ${hours / 24}: ${cohort.length} devices have ${version}, ${fleet.length - cohort.length} never got it`);
      return;
    }
  }
}

rollOut("5.0.2", () => BASE_FAILURE); // healthy release
rollOut("5.0.3", (model) => (model === "touch-panel" ? 0.04 : BASE_FAILURE)); // a bug on one model only

Release 5.0.2
  stage 1 (1%): 381 updated, 350 reported
      all models   0/350 = 0.00% vs control 0.36% ✓
      touch-panel  29 reported, too few to judge
  stage 2 (5%): 2008 updated, 1832 reported
      all models   7/1832 = 0.38% vs control 0.39% ✓
  stage 3 (20%): 8032 updated, 7410 reported
      all models   23/7410 = 0.31% vs control 0.43% ✓
  stage 4 (50%): 20037 updated, 18404 reported
      all models   78/18404 = 0.42% vs control 0.40% ✓
  stage 5 (100%): 40000 devices, rollout complete after 8 days
Release 5.0.3
  stage 1 (1%): 377 updated, 342 reported
      all models   3/342 = 0.88% vs control 0.31% ✓
      touch-panel  31 reported, too few to judge
  stage 2 (5%): 2079 updated, 1905 reported
      all models   16/1905 = 0.84% vs control 0.40% ✓
      touch-panel  9/192 = 4.69% vs control 0.52% ✗
  HALT on day 3: 2079 devices have 5.0.3, 37921 never got it

// Feature-flag evaluation on a device: local, never throws, always returns a value and a reason.
// Reasons and error codes follow the OpenFeature names (TARGETING_MATCH, SPLIT, DISABLED, ERROR / FLAG_NOT_FOUND…).
import { createHash } from "node:crypto";

// The flag config the device last downloaded (signed and versioned in real life), cached on disk.
const config = {
  version: 42,
  flags: {
    "ui.new-settings-page": { type: "boolean", enabled: true, default: false,
      rules: [{ when: { tenant: ["acme"] }, value: true }], rollout: { percent: 25, value: true } },
    "audio.noise-suppression": { type: "string", enabled: true, default: "standard",
      rules: [{ when: { model: ["room-bar"], minFirmware: "5.0.3" }, value: "aggressive" }] },
    "telemetry.verbose-logs": { type: "boolean", enabled: false, default: false }, // kill switch: off for everyone
  },
};

const firmwareAtLeast = (have, need) => {
  const [a, b] = [have, need].map((v) => v.split(".").map(Number));
  for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] > b[i];
  return true;
};
const matches = (when, ctx) =>
  Object.entries(when).every(([k, v]) => (k === "minFirmware" ? firmwareAtLeast(ctx.firmware, v) : v.includes(ctx[k])));
// Same device + same flag → same bucket, on every boot, with no server call.
const bucket = (flagKey, targetingKey) => createHash("sha256").update(`${flagKey}:${targetingKey}`).digest().readUInt32BE(0) % 100;

function evaluate(flagKey, defaultValue, ctx) {
  try {
    const flag = config.flags[flagKey];
    if (!flag) return { value: defaultValue, reason: "ERROR", errorCode: "FLAG_NOT_FOUND" };
    if (flag.type !== typeof defaultValue) return { value: defaultValue, reason: "ERROR", errorCode: "TYPE_MISMATCH" };
    if (!flag.enabled) return { value: flag.default, reason: "DISABLED" };
    for (const rule of flag.rules ?? []) if (matches(rule.when, ctx)) return { value: rule.value, reason: "TARGETING_MATCH" };
    if (flag.rollout) {
      if (!ctx.targetingKey) return { value: defaultValue, reason: "ERROR", errorCode: "TARGETING_KEY_MISSING" };
      if (bucket(flagKey, ctx.targetingKey) < flag.rollout.percent) return { value: flag.rollout.value, reason: "SPLIT" };
    }
    return { value: flag.default, reason: "DEFAULT" };
  } catch {
    return { value: defaultValue, reason: "ERROR", errorCode: "GENERAL" }; // a flag bug must never crash the device
  }
}

const devices = [
  { targetingKey: "SN000017", model: "room-bar", firmware: "5.0.3", tenant: "acme" },
  { targetingKey: "SN000018", model: "room-bar", firmware: "5.0.2", tenant: "globex" },
  { targetingKey: "SN000019", model: "codec", firmware: "5.0.3", tenant: "globex" },
  { model: "touch-panel", firmware: "5.0.3", tenant: "globex" }, // forgot to pass its serial
];
const show = (r) => `${JSON.stringify(r.value)} (${r.reason}${r.errorCode ? " " + r.errorCode : ""})`;
for (const d of devices) {
  console.log(`${d.targetingKey ?? "(no key)"} ${d.model} ${d.firmware} ${d.tenant}`);
  console.log(`  ui.new-settings-page     ${show(evaluate("ui.new-settings-page", false, d))}`);
  console.log(`  audio.noise-suppression  ${show(evaluate("audio.noise-suppression", "standard", d))}`);
}
const d0 = devices[1];
console.log(`telemetry.verbose-logs   ${show(evaluate("telemetry.verbose-logs", false, d0))}`);
console.log(`ui.dark-mode (typo'd)    ${show(evaluate("ui.dark-mode", false, d0))}`);
console.log(`audio.noise-suppression asked as boolean: ${show(evaluate("audio.noise-suppression", true, d0))}`);

// Raising the rollout from 10% to 25% only adds devices: nobody who had the feature loses it.
const serials = Array.from({ length: 10_000 }, (_, i) => `SN${String(i).padStart(6, "0")}`);
const inAt = (pct) => new Set(serials.filter((s) => bucket("ui.new-settings-page", s) < pct));
const [ten, quarter] = [inAt(10), inAt(25)];
console.log(`10%: ${ten.size} devices, 25%: ${quarter.size} devices, in 10% but not 25%: ${[...ten].filter((s) => !quarter.has(s)).length}`);

SN000017 room-bar 5.0.3 acme
  ui.new-settings-page     true (TARGETING_MATCH)
  audio.noise-suppression  "aggressive" (TARGETING_MATCH)
SN000018 room-bar 5.0.2 globex
  ui.new-settings-page     false (DEFAULT)
  audio.noise-suppression  "standard" (DEFAULT)
SN000019 codec 5.0.3 globex
  ui.new-settings-page     true (SPLIT)
  audio.noise-suppression  "standard" (DEFAULT)
(no key) touch-panel 5.0.3 globex
  ui.new-settings-page     false (ERROR TARGETING_KEY_MISSING)
  audio.noise-suppression  "standard" (DEFAULT)
telemetry.verbose-logs   false (DISABLED)
ui.dark-mode (typo'd)    false (ERROR FLAG_NOT_FOUND)
audio.noise-suppression asked as boolean: true (ERROR TYPE_MISMATCH)
10%: 1059 devices, 25%: 2625 devices, in 10% but not 25%: 0