test/check.test.mjs
Commit 01f96e7. All files · Download the tarball.
import { test } from "node:test";
import assert from "node:assert/strict";
import { runCheck, handleFetch } from "../src/index.js";
import { fakeDb, fakeFetch, keysResponse, minuteOf } from "./helpers.mjs";
import { usageFrom, steady } from "./world.mjs";
const HOOK = "https://hooks.slack.com/services/T0/B0/XXXX";
const T0 = Date.parse("2026-11-16T15:00:00Z"); // Monday 10:00 America/New_York
const MIN = 60000;
const envFor = (db, extra = {}) => ({ ANTHROPIC_ADMIN_KEY: "sk-ant-admin01-TEST", ALERT_WEBHOOK_URL: HOOK, DB: db, ...extra });
const webhookBodies = (f) => f.calls.filter((c) => c.url.startsWith("https://hooks.slack.com/")).map((c) => JSON.parse(c.init.body).text);
const hosts = (f) => new Set(f.calls.map((c) => new URL(c.url).hostname));
test("first run seeds the baseline from hourly buckets and sends one 'started' message", async () => {
const db = fakeDb();
const f = fakeFetch({ usage: usageFrom(steady()), keys: () => keysResponse([{ id: "apikey_A", name: "prod" }]) });
const r = await runCheck(envFor(db), { now: T0, fetchImpl: f });
assert.equal(r.ok, true);
assert.equal(r.firstRun, true);
const usageCalls = f.calls.filter((c) => c.url.includes("usage_report"));
assert.match(usageCalls[0].url, /bucket_width=1h/);
assert.match(usageCalls[1].url, /bucket_width=1m/);
const sent = webhookBodies(f);
assert.equal(sent.length, 1);
assert.match(sent[0], /now watching/);
assert.deepEqual([...hosts(f)].sort(), ["api.anthropic.com", "hooks.slack.com"]);
const s = JSON.parse(db.row.v);
assert.equal(s.initialized, true);
assert.ok(s.seenKeys.apikey_A && s.seenModels["claude-sonnet-5-5"]);
});
test("steady usage: no alerts, nothing sent", async () => {
const db = fakeDb();
const f = fakeFetch({ usage: usageFrom(steady()) });
await runCheck(envFor(db), { now: T0, fetchImpl: f });
for (let i = 1; i <= 6; i++) {
const r = await runCheck(envFor(db), { now: T0 + i * 5 * MIN, fetchImpl: f });
assert.equal(r.alerts, 0, `run ${i}`);
}
assert.equal(webhookBodies(f).length, 1, "only the started message");
});
test("a per-key spike above N x baseline and the floor alerts once, then cools down", async () => {
const db = fakeDb();
const spikeAt = minuteOf("2026-11-16T15:22:00Z");
const world = (m) => [{ key: "apikey_A", in: 800, out: 200 }, ...(m >= spikeAt ? [{ key: "apikey_A", model: "claude-opus-5-5", in: 400000, out: 100000 }] : [])];
const f = fakeFetch({ usage: usageFrom(world), keys: () => keysResponse([{ id: "apikey_A", name: "prod-backend" }]) });
await runCheck(envFor(db, { ALERT_NEW_MODELS: "false" }), { now: T0, fetchImpl: f });
const r = await runCheck(envFor(db, { ALERT_NEW_MODELS: "false" }), { now: T0 + 25 * MIN, fetchImpl: f });
assert.equal(r.alerts, 1);
const msg = webhookBodies(f).at(-1);
assert.match(msg, /Spike: key "prod-backend"/);
assert.match(msg, /501,000 tokens/);
assert.match(msg, /Estimated cost of that minute: about \$3\.60/); // Opus 5.5: 400k in x $4/M + 100k out x $20/M = $3.60, plus ~$0.004 of steady Sonnet use
assert.match(msg, /estimates/);
// Same spike five minutes later: inside the 60-minute cooldown.
const r2 = await runCheck(envFor(db, { ALERT_NEW_MODELS: "false" }), { now: T0 + 30 * MIN, fetchImpl: f });
assert.equal(r2.alerts, 0);
// After the cooldown it fires again if the key is still spiking against its (now partly raised) baseline.
const r3 = await runCheck(envFor(db, { ALERT_NEW_MODELS: "false", SPIKE_MULTIPLIER: "1.5" }), { now: T0 + 90 * MIN, fetchImpl: f });
assert.ok(r3.alerts >= 1);
});
test("spikes below the absolute floor are ignored", async () => {
const db = fakeDb();
const world = (m) => [{ key: "apikey_A", in: 10, out: 0 }, ...(m === minuteOf("2026-11-16T15:22:00Z") ? [{ key: "apikey_A", in: 5000 }] : [])];
const f = fakeFetch({ usage: usageFrom(world) });
await runCheck(envFor(db), { now: T0, fetchImpl: f });
const r = await runCheck(envFor(db), { now: T0 + 25 * MIN, fetchImpl: f });
assert.equal(r.alerts, 0, "5,010 tokens/min is 500x baseline but under the 50,000 floor");
});
test("new key created, new key in use, and new model alert once each", async () => {
const db = fakeDb();
const startNew = minuteOf("2026-11-16T15:21:00Z");
const world = (m) => [{ key: "apikey_A", in: 800, out: 200 }, ...(m >= startNew ? [{ key: "apikey_X", model: "claude-fable-5-2", in: 3000, out: 500 }] : [])];
let list = [{ id: "apikey_A", name: "prod" }];
const f = fakeFetch({ usage: usageFrom(world), keys: () => keysResponse(list) });
await runCheck(envFor(db), { now: T0, fetchImpl: f });
list = [...list, { id: "apikey_N", name: "<!channel> new", created_at: "2026-11-16T15:10:00Z" }];
const r = await runCheck(envFor(db), { now: T0 + 25 * MIN, fetchImpl: f });
assert.equal(r.alerts, 3);
const msg = webhookBodies(f).at(-1);
assert.match(msg, /New API key: key "<!channel> new"/, "Slack control characters escaped");
assert.match(msg, /New key in use: key apikey_X/);
assert.match(msg, /New model: claude-fable-5-2/);
const r2 = await runCheck(envFor(db), { now: T0 + 30 * MIN, fetchImpl: f });
assert.equal(r2.alerts, 0, "seen keys and models don't alert again");
});
test("off-hours use alerts in the configured timezone", async () => {
const db = fakeDb();
const env = envFor(db, { TIMEZONE: "America/New_York", WORK_HOURS: "08:00-19:00", WORK_DAYS: "1-5" });
const sat = Date.parse("2026-11-21T08:00:00Z"); // Saturday 03:00 ET
const f = fakeFetch({ usage: usageFrom(steady()) });
await runCheck(env, { now: sat, fetchImpl: f });
const r = await runCheck(env, { now: sat + 5 * MIN, fetchImpl: f });
assert.equal(r.alerts, 1);
assert.match(webhookBodies(f).at(-1), /Off-hours use: key apikey_A .* starting Sat 0[23]:\d\d local time/);
const r2 = await runCheck(env, { now: sat + 10 * MIN, fetchImpl: f });
assert.equal(r2.alerts, 0, "cooldown");
// Weekday working hours: no off-hours alert.
const db2 = fakeDb();
const f2 = fakeFetch({ usage: usageFrom(steady()) });
const env2 = { ...env, DB: db2 };
await runCheck(env2, { now: T0, fetchImpl: f2 });
assert.equal((await runCheck(env2, { now: T0 + 5 * MIN, fetchImpl: f2 })).alerts, 0);
});
test("webhook failure: state is not saved, so the alert is retried next run", async () => {
const db = fakeDb();
const spikeAt = minuteOf("2026-11-16T15:22:00Z");
const world = (m) => [{ key: "apikey_A", in: 800, out: 200 }, ...(m >= spikeAt ? [{ key: "apikey_A", in: 900000 }] : [])];
const good = fakeFetch({ usage: usageFrom(world) });
await runCheck(envFor(db), { now: T0, fetchImpl: good });
const saved = db.row.v;
const bad = fakeFetch({ usage: usageFrom(world), webhookStatus: 500 });
const r = await runCheck(envFor(db), { now: T0 + 25 * MIN, fetchImpl: bad });
assert.equal(r.ok, false);
assert.equal(db.row.v, saved);
const r2 = await runCheck(envFor(db), { now: T0 + 26 * MIN, fetchImpl: good });
assert.equal(r2.alerts, 1);
});
test("Admin API errors alert at most once per 6 hours", async () => {
const db = fakeDb();
const f = fakeFetch({ usage: () => ({}), usageStatus: 401 });
const r = await runCheck(envFor(db), { now: T0, fetchImpl: f });
assert.equal(r.ok, false);
assert.equal(webhookBodies(f).length, 1);
assert.match(webhookBodies(f)[0], /could not read your usage: Anthropic Admin API returned HTTP 401/);
await runCheck(envFor(db), { now: T0 + 5 * MIN, fetchImpl: f });
await runCheck(envFor(db), { now: T0 + 60 * MIN, fetchImpl: f });
assert.equal(webhookBodies(f).length, 1);
await runCheck(envFor(db), { now: T0 + 361 * MIN, fetchImpl: f });
assert.equal(webhookBodies(f).length, 2);
});
test("missing or non-Slack/Discord webhook: nothing is fetched", async () => {
const f = fakeFetch({ usage: usageFrom(steady()) });
const r = await runCheck(envFor(fakeDb(), { ALERT_WEBHOOK_URL: "https://collector.example/hook" }), { now: T0, fetchImpl: f });
assert.equal(r.ok, false);
assert.equal(f.calls.length, 0);
});
test("endpoints: 404 without TEST_SECRET, 401 with a wrong one", async () => {
const f = fakeFetch({ usage: usageFrom(steady()) });
const db = fakeDb();
const post = (path, auth) => new Request(`https://tripwire.example.workers.dev${path}`, { method: "POST", headers: auth ? { authorization: auth } : {} });
assert.equal((await handleFetch(post("/test-alert", "Bearer x"), envFor(db), { fetchImpl: f })).status, 404);
const env = envFor(db, { TEST_SECRET: "s3cret-value" });
assert.equal((await handleFetch(post("/test-alert"), env, { fetchImpl: f })).status, 401);
assert.equal((await handleFetch(post("/dry-run", "Bearer nope"), env, { fetchImpl: f })).status, 401);
assert.equal((await handleFetch(new Request("https://t.example/dry-run"), env, { fetchImpl: f })).status, 405);
assert.equal((await handleFetch(new Request("https://t.example/anything"), env, { fetchImpl: f })).status, 404);
assert.equal(f.calls.length, 0);
});
test("test-alert posts one test message; dry-run sends nothing and saves nothing", async () => {
const db = fakeDb();
const env = envFor(db, { TEST_SECRET: "s3cret-value" });
const f = fakeFetch({ usage: usageFrom(steady()) });
const post = (path) => new Request(`https://t.example${path}`, { method: "POST", headers: { authorization: "Bearer s3cret-value" } });
const t = await handleFetch(post("/test-alert"), env, { fetchImpl: f });
assert.equal(t.status, 200);
assert.deepEqual(await t.json(), { ok: true, sent_to: "slack" });
assert.match(webhookBodies(f)[0], /Test alert/);
const d = await handleFetch(post("/dry-run"), env, { fetchImpl: f, now: T0 });
const body = await d.json();
assert.equal(body.ok, true);
assert.equal(body.dryRun, true);
assert.match(body.message, /now watching/);
assert.equal(webhookBodies(f).length, 1, "dry-run did not post");
assert.equal(db.writes, 0, "dry-run did not save state");
});
test("Discord webhook gets content with mentions disabled", async () => {
const db = fakeDb();
const f = fakeFetch({ usage: usageFrom(steady()) });
await runCheck(envFor(db, { ALERT_WEBHOOK_URL: "https://discord.com/api/webhooks/1/abc" }), { now: T0, fetchImpl: f });
const call = f.calls.find((c) => c.url.startsWith("https://discord.com/"));
const body = JSON.parse(call.init.body);
assert.match(body.content, /now watching/);
assert.deepEqual(body.allowed_mentions, { parse: [] });
});
test("every Anthropic and webhook fetch uses redirect: manual; a 302 from usage fails closed", async () => {
const db = fakeDb();
const f = fakeFetch({ usage: usageFrom(steady()), keys: () => keysResponse([{ id: "apikey_A", name: "prod" }]) });
await runCheck(envFor(db), { now: T0, fetchImpl: f });
assert.ok(f.calls.some((c) => c.url.includes("api.anthropic.com")));
assert.ok(f.calls.some((c) => c.url.startsWith("https://hooks.slack.com/")));
for (const c of f.calls) assert.equal(c.init.redirect, "manual");
const calls = [];
const f302 = async (url, init = {}) => {
calls.push({ url: String(url), init });
return new Response(null, { status: 302, headers: { location: "https://evil.example/" } });
};
const r = await runCheck(envFor(fakeDb()), { now: T0, fetchImpl: f302, dryRun: true });
assert.equal(r.ok, false);
assert.equal(calls.length, 1);
assert.equal(calls[0].init.redirect, "manual");
assert.match(r.error, /returned HTTP 302/);
});