Skip to content
SlashClone
DiscoverUse in ClaudeSubmit a link
Sign in
Made by@agentcosts

Consumption guardrails

@agentcosts/consumption-guardrails

Set and check consumption guardrails for an AI product: per-customer daily and monthly spend caps, month-end forecasts, spend spike rules, runaway agent runs, and failure rate spikes, plus a runbook for what to do when each alert fires. Runs on usage files; exit code 2 on critical alerts for cron or CI.

SkillDevOpsv1.0.0 (latest)Tested
Use in Claude Save Like Clone Comment
Activity
New: be the first to try it
  1. 1#!/usr/bin/env node
  2. 2// Consumption guardrails: caps, forecasts, anomaly rules, and runaway runs on LLM usage.
  3. 3// Usage: node scripts/guardrails.mjs <usage.csv> --config guardrails.json [--prices prices.csv] [--as-of 2026-10-27]
  4. 4// [--fail-on-alert] [--out out/guardrails]
  5. 5// Exit code 0 normally; 2 when --fail-on-alert is set and there is at least one critical alert (for cron or CI).
  6. 6import { DAY_MS, dayKey, fail, mdTable, median, money, monthKey, parseArgs, parseDate, pct, readJson, round, writeOut } from "./lib.mjs";
  7. 7import { loadPrices, readUsage, toRuns, warnStalePrices } from "./usage.mjs";
  8. 8
  9. 9const args = parseArgs(process.argv.slice(2), ["fail-on-alert"]);
  10. 10const input = args._[0];
  11. 11if (!input || !args.config) fail("usage: node scripts/guardrails.mjs <usage.csv> --config guardrails.json [--as-of 2026-10-27] [--fail-on-alert]");
  12. 12const outPrefix = args.out || "out/guardrails";
  13. 13const cfg = readJson(args.config, "guardrails config");
  14. 14const prices = args.prices ? loadPrices(args.prices) : null;
  15. 15if (prices) warnStalePrices(prices, args.prices);
  16. 16const { rows } = readUsage(input, prices);
  17. 17
  18. 18const D = cfg.defaults || {};
  19. 19const limits = (c) => ({ ...D, ...(cfg.customers?.[c] || {}) });
  20. 20const warnAt = cfg.warn_at ?? 0.8;
  21. 21const anomaly = { multiplier: 3, baseline_days: 7, min_usd: 1, ...(cfg.anomaly || {}) };
  22. 22const failure = { max: 0.4, min_runs: 5, ...(cfg.failure_rate || {}) };
  23. 23
  24. 24const lastDay = dayKey(rows.reduce((a, r) => (r.t > a.t ? r : a)).date);
  25. 25const asOf = args["as-of"] ? dayKey(parseDate(args["as-of"]) || fail("--as-of must be a date.")) : lastDay;
  26. 26const asOfT = Date.parse(asOf);
  27. 27const inScope = rows.filter((r) => r.t < asOfT + DAY_MS);
  28. 28const month = asOf.slice(0, 7);
  29. 29const dayOfMonth = Number(asOf.slice(8, 10));
  30. 30const [y, m] = month.split("-").map(Number);
  31. 31const daysInMonth = new Date(Date.UTC(y, m, 0)).getUTCDate();
  32. 32
  33. 33const daily = new Map(); // customer -> day -> cost
  34. 34for (const r of inScope) {
  35. 35 if (!daily.has(r.customer)) daily.set(r.customer, new Map());
  36. 36 const d = daily.get(r.customer);
  37. 37 const k = dayKey(r.date);
  38. 38 d.set(k, (d.get(k) || 0) + r.cost);
  39. 39}
  40. 40
  41. 41const alerts = [];
  42. 42const add = (severity, customer, rule, detail, value, limit) => alerts.push({ severity, customer, rule, detail, value: round(value, 2), limit: limit === null || limit === undefined ? null : round(limit, 2) });
  43. 43const customerRows = [];
  44. 44
  45. 45for (const [c, days] of daily) {
  46. 46 const L = limits(c);
  47. 47 const today = days.get(asOf) || 0;
  48. 48 let mtd = 0;
  49. 49 for (const [k, v] of days) if (k.startsWith(month) && k <= asOf) mtd += v;
  50. 50 const forecast = (mtd / dayOfMonth) * daysInMonth;
  51. 51
  52. 52 if (L.daily_cap_usd && today > L.daily_cap_usd) add(today > 2 * L.daily_cap_usd ? "critical" : "warning", c, "daily cap", `Spent ${money(today, 2)} on ${asOf}, cap ${money(L.daily_cap_usd, 2)}.`, today, L.daily_cap_usd);
  53. 53 if (L.monthly_cap_usd) {
  54. 54 if (mtd > L.monthly_cap_usd) add("critical", c, "monthly cap", `Month to date ${money(mtd, 2)} is over the ${money(L.monthly_cap_usd, 2)} cap.`, mtd, L.monthly_cap_usd);
  55. 55 else if (forecast > L.monthly_cap_usd) add("warning", c, "monthly forecast", `On pace for ${money(forecast, 2)} this month, over the ${money(L.monthly_cap_usd, 2)} cap.`, forecast, L.monthly_cap_usd);
  56. 56 else if (mtd > warnAt * L.monthly_cap_usd) add("info", c, "monthly cap", `Month to date ${money(mtd, 2)} is past ${pct(warnAt, 0)} of the cap.`, mtd, L.monthly_cap_usd);
  57. 57 }
  58. 58 // Anomaly: today vs median of the prior baseline days (days with no spend count as zero).
  59. 59 const base = [];
  60. 60 for (let i = 1; i <= anomaly.baseline_days; i++) base.push(days.get(dayKey(new Date(asOfT - i * DAY_MS))) || 0);
  61. 61 const med = median(base);
  62. 62 // Only customers with a full baseline window of history; new customers have no "normal" yet.
  63. 63 const firstDay = [...days.keys()].sort()[0];
  64. 64 const hasBaseline = Date.parse(firstDay) <= asOfT - anomaly.baseline_days * DAY_MS;
  65. 65 if (hasBaseline && today >= anomaly.min_usd && today > anomaly.multiplier * Math.max(med, 0.01)) add(today > 2 * anomaly.multiplier * Math.max(med, 0.01) ? "critical" : "warning", c, "spend spike", `Spent ${money(today, 2)} on ${asOf}, ${(today / Math.max(med, 0.01)).toFixed(1)}x the ${anomaly.baseline_days}-day median of ${money(med, 2)}.`, today, anomaly.multiplier * med);
  66. 66 customerRows.push({ customer: c, today: round(today, 2), median_prior: round(med, 2), month_to_date: round(mtd, 2), forecast_month: round(forecast, 2), daily_cap: L.daily_cap_usd ?? null, monthly_cap: L.monthly_cap_usd ?? null });
  67. 67}
  68. 68
  69. 69// Runaway runs and failure spikes on the as-of day
  70. 70const todays = toRuns(inScope.filter((r) => dayKey(r.date) === asOf));
  71. 71const runaways = todays
  72. 72 .filter((r) => (limits(r.customer).run_cost_cap_usd && r.cost > limits(r.customer).run_cost_cap_usd) || (limits(r.customer).max_calls_per_run && r.calls > limits(r.customer).max_calls_per_run))
  73. 73 .sort((a, b) => b.cost - a.cost);
  74. 74const runawayBy = new Map();
  75. 75for (const r of runaways) {
  76. 76 const k = `${r.customer}|${r.task}`;
  77. 77 const g = runawayBy.get(k) || { customer: r.customer, task: r.task, runs: 0, cost: 0, max_calls: 0 };
  78. 78 g.runs++;
  79. 79 g.cost += r.cost;
  80. 80 g.max_calls = Math.max(g.max_calls, r.calls);
  81. 81 runawayBy.set(k, g);
  82. 82}
  83. 83for (const g of runawayBy.values()) {
  84. 84 const L = limits(g.customer);
  85. 85 add(g.runs >= 3 ? "critical" : "warning", g.customer, "runaway runs", `${g.runs} ${g.task} run(s) over ${L.run_cost_cap_usd ? `${money(L.run_cost_cap_usd, 2)} per run` : ""}${L.run_cost_cap_usd && L.max_calls_per_run ? " or " : ""}${L.max_calls_per_run ? `${L.max_calls_per_run} calls per run` : ""} (up to ${g.max_calls} calls, ${money(g.cost, 2)} total).`, g.cost, L.run_cost_cap_usd ?? null);
  86. 86}
  87. 87const byTask = new Map();
  88. 88for (const r of todays) {
  89. 89 if (!r.outcome) continue;
  90. 90 const k = `${r.customer}|${r.task}`;
  91. 91 const g = byTask.get(k) || { customer: r.customer, task: r.task, runs: 0, failed: 0 };
  92. 92 g.runs++;
  93. 93 if (r.outcome !== "success") g.failed++;
  94. 94 byTask.set(k, g);
  95. 95}
  96. 96for (const g of byTask.values()) if (g.runs >= failure.min_runs && g.failed / g.runs > failure.max) add("warning", g.customer, "failure rate", `${g.failed} of ${g.runs} ${g.task} runs failed on ${asOf} (${pct(g.failed / g.runs, 0)}). Failed runs still cost money.`, g.failed / g.runs, failure.max);
  97. 97
  98. 98const rank = { critical: 0, warning: 1, info: 2 };
  99. 99alerts.sort((a, b) => rank[a.severity] - rank[b.severity] || b.value - a.value);
  100. 100const counts = { critical: alerts.filter((a) => a.severity === "critical").length, warning: alerts.filter((a) => a.severity === "warning").length, info: alerts.filter((a) => a.severity === "info").length };
  101. 101customerRows.sort((a, b) => b.month_to_date - a.month_to_date);
  102. 102
  103. 103writeOut(`${outPrefix}.json`, JSON.stringify({ as_of: asOf, counts, alerts, customers: customerRows, runaway_runs: runaways.slice(0, 20).map((r) => ({ run_id: r.id, customer: r.customer, task: r.task, calls: r.calls, cost: round(r.cost, 4), outcome: r.outcome })) }, null, 2) + "\n");
  104. 104const icon = { critical: "CRITICAL", warning: "WARNING", info: "info" };
  105. 105const md = [
  106. 106 `# Consumption guardrails: ${asOf}`,
  107. 107 "",
  108. 108 `${counts.critical} critical, ${counts.warning} warning, ${counts.info} info. Rules from \`${args.config}\`.`,
  109. 109 "",
  110. 110 "## Alerts",
  111. 111 "",
  112. 112 alerts.length ? mdTable(["Severity", "Customer", "Rule", "Detail"], alerts.map((a) => [icon[a.severity], a.customer, a.rule, a.detail])) : "No alerts.",
  113. 113 "",
  114. 114 "## Spend by customer",
  115. 115 "",
  116. 116 mdTable(
  117. 117 ["Customer", asOf, "Prior median", "Month to date", "Month forecast", "Daily cap", "Monthly cap"],
  118. 118 customerRows.map((c) => [c.customer, money(c.today, 2), money(c.median_prior, 2), money(c.month_to_date, 2), money(c.forecast_month, 2), c.daily_cap === null ? "none" : money(c.daily_cap, 2), c.monthly_cap === null ? "none" : money(c.monthly_cap, 2)]),
  119. 119 ),
  120. 120 "",
  121. 121 runaways.length ? `## Most expensive runs over the limit\n\n${mdTable(["Run", "Customer", "Task", "Calls", "Cost", "Outcome"], runaways.slice(0, 10).map((r) => [r.id, r.customer, r.task, r.calls, money(r.cost, 2), r.outcome || ""]))}\n` : "",
  122. 122].join("\n");
  123. 123writeOut(`${outPrefix}.md`, md);
  124. 124
  125. 125process.stdout.write(`Wrote ${outPrefix}.md and ${outPrefix}.json\n`);
  126. 126process.stdout.write(`${asOf}: ${counts.critical} critical, ${counts.warning} warning, ${counts.info} info.\n`);
  127. 127if (args["fail-on-alert"] && counts.critical) process.exit(2);
Sign in to comment on a line