Skip to content
SlashClone
DiscoverUse in ClaudeSubmit a link
Sign in
Made by@gtmengineer

Enrichment waterfall planner

@gtmengineer/enrichment-waterfall-planner

Plan an enrichment waterfall with real math. Give it your providers with cost, match rate, and accuracy (or a sample test run), and it tries every order to find the cheapest setup that hits your fill target, with cost per usable record, the marginal cost of each step, and where to stop. Vendor-neutral.

SkillDatav1.0.0 (latest)Tested
Use in Claude Save Like Clone Comment
Activity
New: be the first to try it
  1. 1#!/usr/bin/env node
  2. 2// enrichment-waterfall-planner: find the provider order (and where to stop) for an
  3. 3// enrichment waterfall. Models fill rate, accuracy, optional verification, and cost
  4. 4// per lookup or per match, from your estimates or from a sample test run.
  5. 5// Node 22, built-ins only. Vendor-neutral: providers are whatever you name them.
  6. 6//
  7. 7// Usage:
  8. 8// node scripts/waterfall.mjs <providers.json> [--sample sample.csv] [--out out] [--name waterfall]
  9. 9
  10. 10import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
  11. 11import { join } from "node:path";
  12. 12
  13. 13function fail(msg) {
  14. 14 console.error(`enrichment-waterfall-planner: ${msg}`);
  15. 15 process.exit(1);
  16. 16}
  17. 17
  18. 18const args = process.argv.slice(2);
  19. 19const opts = { out: "out", name: "waterfall", sample: null };
  20. 20const pos = [];
  21. 21for (let i = 0; i < args.length; i++) {
  22. 22 if (args[i] === "--out") opts.out = args[++i];
  23. 23 else if (args[i] === "--name") opts.name = args[++i];
  24. 24 else if (args[i] === "--sample") opts.sample = args[++i];
  25. 25 else if (args[i] === "--help" || args[i] === "-h") {
  26. 26 console.log("Usage: node scripts/waterfall.mjs <providers.json> [--sample sample.csv] [--out out] [--name waterfall]");
  27. 27 process.exit(0);
  28. 28 } else pos.push(args[i]);
  29. 29}
  30. 30if (pos.length !== 1) fail("give a providers JSON file. Try --help.");
  31. 31
  32. 32const read = (p) => {
  33. 33 try {
  34. 34 return readFileSync(p, "utf8");
  35. 35 } catch {
  36. 36 fail(`cannot read ${p}`);
  37. 37 }
  38. 38};
  39. 39let cfg;
  40. 40try {
  41. 41 cfg = JSON.parse(read(pos[0]));
  42. 42} catch (e) {
  43. 43 fail(`${pos[0]} is not valid JSON: ${e.message}`);
  44. 44}
  45. 45const providers = cfg.providers;
  46. 46if (!Array.isArray(providers) || providers.length === 0) fail("providers must be a non-empty array");
  47. 47if (providers.length > 8) fail("at most 8 providers (the planner tries every order)");
  48. 48const N = Number(cfg.records ?? 1000);
  49. 49const target = cfg.min_fill_rate != null ? Number(cfg.min_fill_rate) : null;
  50. 50const maxPerRecord = cfg.max_cost_per_record != null ? Number(cfg.max_cost_per_record) : null;
  51. 51const verifyCost = cfg.verify ? Number(cfg.verify.cost ?? 0) : null;
  52. 52const names = new Set();
  53. 53for (const p of providers) {
  54. 54 if (!p.name) fail("every provider needs a name");
  55. 55 if (names.has(p.name)) fail(`duplicate provider name "${p.name}"`);
  56. 56 names.add(p.name);
  57. 57 if (!(Number(p.cost) >= 0)) fail(`${p.name}: cost must be a number of at least 0`);
  58. 58 p.cost = Number(p.cost);
  59. 59 p.charge = p.charge === "per_match" ? "per_match" : "per_lookup";
  60. 60 p.accuracy = p.accuracy == null ? 1 : Number(p.accuracy);
  61. 61 if (!(p.accuracy > 0 && p.accuracy <= 1)) fail(`${p.name}: accuracy must be between 0 and 1`);
  62. 62 if (!opts.sample) {
  63. 63 p.match_rate = Number(p.match_rate);
  64. 64 if (!(p.match_rate >= 0 && p.match_rate <= 1)) fail(`${p.name}: match_rate must be between 0 and 1 (or pass --sample)`);
  65. 65 }
  66. 66}
  67. 67
  68. 68// ---------- sample ----------
  69. 69let sample = null;
  70. 70if (opts.sample) {
  71. 71 const text = read(opts.sample).replace(/\r\n?/g, "\n");
  72. 72 const lines = text.split("\n").filter((l) => l.trim());
  73. 73 const split = (l) => {
  74. 74 const out = [];
  75. 75 let f = "";
  76. 76 let q = false;
  77. 77 for (let i = 0; i < l.length; i++) {
  78. 78 const c = l[i];
  79. 79 if (q) {
  80. 80 if (c === '"' && l[i + 1] === '"') {
  81. 81 f += '"';
  82. 82 i++;
  83. 83 } else if (c === '"') q = false;
  84. 84 else f += c;
  85. 85 } else if (c === '"') q = true;
  86. 86 else if (c === ",") {
  87. 87 out.push(f.trim());
  88. 88 f = "";
  89. 89 } else f += c;
  90. 90 }
  91. 91 out.push(f.trim());
  92. 92 return out;
  93. 93 };
  94. 94 const header = split(lines[0]);
  95. 95 for (const p of providers) if (!header.includes(p.name)) fail(`sample has no column named "${p.name}". Columns: ${header.join(", ")}`);
  96. 96 sample = lines.slice(1).map((l) => {
  97. 97 const cells = split(l);
  98. 98 return Object.fromEntries(providers.map((p) => [p.name, (cells[header.indexOf(p.name)] || "") !== ""]));
  99. 99 });
  100. 100 if (!sample.length) fail("sample has no rows");
  101. 101 for (const p of providers) p.match_rate = sample.filter((r) => r[p.name]).length / sample.length;
  102. 102}
  103. 103
  104. 104// ---------- simulate one order ----------
  105. 105// Each record carries probability mass that it still needs a value. A provider that
  106. 106// returns a value stops the waterfall for that record, unless verification is on and
  107. 107// the value fails it (probability 1 - accuracy), in which case the record continues.
  108. 108function simulate(order) {
  109. 109 const steps = order.map((p) => ({ provider: p.name, reach: 0, found: 0, usable: 0, cost: 0 }));
  110. 110 const run = (has) => {
  111. 111 let mass = 1;
  112. 112 order.forEach((p, i) => {
  113. 113 if (mass <= 0) return;
  114. 114 const s = steps[i];
  115. 115 s.reach += mass;
  116. 116 const hit = has(p) * mass;
  117. 117 s.found += hit;
  118. 118 s.usable += hit * p.accuracy;
  119. 119 s.cost += p.charge === "per_lookup" ? mass * p.cost : hit * p.cost;
  120. 120 if (verifyCost != null) s.cost += hit * verifyCost;
  121. 121 mass -= verifyCost != null ? hit * p.accuracy : hit;
  122. 122 });
  123. 123 };
  124. 124 if (sample) {
  125. 125 for (const r of sample) run((p) => (r[p.name] ? 1 : 0));
  126. 126 const scale = N / sample.length;
  127. 127 for (const s of steps) for (const k of ["reach", "found", "usable", "cost"]) s[k] *= scale;
  128. 128 } else {
  129. 129 run((p) => p.match_rate);
  130. 130 for (const s of steps) for (const k of ["reach", "found", "usable", "cost"]) s[k] *= N;
  131. 131 }
  132. 132 let cumUsable = 0;
  133. 133 let cumCost = 0;
  134. 134 for (const s of steps) {
  135. 135 s.marginal_cost_per_usable = s.usable > 0 ? s.cost / s.usable : null;
  136. 136 cumUsable += s.usable;
  137. 137 cumCost += s.cost;
  138. 138 s.cumulative_fill_rate = cumUsable / N;
  139. 139 s.cumulative_cost = cumCost;
  140. 140 }
  141. 141 return { order: order.map((p) => p.name), steps, usable: cumUsable, fill_rate: cumUsable / N, cost: cumCost, cost_per_usable: cumUsable ? cumCost / cumUsable : null };
  142. 142}
  143. 143
  144. 144function* permutations(arr) {
  145. 145 if (arr.length <= 1) {
  146. 146 yield arr;
  147. 147 return;
  148. 148 }
  149. 149 for (let i = 0; i < arr.length; i++) for (const rest of permutations([...arr.slice(0, i), ...arr.slice(i + 1)])) yield [arr[i], ...rest];
  150. 150}
  151. 151// every ordered subset = every prefix of every permutation
  152. 152const seen = new Map();
  153. 153for (const perm of permutations(providers))
  154. 154 for (let k = 1; k <= perm.length; k++) {
  155. 155 const key = perm.slice(0, k).map((p) => p.name).join(">");
  156. 156 if (!seen.has(key)) seen.set(key, simulate(perm.slice(0, k)));
  157. 157 }
  158. 158const all = [...seen.values()];
  159. 159const EPS = 1e-9;
  160. 160const bestByDepth = [];
  161. 161for (let k = 1; k <= providers.length; k++) {
  162. 162 const opts2 = all.filter((r) => r.order.length === k);
  163. 163 const maxFill = Math.max(...opts2.map((r) => r.fill_rate));
  164. 164 // best = highest fill, then lowest cost
  165. 165 const best = opts2.filter((r) => r.fill_rate >= maxFill - 0.005).sort((a, b) => a.cost - b.cost)[0];
  166. 166 bestByDepth.push(best);
  167. 167}
  168. 168let recommended;
  169. 169let why;
  170. 170if (target != null) {
  171. 171 const ok = all.filter((r) => r.fill_rate >= target - EPS).sort((a, b) => a.cost - b.cost || a.order.length - b.order.length);
  172. 172 if (ok.length) {
  173. 173 recommended = ok[0];
  174. 174 why = `cheapest setup that reaches the ${(target * 100).toFixed(0)}% usable fill target`;
  175. 175 } else {
  176. 176 recommended = bestByDepth[bestByDepth.length - 1];
  177. 177 why = `no setup reaches ${(target * 100).toFixed(0)}%; this is the highest fill at the lowest cost`;
  178. 178 }
  179. 179} else {
  180. 180 recommended = bestByDepth[bestByDepth.length - 1];
  181. 181 why = "highest usable fill at the lowest cost (no target set)";
  182. 182}
  183. 183// trim tail steps whose marginal cost is above the cap
  184. 184const cutSuggestions = [];
  185. 185if (maxPerRecord != null)
  186. 186 for (const s of recommended.steps)
  187. 187 if (s.marginal_cost_per_usable != null && s.marginal_cost_per_usable > maxPerRecord)
  188. 188 cutSuggestions.push(`${s.provider}: each extra usable record costs ${s.marginal_cost_per_usable.toFixed(3)}, above your cap of ${maxPerRecord}`);
  189. 189
  190. 190const r2 = (x) => (x == null ? null : Math.round(x * 100) / 100);
  191. 191const r3 = (x) => (x == null ? null : Math.round(x * 1000) / 1000);
  192. 192const pctS = (x) => `${(x * 100).toFixed(1)}%`;
  193. 193const shape = (r) => ({
  194. 194 order: r.order,
  195. 195 usable_records: Math.round(r.usable),
  196. 196 fill_rate: r3(r.fill_rate),
  197. 197 total_cost: r2(r.cost),
  198. 198 cost_per_usable: r3(r.cost_per_usable),
  199. 199 steps: r.steps.map((s) => ({
  200. 200 provider: s.provider,
  201. 201 lookups: Math.round(s.reach),
  202. 202 found: Math.round(s.found),
  203. 203 usable: Math.round(s.usable),
  204. 204 cost: r2(s.cost),
  205. 205 marginal_cost_per_usable: r3(s.marginal_cost_per_usable),
  206. 206 cumulative_fill_rate: r3(s.cumulative_fill_rate),
  207. 207 })),
  208. 208});
  209. 209const result = {
  210. 210 data_point: cfg.data_point || "data point",
  211. 211 records: N,
  212. 212 source: sample ? `sample of ${sample.length} records` : "estimated match rates (independence assumed)",
  213. 213 verification: verifyCost != null,
  214. 214 provider_rates: providers.map((p) => ({ name: p.name, match_rate: r3(p.match_rate), accuracy: p.accuracy, cost: p.cost, charge: p.charge })),
  215. 215 recommended: { ...shape(recommended), why },
  216. 216 best_by_depth: bestByDepth.map(shape),
  217. 217 cut_suggestions: cutSuggestions,
  218. 218};
  219. 219
  220. 220const md = [`# Enrichment waterfall: ${result.data_point}`, ""];
  221. 221md.push(`${N.toLocaleString("en-US")} records. Rates from ${result.source}. Verification ${verifyCost != null ? `on (${verifyCost} per found value)` : "off"}.`, "");
  222. 222md.push(`**Recommended order: ${recommended.order.join(" > ")}**: ${pctS(recommended.fill_rate)} usable fill, total ${r2(recommended.cost)}, ${r3(recommended.cost_per_usable)} per usable record (${why}).`, "");
  223. 223md.push("| Step | Provider | Lookups | Found | Usable | Step cost | Cost per extra usable | Cumulative fill |", "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: |");
  224. 224shape(recommended).steps.forEach((s, i) =>
  225. 225 md.push(`| ${i + 1} | ${s.provider} | ${s.lookups} | ${s.found} | ${s.usable} | ${s.cost} | ${s.marginal_cost_per_usable ?? "-"} | ${pctS(s.cumulative_fill_rate)} |`),
  226. 226);
  227. 227md.push("", "## Best setup by number of providers", "", "| Providers | Order | Usable fill | Total cost | Per usable |", "| ---: | --- | ---: | ---: | ---: |");
  228. 228bestByDepth.forEach((b, i) => md.push(`| ${i + 1} | ${b.order.join(" > ")} | ${pctS(b.fill_rate)} | ${r2(b.cost)} | ${r3(b.cost_per_usable)} |`));
  229. 229if (cutSuggestions.length) md.push("", "## Consider cutting", "", ...cutSuggestions.map((c) => `- ${c}`));
  230. 230md.push("", "Estimates, not guarantees. Re-test on a fresh sample before committing spend.", "");
  231. 231
  232. 232mkdirSync(opts.out, { recursive: true });
  233. 233writeFileSync(join(opts.out, `${opts.name}.json`), JSON.stringify(result, null, 2) + "\n");
  234. 234writeFileSync(join(opts.out, `${opts.name}.md`), md.join("\n"));
  235. 235console.log(`Recommended ${recommended.order.join(" > ")}: ${pctS(recommended.fill_rate)} usable fill at ${r3(recommended.cost_per_usable)} per usable record. Wrote ${join(opts.out, opts.name)}.{md,json}`);
Sign in to comment on a line