Methodology: prompt cache hit rate in agent loops

// Cache hit rate under "correct" agent-loop usage: append-only threads, gaps well inside every TTL.
// Usage: pnpm tsx missrate.ts run claude,gpt        (about 40 min, writes missrate-<providers>.jsonl)
//        pnpm tsx missrate.ts report <files...>     (prints hit rates and cost vs a perfect cache)
import Anthropic from "@anthropic-ai/sdk";
import OpenAI from "openai";
import { GoogleGenAI } from "@google/genai";
import { appendFileSync, readFileSync } from "node:fs";
import { randomUUID } from "node:crypto";
import { setTimeout as sleep } from "node:timers/promises";

const GAPS_SEC = [5, 30, 120];
const THREADS_PER_GAP = 4;
const TURNS = 20;

const PROVIDERS = ["claude", "gpt", "gemini"] as const;
type Provider = (typeof PROVIDERS)[number];
type Usage = { prompt: number; read: number; write: number; output: number };

// $ per 1M tokens. Gemini has no write fee, so its "write" rate is the plain input rate.
const PRICE: Record<Provider, { input: number; write: number; read: number; output: number }> = {
  claude: { input: 2, write: 2.5, read: 0.1, output: 10 },
  gpt: { input: 2, write: 2.5, read: 0.1, output: 10 },
  gemini: { input: 2, write: 2, read: 0.1, output: 10 },
};

type Row = {
  provider: Provider;
  gapSec: number;
  thread: string;
  turn: number;
  expected: number; // previous request's full prompt: what a perfect cache would read
  ms: number;
  error?: string;
} & Partial<Usage>;

// One API call per provider. `users` alternate with a fixed "OK." assistant turn, so each request
// is a pure append of the one before.
const call: Record<Provider, (system: string, users: string[]) => Promise<Usage>> = {
  claude: async (system, users) => {
    const client = (clients.claude ??= new Anthropic({
      apiKey: process.env.ANTHROPIC_API_KEY,
      authToken: null,
      baseURL: "https://api.anthropic.com",
    }));
    const res = await client.messages.create({
      model: "claude-sonnet-5-5",
      max_tokens: 16,
      thinking: { type: "between_tools" },
      system,
      messages: users.flatMap((text, i) =>
        i === 0
          ? [{ role: "user" as const, content: text }]
          : [{ role: "assistant" as const, content: "OK." }, { role: "user" as const, content: text }],
      ),
      cache_control: { type: "ephemeral", ttl: "5m" },
    });
    const u = res.usage;
    const read = u.cache_read_input_tokens ?? 0;
    const write = u.cache_creation_input_tokens ?? 0;
    return { prompt: u.input_tokens + read + write, read, write, output: u.output_tokens };
  },
  gpt: async (system, users) => {
    const client = (clients.gpt ??= new OpenAI({
      apiKey: process.env.OPENAI_API_KEY,
      baseURL: "https://api.openai.com/v1",
    }));
    const res = await client.responses.create({
      model: "gpt-6.1-sol",
      max_output_tokens: 16,
      store: false,
      input: [
        { role: "developer", content: system },
        ...users.flatMap((text, i) =>
          i === 0
            ? [{ role: "user" as const, content: text }]
            : [{ role: "assistant" as const, content: "OK." }, { role: "user" as const, content: text }],
        ),
      ],
    });
    const u = res.usage;
    if (!u) throw new Error(`no usage, status=${res.status}`);
    return {
      prompt: u.input_tokens,
      read: u.input_tokens_details.cached_tokens,
      write: u.input_tokens_details.cache_write_tokens,
      output: u.output_tokens,
    };
  },
  gemini: async (system, users) => {
    const client = (clients.gemini ??= new GoogleGenAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY }));
    const res = await client.models.generateContent({
      model: "gemini-4-argon",
      contents: users.flatMap((text, i) => {
        const user = { role: "user", parts: [{ text }] };
        return i === 0 ? [user] : [{ role: "model", parts: [{ text: "OK." }] }, user];
      }),
      config: { systemInstruction: system, maxOutputTokens: 16 },
    });
    const u = res.usageMetadata;
    if (!u?.promptTokenCount) throw new Error("no usage");
    return {
      prompt: u.promptTokenCount,
      read: u.cachedContentTokenCount ?? 0,
      write: 0,
      output: (u.candidatesTokenCount ?? 0) + (u.thoughtsTokenCount ?? 0),
    };
  },
};
const clients: { claude?: Anthropic; gpt?: OpenAI; gemini?: GoogleGenAI } = {};

// About 20-25k tokens of instructions and reference text.
const systemPrompt = (nonce: string) =>
  [
    `Session ${nonce}. You are a coding agent. Reply with only OK.`,
    ...Array.from(
      { length: 300 },
      (_, i) => `Reference line ${i} [${nonce}]: alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima.`,
    ),
  ].join("\n");

// About 1.5k tokens of fake tool output with real entropy, like file reads or command logs.
const toolResult = (turn: number) =>
  [
    `Tool result for step ${turn}:`,
    ...Array.from({ length: 40 }, (_, i) => `src/module_${turn}/file_${i}.ts:${i * 7 + 3} ${randomUUID()} ok`),
  ].join("\n");

async function thread(provider: Provider, gapSec: number, out: string) {
  const id = randomUUID().slice(0, 8);
  const system = systemPrompt(id);
  const users = ["Start the task."];
  let expected = 0;
  for (let turn = 0; turn < TURNS; turn++) {
    if (turn > 0) users.push(toolResult(turn));
    const started = Date.now();
    const base = { provider, gapSec, thread: id, turn, expected };
    try {
      const usage = await call[provider](system, users);
      expected = usage.prompt;
      appendFileSync(out, JSON.stringify({ ...base, ...usage, ms: Date.now() - started } satisfies Row) + "\n");
    } catch (e) {
      const row: Row = { ...base, ms: Date.now() - started, error: String(e).slice(0, 300) };
      appendFileSync(out, JSON.stringify(row) + "\n");
    }
    await sleep(gapSec * 1000);
  }
}

async function run(providers: Provider[]) {
  process.loadEnvFile(".env");
  const out = `missrate-${providers.join("-")}.jsonl`;
  await Promise.all(
    providers.flatMap((p) =>
      GAPS_SEC.flatMap((gap) => Array.from({ length: THREADS_PER_GAP }, () => thread(p, gap, out))),
    ),
  );
  console.log(`done, wrote ${out}`);
}

// A turn is a "full hit" when it reads at least 90% of the previous prompt, a "miss" when it reads nothing.
// Cost inflation compares the real bill to one where every turn reads the whole previous prompt.
function report(files: string[]) {
  const rows = files.flatMap((f) =>
    readFileSync(f, "utf8")
      .trim()
      .split("\n")
      .map((l) => JSON.parse(l) as Row),
  );
  const cost = (p: Provider, prompt: number, read: number, write: number, output: number) => {
    const r = PRICE[p];
    return (read * r.read + write * r.write + (prompt - read - write) * r.input + output * r.output) / 1e6;
  };
  const groups = Map.groupBy(rows, (r) => `${r.provider}|${r.gapSec}`);
  const table = [...groups].map(([key, group]) => {
    const [provider, gap] = key.split("|");
    const p = provider as Provider;
    const scored = group.filter((r) => r.turn > 0 && !r.error && r.prompt !== undefined);
    const ratio = (r: Row) => (r.read ?? 0) / r.expected;
    const full = scored.filter((r) => ratio(r) >= 0.9).length;
    const miss = scored.filter((r) => (r.read ?? 0) === 0).length;
    const actual = scored.reduce((s, r) => s + cost(p, r.prompt!, r.read!, r.write!, r.output!), 0);
    const ideal = scored.reduce((s, r) => {
      const read = Math.min(r.expected, r.prompt!);
      // Ideal write: Claude/GPT write the new tail; Gemini bills it as input (its write rate equals input).
      return s + cost(p, r.prompt!, read, r.prompt! - read, r.output!);
    }, 0);
    return {
      provider,
      gapSec: Number(gap),
      turns: scored.length,
      errors: group.filter((r) => r.error).length,
      "full hit %": +((100 * full) / scored.length).toFixed(1),
      "partial %": +((100 * (scored.length - full - miss)) / scored.length).toFixed(1),
      "miss %": +((100 * miss) / scored.length).toFixed(1),
      "actual $": +actual.toFixed(3),
      "perfect-cache $": +ideal.toFixed(3),
      "cost x": +(actual / ideal).toFixed(2),
    };
  });
  console.table(table.sort((a, b) => a.provider.localeCompare(b.provider) || a.gapSec - b.gapSec));
}

const [cmd, ...rest] = process.argv.slice(2);
if (cmd === "run") {
  const providers = (rest[0] ?? "").split(",").filter((p): p is Provider => PROVIDERS.includes(p as Provider));
  if (!providers.length) throw new Error(`usage: run ${PROVIDERS.join(",")}`);
  await run(providers);
} else if (cmd === "report" && rest.length) {
  report(rest);
} else {
  throw new Error("usage: pnpm tsx missrate.ts run <providers> | report <files...>");
}

{
  "name": "cache-probe",
  "private": true,
  "type": "module",
  "dependencies": {
    "@anthropic-ai/sdk": "^0.135.0",
    "@google/genai": "^2.30.0",
    "openai": "^7.34.0"
  },
  "devDependencies": {
    "@types/node": "^26.6.5",
    "tsx": "^4.23.15",
    "typescript": "^7.0.2"
  }
}

{
  "compilerOptions": {
    "target": "es2024",
    "module": "nodenext",
    "strict": true,
    "noEmit": true,
    "skipLibCheck": true,
    "types": ["node"]
  },
  "include": ["*.ts"]
}

allowBuilds:
  '@google/genai': false
  esbuild: true
  protobufjs: false