{"openapi":"3.1.0","info":{"title":"AI Apps That Work free tools API","version":"1.0.0","description":"The free tools at https://aiappsthatwork.com/tools as JSON. No key and no sign-up. Inputs are computed and thrown away, never stored. Each response includes a permalink: a page on this site showing the same result. The same tools are on the MCP server at https://aiappsthatwork.com/mcp."},"servers":[{"url":"https://aiappsthatwork.com"}],"paths":{"/api/tools/real-cost-per-ai-result":{"post":{"operationId":"real_cost_per_result","summary":"Real cost per AI result","description":"Prices the AI steps behind one request from current list prices (LLM tokens, images, video seconds or a fixed amount) and returns the cost per request, the real cost per result that worked first time (cost per request divided by the worked-first-time share), the monthly spend wasted on results that didn't work, and credit-pricing numbers: margin per result and the credits per request needed to break even or reach a target margin.","externalDocs":{"url":"https://aiappsthatwork.com/tools/real-cost-per-ai-result"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"steps":{"minItems":1,"maxItems":4,"type":"array","items":{"oneOf":[{"type":"object","properties":{"kind":{"type":"string","const":"llm"},"model":{"type":"string","minLength":1,"description":"LLM model id from /models.json, for example anthropic/claude-haiku-5.5"},"inputTokens":{"type":"integer","minimum":0,"maximum":10000000,"description":"Input tokens per request"},"outputTokens":{"type":"integer","minimum":0,"maximum":10000000,"description":"Output tokens per request"},"cachedInputPct":{"default":0,"description":"Share of input tokens billed at the provider's cached-input price, 0 to 100","type":"number","minimum":0,"maximum":100},"note":{"description":"What the step does, for your own reference","type":"string","maxLength":120}},"required":["kind","model","inputTokens","outputTokens"]},{"type":"object","properties":{"kind":{"type":"string","const":"image"},"model":{"type":"string","minLength":1,"description":"Image model id from /models.json, for example openai/gpt-image-2"},"variant":{"type":"string","minLength":1,"description":"Quality and size label exactly as listed for the model in /models.json"},"images":{"default":1,"description":"Images per request","type":"integer","minimum":1,"maximum":100},"note":{"description":"What the step does, for your own reference","type":"string","maxLength":120}},"required":["kind","model","variant"]},{"type":"object","properties":{"kind":{"type":"string","const":"video"},"model":{"type":"string","minLength":1,"description":"Video model id from /models.json, for example google/veo-3.1-fast"},"variant":{"type":"string","minLength":1,"description":"Resolution and audio label exactly as listed for the model in /models.json"},"seconds":{"type":"number","exclusiveMinimum":0,"maximum":600,"description":"Seconds of video per request"},"note":{"description":"What the step does, for your own reference","type":"string","maxLength":120}},"required":["kind","model","variant","seconds"]},{"type":"object","properties":{"kind":{"type":"string","const":"custom"},"usd":{"type":"number","minimum":0,"maximum":1000,"description":"What this step costs per request, in USD"},"label":{"description":"What the step is, for example a speech-to-text API","type":"string","maxLength":120}},"required":["kind","usd"]}]},"description":"The AI steps behind one request, priced from /models.json. Their costs add up to the cost per request."},"workedFirstTimePct":{"type":"number","minimum":1,"maximum":100,"description":"Share of results that worked first time: no try again, undo, fix, re-ask or overrule. 1 to 100."},"requestsPerMonth":{"type":"number","minimum":0,"maximum":1000000000,"description":"Requests per month, try-agains included"},"pricing":{"description":"Credit pricing. Defaults: $10 for 1,000 credits, 1 credit per request, a 2.9% + $0.30 fee and a 70% target margin.","default":{},"type":"object","properties":{"packPriceUsd":{"default":10,"description":"What a user pays for one credit pack, USD","type":"number","exclusiveMinimum":0},"packCredits":{"default":1000,"description":"Credits in that pack","type":"number","exclusiveMinimum":0},"creditsPerRequest":{"default":1,"description":"Credits charged each time the AI runs","type":"number","exclusiveMinimum":0},"chargeOnlyWorked":{"default":false,"description":"True if users are charged only for results that worked","type":"boolean"},"feePct":{"default":2.9,"description":"Payment processor fee, percent of the pack price","type":"number","minimum":0,"exclusiveMaximum":100},"feeFixedUsd":{"default":0.3,"description":"Payment processor fixed fee per pack, USD","type":"number","minimum":0},"targetMarginPct":{"default":70,"description":"Gross margin wanted on each result that worked, percent","type":"number","minimum":0,"exclusiveMaximum":100}}}},"required":["steps","workedFirstTimePct","requestsPerMonth"]},"example":{"steps":[{"kind":"llm","model":"google/gemini-3.8-flash","inputTokens":4000,"outputTokens":800,"note":"Reads the invoice into fields"},{"kind":"llm","model":"anthropic/claude-haiku-5.5","inputTokens":1500,"outputTokens":100,"note":"Checks the totals"}],"workedFirstTimePct":75,"requestsPerMonth":20000}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"costPerRequestUsd":{"type":"number","description":"Sum of the steps' costs for one request"},"realCostPerResultUsd":{"type":"number","description":"Cost per request divided by the worked-first-time share: what each result that worked costs, counting failed tries"},"requestsPerResult":{"type":"number","description":"Requests paid for per result that worked"},"monthlySpendUsd":{"type":"number"},"wastedSpendPerMonthUsd":{"type":"number","description":"Monthly spend on results that didn't work first time"},"steps":{"type":"array","items":{"type":"object","properties":{"usd":{"type":"number","description":"This step's cost per request"},"detail":{"type":"string","description":"How the cost was worked out"},"model":{"type":["string","null"]},"priceSource":{"description":"The provider's pricing page","type":["string","null"]},"shutdown":{"anyOf":[{"type":"object","properties":{"date":{"type":"string"},"replacement":{"type":["string","null"]}},"required":["date","replacement"],"additionalProperties":false},{"type":"null"}],"description":"The model's announced API shutdown, if any"}},"required":["usd","detail","model","priceSource","shutdown"],"additionalProperties":false}},"credits":{"type":"object","properties":{"netPerCreditUsd":{"type":"number","description":"What one credit brings in after payment fees"},"revenuePerResultUsd":{"type":"number","description":"Net revenue per result that worked"},"marginPct":{"description":"Gross margin per result that worked; null when revenue is zero","type":["number","null"]},"userPricePerResultUsd":{"type":"number","description":"What the user pays per result that worked, before fees"},"breakEvenCreditsPerRequest":{"type":"number","description":"Fewest whole credits per request that don't lose money"},"suggestedCreditsPerRequest":{"type":"number","description":"Fewest whole credits per request that reach the target margin"},"targetMarginPct":{"type":"number"}},"required":["netPerCreditUsd","revenuePerResultUsd","marginPct","userPricePerResultUsd","breakEvenCreditsPerRequest","suggestedCreditsPerRequest","targetMarginPct"],"additionalProperties":false},"pricesChecked":{"type":"string","description":"Date the model prices were last checked"},"summary":{"type":"string"},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["costPerRequestUsd","realCostPerResultUsd","requestsPerResult","monthlySpendUsd","wastedSpendPerMonthUsd","steps","credits","pricesChecked","summary","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/tools/model-switch-cost":{"post":{"operationId":"model_switch_cost","summary":"Model switch cost","description":"Compares two priced LLMs on six common task shapes (support answer, invoice extraction, lead scoring, email draft, translation, search re-ranking). Returns list prices, cost per 1,000 requests on each model, the cost ratio, the break-even (how often the new model's results must work first time, relative to the current one, to cost no more per result that works) and the current model's announced shutdown.","externalDocs":{"url":"https://aiappsthatwork.com/models"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"from":{"type":"string","minLength":1,"maxLength":100,"description":"The current model: an LLM id from /models.json (openai/gpt-4o), an OpenRouter id or the model's name"},"to":{"type":"string","minLength":1,"maxLength":100,"description":"The new model: an LLM id from /models.json (openai/gpt-6-luna), an OpenRouter id or the model's name"}},"required":["from","to"]},"example":{"from":"openai/gpt-4o","to":"openai/gpt-6-luna"}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"from":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"provider":{"type":"string"},"inputPerMtokUsd":{"type":"number"},"outputPerMtokUsd":{"type":"number"},"cachedInputPerMtokUsd":{"type":["number","null"]},"priceSource":{"type":"string"},"priceChecked":{"type":"string","description":"Date this model's price was last confirmed (YYYY-MM-DD)"},"priceChanged":{"description":"Date this model's price last changed; null if unchanged since tracking began","type":["string","null"]}},"required":["id","name","provider","inputPerMtokUsd","outputPerMtokUsd","cachedInputPerMtokUsd","priceSource","priceChecked","priceChanged"],"additionalProperties":false},"to":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"provider":{"type":"string"},"inputPerMtokUsd":{"type":"number"},"outputPerMtokUsd":{"type":"number"},"cachedInputPerMtokUsd":{"type":["number","null"]},"priceSource":{"type":"string"},"priceChecked":{"type":"string","description":"Date this model's price was last confirmed (YYYY-MM-DD)"},"priceChanged":{"description":"Date this model's price last changed; null if unchanged since tracking began","type":["string","null"]}},"required":["id","name","provider","inputPerMtokUsd","outputPerMtokUsd","cachedInputPerMtokUsd","priceSource","priceChecked","priceChanged"],"additionalProperties":false},"tokenFactor":{"type":"number","description":"Tokens the new model counts for text that is 1 token on the current one, where the provider publishes it; otherwise 1"},"costChange":{"type":"string","description":"The change in cost per request across the tasks"},"tasks":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"inputTokens":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"outputTokens":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"assumption":{"type":"string","description":"What goes in and comes out; illustrative sizes, not measurements"},"fromCostPer1000Usd":{"type":"number"},"toCostPer1000Usd":{"type":"number"},"costRatio":{"type":"number","description":"New cost divided by current cost per request"},"breakEven":{"type":"string","description":"How often the new model's results must work first time, relative to the current model's, to cost no more per result that works"}},"required":["id","name","inputTokens","outputTokens","assumption","fromCostPer1000Usd","toCostPer1000Usd","costRatio","breakEven"],"additionalProperties":false}},"fromShutdown":{"anyOf":[{"type":"object","properties":{"date":{"type":"string"},"replacement":{"type":["string","null"]},"autoRoutes":{"type":"boolean"},"source":{"type":"string"}},"required":["date","replacement","autoRoutes","source"],"additionalProperties":false},{"type":"null"}],"description":"The current model's announced API shutdown, if any"},"note":{"type":["string","null"]},"switchTestUrl":{"description":"The browser test that runs your own prompt on both models with your own API key","type":["string","null"]},"pricesChecked":{"type":"string"},"summary":{"type":"string"},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["from","to","tokenFactor","costChange","tasks","fromShutdown","note","switchTestUrl","pricesChecked","summary","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/tools/model-prices":{"post":{"operationId":"model_prices","summary":"AI model prices","description":"Looks up list prices for LLMs (USD per million input, output and cached input tokens, and context size), image models (USD per image by quality and size) and video models (USD per second by resolution), with each price's source page, the date prices were checked and any announced API shutdown.","externalDocs":{"url":"https://aiappsthatwork.com/models"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"query":{"description":"A model id such as 'gpt-4o' (only exact id matches, if any), or words that must all appear in the id, name or provider, such as 'claude sonnet'. Omitted: every model.","type":"string","maxLength":100},"kind":{"default":"all","description":"llm for text models priced per token, image per image, video per second","type":"string","enum":["llm","image","video","all"]},"limit":{"default":25,"description":"Most models to return","type":"integer","minimum":1,"maximum":100}}},"example":{"query":"claude","kind":"llm"}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"pricesChecked":{"type":"string","description":"Date of the last full price check. Each model also has its own priceChecked and priceChanged."},"totalMatches":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"models":{"type":"array","items":{"type":"object","properties":{"kind":{"type":"string","enum":["llm","image","video"]},"id":{"type":"string"},"name":{"type":"string"},"provider":{"type":"string"},"inputPerMtokUsd":{"description":"LLMs: USD per million input tokens","type":["number","null"]},"outputPerMtokUsd":{"description":"LLMs: USD per million output tokens","type":["number","null"]},"cachedInputPerMtokUsd":{"description":"LLMs: USD per million cached input tokens, where offered","type":["number","null"]},"contextTokens":{"type":["number","null"]},"variants":{"anyOf":[{"type":"array","items":{"type":"object","properties":{"label":{"type":"string"},"usdPerImage":{"type":["number","null"]},"usdPerSecond":{"type":["number","null"]}},"required":["label","usdPerImage","usdPerSecond"],"additionalProperties":false}},{"type":"null"}],"description":"Image and video models: price per quality, size or resolution"},"notes":{"type":"string"},"priceSource":{"type":"string"},"priceChecked":{"type":"string","description":"Date this model's price was last confirmed against its source (YYYY-MM-DD)"},"priceChanged":{"description":"Date this model's price last changed; null if unchanged since tracking began","type":["string","null"]},"shutdown":{"anyOf":[{"type":"object","properties":{"date":{"type":"string"},"replacement":{"type":["string","null"]}},"required":["date","replacement"],"additionalProperties":false},{"type":"null"}],"description":"Announced API shutdown, if any"},"page":{"type":"string","description":"The page on this site that lists this model"}},"required":["kind","id","name","provider","inputPerMtokUsd","outputPerMtokUsd","cachedInputPerMtokUsd","contextTokens","variants","notes","priceSource","priceChecked","priceChanged","shutdown","page"],"additionalProperties":false}},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["pricesChecked","totalMatches","models","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/tools/model-shutdown-calendar":{"post":{"operationId":"model_shutdowns","summary":"AI model shutdown dates","description":"Lists announced API shutdown dates for AI models from OpenAI, Anthropic, Google and other providers, with days left, the suggested replacement, the source page, and links to a cost comparison and a side-by-side test of the replacement where this site has them.","externalDocs":{"url":"https://aiappsthatwork.com/tools/model-shutdown-calendar"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"query":{"description":"A model id such as 'gpt-4o' (only exact id matches, if any), or words that must all appear in the model ids or provider, such as 'anthropic'. Omitted: every model.","type":"string","maxLength":100},"includePast":{"default":false,"description":"Include shutdowns whose date has passed","type":"boolean"},"today":{"description":"Date to count days from, YYYY-MM-DD. Omitted: today in UTC.","type":"string","pattern":"^\\d{4}-\\d{2}-\\d{2}$"}}},"example":{"query":"gpt-4o"}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"today":{"type":"string"},"listChecked":{"type":"string","description":"Date the list was last checked against the providers' deprecation pages"},"count":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"shutdowns":{"type":"array","items":{"type":"object","properties":{"model":{"type":"string","description":"The model ids as the provider's deprecation page writes them"},"provider":{"type":"string"},"shutdownDate":{"type":"string","description":"When API calls naming the model stop working, YYYY-MM-DD"},"daysLeft":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Negative once the date has passed"},"replacement":{"description":"The provider's suggested replacement","type":["string","null"]},"source":{"type":"string"},"confidence":{"type":"string"},"switchPage":{"description":"This site's cost comparison for the move to the replacement","type":["string","null"]},"switchTestUrl":{"description":"The browser test with both models picked, run with your own API key","type":["string","null"]}},"required":["model","provider","shutdownDate","daysLeft","replacement","source","confidence","switchPage","switchTestUrl"],"additionalProperties":false}},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["today","listChecked","count","shutdowns","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/tools/prompt-ab-test-sample-size":{"post":{"operationId":"ab_test_sample_size","summary":"Prompt or model A/B test sample size","description":"Returns how many results each version of a prompt or model needs before a difference in how often results work first time is statistically real (two-proportion z-test, two-sided), the total across both versions, and how many days that takes at a given daily volume.","externalDocs":{"url":"https://aiappsthatwork.com/tools/prompt-ab-test-sample-size"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"baselinePct":{"type":"number","exclusiveMinimum":0,"exclusiveMaximum":100,"description":"Share of version A's (the current prompt or model) results that work first time, percent (above 0, below 100)"},"targetPct":{"type":"number","exclusiveMinimum":0,"exclusiveMaximum":100,"description":"Share of version B's results that work first time, percent (above 0, below 100)"},"confidencePct":{"default":95,"description":"Confidence level, percent. Two-sided.","type":"number","enum":[90,95,99]},"powerPct":{"default":80,"description":"Chance of detecting a real difference of this size, percent","type":"number","enum":[80,90]},"resultsPerDay":{"description":"Results per day across both versions, split evenly. Gives the number of days.","type":"number","exclusiveMinimum":0}},"required":["baselinePct","targetPct"]},"example":{"baselinePct":60,"targetPct":70,"resultsPerDay":100}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"perVersion":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"total":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"days":{"anyOf":[{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},{"type":"null"}],"description":"Null when resultsPerDay isn't given"},"baselinePct":{"type":"number"},"targetPct":{"type":"number"},"alpha":{"type":"number","description":"False-positive rate, 1 minus the confidence level"},"power":{"type":"number"},"method":{"type":"string"},"otherLifts":{"type":"array","items":{"type":"object","properties":{"liftPts":{"type":"number"},"targetPct":{"type":"number"},"perVersion":{"anyOf":[{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},{"type":"null"}]},"total":{"anyOf":[{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},{"type":"null"}]},"days":{"anyOf":[{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},{"type":"null"}]}},"required":["liftPts","targetPct","perVersion","total","days"],"additionalProperties":false},"description":"The same test for B at +2, +5, +10 and +15 points; null where B would pass 100%"},"summary":{"type":"string"},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["perVersion","total","days","baselinePct","targetPct","alpha","power","method","otherLifts","summary","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/tools/llm-judge-agreement":{"post":{"operationId":"judge_agreement","summary":"LLM judge agreement","description":"Compares a person's verdicts with an AI judge's verdicts on the same results. Returns the agreement rate, Cohen's kappa with confidence intervals, precision and recall per verdict, the agreement of a judge that always gives the most common verdict, a trust verdict (kappa 0.6 or more with at least 80% recall on every verdict; 0.4 to 0.6 is borderline; under 30 results is too few) and the rows where the two disagree.","externalDocs":{"url":"https://aiappsthatwork.com/tools/llm-judge-agreement"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"rows":{"description":"One row per result. Common words are mapped onto two verdicts: worked (worked, pass, yes, true, 1, good, kept, accepted, ok) and didn't work (didn't work, fail, no, false, 0, bad, redo, retry, rejected, wrong). Other labels are compared as written.","maxItems":5000,"type":"array","items":{"type":"object","properties":{"human":{"type":"string","minLength":1,"maxLength":100,"description":"The person's verdict on one result"},"judge":{"type":"string","minLength":1,"maxLength":100,"description":"The AI judge's verdict on the same result"},"note":{"description":"What the result was, to recognise it in the disagreements","type":"string","maxLength":300}},"required":["human","judge"]}},"csv":{"description":"The same data as CSV or TSV text instead of rows: columns named human and judge (and optionally note), or the first two columns in that order.","type":"string","maxLength":500000}}},"example":{"rows":[{"human":"worked","judge":"pass"},{"human":"worked","judge":"fail","note":"Correct two-line answer the judge called too short"},{"human":"didn't work","judge":"pass","note":"Quoted a 30-day refund window; the policy is 14 days"},{"human":"didn't work","judge":"fail"}]}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"n":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Results compared"},"skippedRows":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"CSV lines without both verdicts"},"agreement":{"type":"number","description":"Share of results where the judge and the person gave the same verdict"},"agreementCi":{"type":"array","prefixItems":[{"type":"number"},{"type":"number"}],"items":false,"minItems":2,"maxItems":2,"description":"95% confidence interval"},"kappa":{"description":"Cohen's kappa: agreement beyond chance. Null when it is undefined.","type":["number","null"]},"kappaCi":{"anyOf":[{"type":"array","prefixItems":[{"type":"number"},{"type":"number"}],"items":false,"minItems":2,"maxItems":2},{"type":"null"}]},"kappaBand":{"type":"string","description":"Landis and Koch label for kappa"},"alwaysSameVerdict":{"anyOf":[{"type":"object","properties":{"label":{"type":"string"},"agreement":{"type":"number","description":"0 to 1"}},"required":["label","agreement"],"additionalProperties":false},{"type":"null"}],"description":"The agreement a judge would get by always giving the most common human verdict"},"verdict":{"type":"object","properties":{"level":{"type":"string","enum":["too-few","trust","borderline","dont-trust"]},"headline":{"type":"string"},"detail":{"type":"string"}},"required":["level","headline","detail"],"additionalProperties":false},"warnings":{"type":"array","items":{"type":"string"},"description":"Problems with the labelled sample itself"},"perVerdict":{"type":"array","items":{"type":"object","properties":{"label":{"type":"string"},"precision":{"type":"number","description":"Of the results the judge gave this verdict, share the person agreed with"},"recall":{"type":"number","description":"Of the results the person gave this verdict, share the judge also gave"},"support":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Results the person gave this verdict"},"judged":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"Results the judge gave this verdict"}},"required":["label","precision","recall","support","judged"],"additionalProperties":false}},"matrix":{"type":"object","properties":{"labels":{"type":"array","items":{"type":"string"}},"counts":{"type":"array","items":{"type":"array","items":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"description":"counts[person][judge]"}},"required":["labels","counts"],"additionalProperties":false},"disagreementCount":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"disagreements":{"type":"array","items":{"type":"object","properties":{"row":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991,"description":"1-based position in the rows sent"},"human":{"type":"string"},"judge":{"type":"string"},"note":{"type":["string","null"]}},"required":["row","human","judge","note"],"additionalProperties":false},"description":"The first 25 results where the verdicts differ"},"summary":{"type":"string"},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["n","skippedRows","agreement","agreementCi","kappa","kappaCi","kappaBand","alwaysSameVerdict","verdict","warnings","perVerdict","matrix","disagreementCount","disagreements","summary","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/tools/judge-prompt-builder":{"post":{"operationId":"build_judge_prompt","summary":"Judge prompt builder","description":"Builds a pass/fail LLM-as-a-judge prompt for one kind of AI step (make, answer, transform, extract, judge, find, predict, plan, act or learn). Returns the prompt with {{input}} and {{result}} placeholders, numbered pass conditions (the block's starter conditions unless you send your own), up to 3 pass and 3 fail examples interleaved, and a JSON or XML answer format.","externalDocs":{"url":"https://aiappsthatwork.com/tools/judge-prompt-builder"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"block":{"type":"string","enum":["make","answer","transform","extract","judge","find","predict","plan","act","learn"],"description":"What the AI step does. make: creates new content from a request (an image, a post, slides, advice); answer: replies to a question that has a right answer (a support reply, a policy question, a how-to); transform: reshapes material faithfully (a rewrite, a translation, text to speech); extract: pulls out what's in the input (fields from a PDF, speech to text, text from a photo); judge: gives a label, score or choice that can be checked now (moderation, routing, a guardrail); find: picks the few relevant items out of many (search, recommendations); predict: makes an estimate that can only be checked later (will this lead buy, is this fraud, when will it arrive); plan: turns a goal into steps (a research plan, a multi-step task); act: takes one action with side effects (send an email, take a payment, call an API); learn: updates a model or prompt from examples (a voice clone, prompt tuning)."},"featureDescription":{"default":"","description":"Who uses the feature, what they ask for and what a good result gives them","type":"string","maxLength":2000},"criteria":{"description":"Pass conditions, one per item. Omitted: the starter conditions for the block.","minItems":1,"maxItems":10,"type":"array","items":{"type":"string","minLength":1,"maxLength":500}},"workedExamples":{"default":[],"description":"Real results that worked first time; labelled pass in the prompt","maxItems":3,"type":"array","items":{"type":"object","properties":{"input":{"default":"","description":"What the step received","type":"string","maxLength":4000},"result":{"type":"string","maxLength":4000,"description":"What the step produced, in full"}},"required":["result"]}},"failedExamples":{"default":[],"description":"Real results that didn't work; labelled fail in the prompt","maxItems":3,"type":"array","items":{"type":"object","properties":{"input":{"default":"","description":"What the step received","type":"string","maxLength":4000},"result":{"type":"string","maxLength":4000,"description":"What the step produced, in full"}},"required":["result"]}},"outputFormat":{"default":"json","description":"How the judge answers","type":"string","enum":["json","xml-tags"]},"includeReasoning":{"default":true,"description":"Whether the judge writes a short reason before its verdict","type":"boolean"}},"required":["block"]},"example":{"block":"extract","featureDescription":"An invoice reader for a bookkeeping app. It fills in supplier, invoice number, date, line items, VAT and total."}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string","description":"The judge prompt. Fill {{input}} and {{result}} for each result you check."},"block":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"},"description":"The pass conditions used"},"placeholders":{"type":"array","items":{"type":"string"}},"unfilled":{"type":"object","properties":{"feature":{"type":"boolean"},"criteria":{"type":"boolean"},"workedExamples":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"failedExamples":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"required":["feature","criteria","workedExamples","failedExamples"],"additionalProperties":false,"description":"Parts of the prompt still shown as [bracketed] placeholders"},"leftOutOfPermalink":{"type":"array","items":{"type":"string"},"description":"Inputs too long for the permalink, so its page shows the prompt without them"},"summary":{"type":"string"},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["prompt","block","criteria","placeholders","unfilled","leftOutOfPermalink","summary","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/api/tools/ai-launch-checklist":{"post":{"operationId":"launch_checklist","summary":"AI launch checklist","description":"The 20 AI-specific launch checks for an AI app (spending caps, rate limits, server-side keys, credit charging, version tags, records of which results worked first time, real cost per result, switching models safely). Returns the checks not yet done, each with why it matters and a request for a coding agent, plus one combined request.","externalDocs":{"url":"https://aiappsthatwork.com/tools/ai-launch-checklist"},"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"done":{"default":[],"description":"Ids of checks already done. Ids: spend-cap, rate-limits, server-keys, capped-retries, provider-budget, every-call-charged, deduct-server-side, refund-failures, one-charge-per-request, output-metadata, versioned-regenerations, outcome-events, events-by-version, step-costs, wasted-spend, who-it-fails-for, prompts-in-git, switch-flags, replay-set, shutdown-plan.","maxItems":30,"type":"array","items":{"type":"string","maxLength":60}},"group":{"description":"Only checks in one group: cost (cost and abuse), charging (charging for ai use), traceability (a record of every ai result), signals (what worked, for whom, at what cost), change (switching safely)","type":"string","enum":["cost","charging","traceability","signals","change"]}}},"example":{"done":["spend-cap","server-keys"]}}}},"responses":{"200":{"description":"The result, with a permalink","content":{"application/json":{"schema":{"type":"object","properties":{"total":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"doneCount":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"groups":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"title":{"type":"string"},"done":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991},"total":{"type":"integer","minimum":-9007199254740991,"maximum":9007199254740991}},"required":["id","title","done","total"],"additionalProperties":false}},"remaining":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"group":{"type":"string"},"title":{"type":"string"},"why":{"type":"string"},"prompt":{"type":"string","description":"A request for a coding agent that adds this check"}},"required":["id","group","title","why","prompt"],"additionalProperties":false}},"combinedPrompt":{"type":"string","description":"One message covering every remaining check, numbered; empty when none remain"},"unknownIds":{"type":"array","items":{"type":"string"},"description":"Ids in done that aren't checks"},"summary":{"type":"string"},"permalink":{"type":"string","description":"A page on this site that shows this result, for a person to open"}},"required":["total","doneCount","groups","remaining","combinedPrompt","unknownIds","summary","permalink"],"additionalProperties":false}}}},"400":{"description":"The body isn't valid JSON or doesn't match the schema","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"413":{"description":"The body is too large","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The input matches the schema but can't be computed, such as an unknown model id","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Too many requests from this address; see the Retry-After header","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}},"/models.json":{"get":{"operationId":"model_catalogue","summary":"Model prices and shutdowns","description":"Every priced LLM, image and video model with its source page, and every announced API shutdown, as one JSON file.","responses":{"200":{"description":"The catalogue","content":{"application/json":{"schema":{"type":"object"}}}}}}}},"components":{"schemas":{"Error":{"type":"object","properties":{"error":{"type":"string"}},"required":["error"]}}}}