{ "openapi": "3.1.0", "info": { "title": "Benchlist API", "version": "1.1.0", "description": "Run AI benchmarks with cryptographic attestation. Free anonymous probe at /probe, paid n=50 attestations at /run. Every receipt is Ed25519 signed and replayable for $0.50. Optional ZK anchor on Ethereum L1 via Aligned Layer (queued; not yet on mainnet). See /llms.txt for full agent guide.", "contact": { "name": "Benchlist support", "email": "dev@remlabs.ai", "url": "https://benchlist.ai/docs" }, "license": { "name": "MIT", "url": "https://github.com/benchlist/runner/blob/main/LICENSE" } }, "servers": [ { "url": "https://benchlist.ai/api/v1", "description": "Production" } ], "security": [ { "bearerAuth": [] } ], "paths": { "/submit": { "post": { "summary": "Signup (email-only) and form submissions", "description": "Set `kind=signup` with a contact email to receive a free API key by email. Other kinds: `list` (list a service), `run` (publish a pre-run transcript), `waitlist`, `dispute`, `contact`, `attestor` (register an Ed25519 pubkey), `newsletter` (Contamination Index subscribe).", "security": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "kind" ], "properties": { "kind": { "type": "string", "enum": [ "signup", "list", "run", "waitlist", "dispute", "contact", "attestor", "newsletter" ] }, "contact": { "type": "string", "format": "email" }, "run": { "type": "object", "description": "Canonical run.json (for kind=run)" } } }, "example": { "kind": "signup", "contact": "you@company.com" } } } }, "responses": { "200": { "description": "Accepted", "content": { "application/json": { "example": { "ok": true, "id": "sub_abc123", "request_id": "req_\u2026", "issued": true, "key_delivered_to": "you@company.com", "forwarded": [ { "channel": "resend:user", "ok": true, "status": 200 } ] } } } } } } }, "/run": { "post": { "summary": "Queue a benchmark run", "description": "The attestor runs the benchmark against your service, Merkle-commits the transcript, generates a ZK proof, and submits to Aligned Layer. Bring your own model provider API key via environment variables on the attestor side \u2014 Benchlist does not proxy inference.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "service", "model", "benchmark" ], "properties": { "service": { "type": "string", "description": "Service ID from /services" }, "model": { "type": "string", "description": "Model identifier (e.g. claude-sonnet-4-5-20250929)" }, "benchmark": { "type": "string", "description": "Benchmark ID from /benchmarks" }, "runs": { "type": "integer", "minimum": 1, "maximum": 10, "default": 1 }, "proof_system": { "type": "string", "enum": [ "sp1", "risc0", "halo2", "groth16", "plonk", "signed-attestation", "auto" ], "default": "auto" }, "webhook": { "type": "string", "format": "uri" }, "idempotency_key": { "type": "string", "maxLength": 128 } } }, "example": { "service": "anthropic-claude", "model": "claude-sonnet-4-5-20250929", "benchmark": "humaneval", "runs": 3 } } } }, "responses": { "202": { "description": "Accepted", "content": { "application/json": { "example": { "run_id": "run-8f3a\u2026", "status": "queued", "est_seconds": 180, "charge": { "credits": 3, "usd": 15 }, "verify_url": "https://benchlist.ai/verify/run-8f3a\u2026" } } } }, "401": { "description": "Missing or invalid Bearer key" }, "400": { "description": "Invalid payload" } } } }, "/keys": { "post": { "summary": "Verify or rotate API keys", "description": "Two actions supported: `verify` checks a key's HMAC; `rotate` (requires valid current Bearer) issues a new key for the same email.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "action" ], "properties": { "action": { "type": "string", "enum": [ "verify", "rotate" ] }, "key": { "type": "string", "description": "Required for action=verify" } } } } } }, "responses": { "200": { "description": "verify returns {valid, payload}; rotate returns {rotated, key, payload}" } } } }, "/crypto": { "post": { "summary": "Native ETH payment flow", "description": "Two-mode: init (returns receiving address + live ETH price) and verify (validates a submitted tx hash on-chain).", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "plan" ], "properties": { "plan": { "type": "string", "enum": [ "test_1", "credits_25", "credits_100", "credits_500", "credits_2000" ] }, "chain": { "type": "string", "enum": [ "base", "ethereum", "arbitrum" ] }, "txHash": { "type": "string", "pattern": "^0x[0-9a-fA-F]{64}$" } } } } } }, "responses": { "200": { "description": "init \u2192 {amountEth, ethPriceUsd, receiver, chains} \u00b7 verify \u2192 {verified, sentEth, confirmations, blockNumber}" } } } }, "/checkout": { "post": { "summary": "Stripe Checkout session", "description": "Returns a hosted Stripe Checkout URL. Use this for card payments.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "plan" ], "properties": { "plan": { "type": "string" } } } } } }, "responses": { "200": { "description": "{id, url, plan}" } } } }, "/replay": { "post": { "summary": "Queue an independent replay of a published run", "description": "Re-executes any published run.json in a clean container, issues a fresh Ed25519 signature, returns a new run_id linked via replayOf. $0.50 per replay.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "format": "uri", "description": "Benchlist verify URL or run.json URL" }, "priceUsd": { "type": "number", "default": 0.5 } } } } } }, "responses": { "202": { "description": "Accepted", "content": { "application/json": { "example": { "replay_id": "rep_abc123", "est_seconds": 600 } } } }, "501": { "description": "Replay service in private beta" } } } }, "/runs": { "get": { "summary": "List signed runs", "description": "Optional filters. Same shape as /api/runs.json but filterable without pulling the full registry.", "security": [], "parameters": [ { "name": "benchmark", "in": "query", "schema": { "type": "string" } }, { "name": "service", "in": "query", "schema": { "type": "string" } }, { "name": "model", "in": "query", "schema": { "type": "string" } }, { "name": "contaminationRisk", "in": "query", "schema": { "type": "string", "enum": [ "low", "medium", "high" ] } }, { "name": "limit", "in": "query", "schema": { "type": "integer", "minimum": 1, "maximum": 500, "default": 50 } } ], "responses": { "200": { "description": "{count, runs: [...]}" } } } }, "/probe": { "post": { "summary": "Anonymous bootstrap probe (no auth, free, rate-limited)", "description": "Run a small (n=3) attested benchmark probe with no API key required. Same shape as a paid run \u2014 Ed25519 signed, public verify URL \u2014 but capped at 3 samples and rate-limited to 1 successful probe per IP per hour, 5 per day. Inference cost goes through our pool. Use this to bootstrap integrations or validate a model before signing up.", "security": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "benchmark", "model" ], "properties": { "benchmark": { "type": "string", "enum": [ "gsm8k", "mmlu-pro", "gpqa", "arc-challenge", "hellaswag", "winogrande", "openbookqa", "truthfulqa", "commonsenseqa" ], "example": "gsm8k" }, "model": { "type": "string", "example": "anthropic/claude-haiku-4-5", "description": "Any HuggingFace or OpenRouter model id" }, "n": { "type": "integer", "minimum": 1, "maximum": 3, "default": 3, "description": "Sample count (capped at 3 for free tier)" } } } } } }, "responses": { "200": { "description": "Probe completed; signed receipt issued", "content": { "application/json": { "schema": { "type": "object", "properties": { "run_id": { "type": "string" }, "verify_url": { "type": "string", "format": "uri" }, "score": { "type": "number" }, "score_ci": { "type": "object", "properties": { "lower": { "type": "number" }, "upper": { "type": "number" }, "half": { "type": "number" } } }, "n": { "type": "integer" }, "benchmark": { "type": "string" }, "model": { "type": "string" }, "attestor": { "type": "string" }, "signature": { "type": "string" }, "pubkey": { "type": "string" }, "duration_ms": { "type": "integer" }, "upgrade": { "type": "string" } } } } } }, "400": { "description": "Unknown benchmark or invalid body" }, "429": { "description": "Rate limit (1/hour, 5/day per IP)" }, "503": { "description": "Inference pool unavailable" } } }, "get": { "summary": "Probe spec discovery (GET returns the schema)", "security": [], "responses": { "200": { "description": "Schema and example bodies" } } } } }, "components": { "securitySchemes": { "bearerAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "bl_live_." } } }, "tags": [ { "name": "auth", "description": "Signup + key lifecycle" }, { "name": "run", "description": "Queue benchmark runs" }, { "name": "payments", "description": "Card + native ETH flows" } ], "externalDocs": { "url": "https://benchlist.ai/llms.txt", "description": "LLM-agent quickstart" } }