{"version":"1","generated_at":"2026-10-22T07:59:06.487Z","tests":[{"service_id":"svc_official_microsoft_learn","task":"technical_docs","case_id":"microsoft-lifecycle-v1","passed":true,"assertion":"Official Microsoft documentation URL and lifecycle topic returned.","failure":null,"executed_at":"2026-10-22T03:50:08.281Z","latency_ms":3091,"tool":"microsoft_docs_search","arguments":{"query":"Azure Blob Storage lifecycle management"},"fixture_kind":"public_read_only_task","limitations":"One public query on one date. No private data, paid credentials, answer-quality evaluation or reliability percentage."},{"service_id":"svc_official_cloudflare_docs","task":"technical_docs","case_id":"cloudflare-cpu-v1","passed":true,"assertion":"Official Cloudflare documentation URL and CPU topic returned.","failure":null,"executed_at":"2026-10-22T03:50:21.486Z","latency_ms":13196,"tool":"search_cloudflare_documentation","arguments":{"query":"Workers CPU limits"},"fixture_kind":"public_read_only_task","limitations":"One public query on one date. No private data, paid credentials, answer-quality evaluation or reliability percentage."},{"service_id":"svc_official_deepwiki","task":"repository_research","case_id":"sdk-repository-structure-v1","passed":true,"assertion":"A non-empty repository documentation outline returned; not a code correctness test.","failure":null,"executed_at":"2026-10-22T03:50:23.696Z","latency_ms":2207,"tool":"read_wiki_structure","arguments":{"repoName":"modelcontextprotocol/typescript-sdk"},"fixture_kind":"public_read_only_task","limitations":"One public query on one date. No private data, paid credentials, answer-quality evaluation or reliability percentage."},{"service_id":"svc_official_exa","task":"web_research","case_id":"mcp-primary-source-v1","passed":true,"assertion":"Search returned a modelcontextprotocol.io source URL for the MCP tools specification.","failure":null,"executed_at":"2026-10-22T03:50:27.818Z","latency_ms":4121,"tool":"web_search_exa","arguments":{"query":"MCP specification tools structuredContent","objective":"Find the official Model Context Protocol tools specification and return source URLs.","numResults":3},"fixture_kind":"public_read_only_task","limitations":"One public query on one date. No private data, paid credentials, answer-quality evaluation or reliability percentage."},{"case_id":"pdf-text-english-v1","service_id":"svc_evident_pdf_local","task":"pdf_text","passed":true,"assertion":"Extracted exact row text from a selectable-text PDF. English fixture only.","fixture_kind":"generated_public_test_document","executed_at":"2026-10-22T03:50:28.293Z"},{"case_id":"pdf-fixed-columns-v1","service_id":"svc_evident_pdf_local","task":"pdf_tables","passed":true,"assertion":"All 9 cells equal expected text in a simple 3-column fixture with caller-supplied boundaries.","fixture_kind":"generated_public_test_document","executed_at":"2026-10-22T03:50:28.300Z"},{"case_id":"no-text-rejection-v1","service_id":"svc_evident_pdf_local","task":"pdf_text","passed":true,"assertion":"A page without a text layer is marked OCR_REQUIRED, never a successful extraction. This is a rejection test, not OCR capability.","fixture_kind":"generated_public_test_document","executed_at":"2026-10-22T03:50:28.308Z"},{"case_id":"mixed-pages-v1","service_id":"svc_evident_pdf_local","task":"pdf_text","passed":true,"assertion":"Preserved extracted text and flagged the second page without a text layer.","fixture_kind":"generated_public_test_document","executed_at":"2026-10-22T03:50:28.316Z"},{"case_id":"bad-columns-v1","service_id":"svc_evident_pdf_local","task":"pdf_tables","passed":true,"assertion":"Rejected reversed column boundaries.","fixture_kind":"generated_public_test_document","executed_at":"2026-10-22T03:50:28.316Z"},{"service_id":"svc_official_crossref","task":"scholarly_research","case_id":"crossref-public-metadata-v1","passed":true,"assertion":"The public query returns a work with a DOI, title and HTTPS source URL.","failure":null,"executed_at":"2026-10-22T07:59:05.419Z","latency_ms":1109,"tool":"GET public metadata API","arguments":{"url":"https://api.crossref.org/works?query=document%20information%20extraction&rows=2"},"fixture_kind":"public_read_only_task","limitations":"One query checks metadata fields only. No full-text extraction, source quality, general relevance or reliability claim."},{"service_id":"svc_official_arxiv","task":"scholarly_research","case_id":"arxiv-atom-metadata-v1","passed":true,"assertion":"The public query returns an Atom entry with a title and arXiv identifier.","failure":null,"executed_at":"2026-10-22T07:59:06.486Z","latency_ms":1066,"tool":"GET public metadata API","arguments":{"url":"https://export.arxiv.org/api/query?search_query=all%3Adocument%20AND%20all%3Aextraction&start=0&max_results=2"},"fixture_kind":"public_read_only_task","limitations":"One query checks metadata fields only. No full-text extraction, source quality, general relevance or reliability claim."}],"methodology":"Named assertions on public research calls and generated English PDF fixtures. Failing checks remain visible. No general benchmark ranking, customer success rate or scanned-document accuracy claim."}