[ { "slug": "evals-cant-tell-noise", "title": "Most AI benchmarks never check whether a difference is real. Ours couldn't either", "date": "2026-11-02", "stream": "lab", "url": "/blog/evals-cant-tell-noise.html", "summary": "With 40 gold questions, Arbiter's eval could not see any change smaller than about 15 points. A review of 445 benchmark papers found 16% used any statistical test. The power curve, the field's numbers, and the release gate the agent now enforces.\n", "policy_rules": [ "F-EVAL-POWER", "F-STRICT-RECALL", "F-NO-LEAKS" ], "sources": [ "https://arxiv.org/abs/2511.04703", "https://aclanthology.org/2026.acl-long.1996/", "https://arxiv.org/abs/2411.00640", "https://arxiv.org/abs/2608.21382", "https://arxiv.org/abs/2605.24213", "https://arxiv.org/abs/2411.12990", "https://github.com/yoonbo1/arbiterai/blob/main/CHANGELOG.md", "https://www.aisi.gov.uk/blog/more-compute-more-capability-why-ai-agent", "https://aclanthology.org/2020.emnlp-main.745/" ] }, { "slug": "hti-5-model-cards", "title": "HHS just removed the AI \"model card\" rule from certified EHRs. Here is what goes, and what still applies", "date": "2026-11-02", "stream": "field", "url": "/blog/hti-5-model-cards.html", "summary": "ONC's HTI-5 final rule, published 28 October 2026, deletes the 44 source attributes and the risk management that certified EHRs had to provide for decision-support AI. 251 products from 206 developers carried the rule. What goes on 28 December, what stays, and what still governs AI in the EHR.\n", "policy_rules": [ "S1557-INVENTORY", "CA-AB3030-DISCLOSURE" ], "sources": [ "https://www.federalregister.gov/documents/2026/10/28/2026-21943/health-data-technology-and-interoperability-onc-deregulatory-actions-to-unleash-prosperity", "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-D/part-170/subpart-C/section-170.315", "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-A/part-92/subpart-C/section-92.210", "https://www.federalregister.gov/documents/2024/01/09/2023-28857/health-data-technology-and-interoperability-certification-program-updates-algorithm-transparency-and", "https://searchlf.ama-assn.org/letter/documentDownload?uri=/unstructured/binary/letter/LETTERS/lfk.zip/2026-2-27-Letter-to-Keane-re-HDTI-5-NPRM.pdf", "https://www.aha.org/lettercomment/2026-02-27-aha-comments-astponc-health-care-technology-interoperability-proposed-rule", "https://www.ahcancal.org/News-and-Communications/Fact-Sheets/Documents/HTI-5_AHCA_NCAL%20Comments_Final_02.24.2026.pdf", "https://www.fda.gov/medical-devices/medical-devices-news-and-events/town-hall-clinical-decision-support-software-final-guidance-03112026", "https://www.ehra.org/sites/ehra.org/files/2026-02/EHR%20Association%20Comments%20to%20ASTP%20on%20HTI-5%20Proposed%20Rule.pdf", "https://www.ama-assn.org/health-care-advocacy/advocacy-update/oct-30-2026-national-advocacy-update", "https://healthit.gov/regulations/hti-rules/hti-5-final-rule/" ] }, { "slug": "ios-27-core-ai-on-device-privacy", "title": "iOS 27 can run open AI models entirely on your iPhone. What that does, and doesn't, mean for privacy", "date": "2026-11-02", "stream": "field", "url": "/blog/ios-27-core-ai-on-device-privacy.html", "summary": "Apple's Core AI, shipped with iOS 27 on 14 September 2026, lets apps run open models such as Qwen, Gemma, Mistral and Whisper on the device. Prompts no longer have to reach a server to be processed. Three things it does not settle, and the memory arithmetic that decides which models fit.\n", "policy_rules": [ "HIPAA-ENDPOINTS" ], "sources": [ "https://machinelearning.apple.com/research/apple-foundation-models-2025-updates", "https://developer.apple.com/documentation/coreai", "https://github.com/apple/coreai-models", "https://9to5mac.com/2026/09/09/apple-confirms-ios-27-release-date-september-14/", "https://www.infoq.com/news/2026/06/apple-core-ai-wwdc/", "https://github.com/a-ghorbani/pocketpal-ai", "https://www.arbiterai.tech/blog/7b-judge-citation-placement.html" ] }, { "slug": "jan-1-2027-ai-privacy-laws", "title": "46 AI and privacy duties start on 1 January 2027. The ones that matter if you build on health data", "date": "2026-11-02", "stream": "field", "url": "/blog/jan-1-2027-ai-privacy-laws.html", "summary": "Arbiter's law tracker now holds 95 AI and privacy duties starting before the end of 2027, each date read in the primary text. 46 start on 1 January 2027, 20 of them in California. For health AI, the federal \"model card\" rule ends on 28 December and three state laws take over the same week.\n", "policy_rules": [ "S1557-INVENTORY", "NO-AUTOMATED-DETERMINATIONS", "CA-AB3030-DISCLOSURE" ], "sources": [ "https://www.federalregister.gov/documents/2026/10/28/2026-21943/health-data-technology-and-interoperability-onc-deregulatory-actions-to-unleash-prosperity", "https://leginfo.legislature.ca.gov/faces/billTextClient.xhtml?bill_id=202520260AB1979", "https://leginfo.legislature.ca.gov/faces/billTextClient.xhtml?bill_id=202520260SB503", "https://leg.colorado.gov/laws/session-laws/HB26-1139/325/download", "https://cppa.ca.gov/regulations/pdf/ccpa_updates_cyber_risk_admt_appr_text.pdf", "https://leg.colorado.gov/bill_files/116489/download", "https://www.arbiterai.tech/data/law-dates.csv", "https://eur-lex.europa.eu/eli/reg/2026/1744/oj/eng", "https://www.nysenate.gov/legislation/bills/2025/S8828", "https://www.ilga.gov/Documents/legislation/publicacts/104/PDF/104-0538.pdf", "https://leginfo.legislature.ca.gov/faces/billTextClient.xhtml?bill_id=202520260SB1119", "https://leginfo.legislature.ca.gov/faces/billTextClient.xhtml?bill_id=202520260AB1159" ] }, { "slug": "judge-cant-verify-no", "title": "Our safety check couldn't verify a \"no\". AI judges everywhere share the blind spot", "date": "2026-11-02", "stream": "lab", "url": "/blog/judge-cant-verify-no.html", "summary": "The check that decides whether Arbiter's answers are grounded scored correct negations 0.00, the same as fabrications. Re-measured today and fixed by restructuring the task, not by changing the model, alongside 2026 research showing AI judges miss what is absent.\n", "policy_rules": [ "F-NEGATION", "CITATIONS-REQUIRED" ], "sources": [ "https://github.com/yoonbo1/arbiterai/blob/main/CHANGELOG.md", "https://www.arbiterai.tech/data/judge-cant-verify-no/cases.csv", "https://arxiv.org/abs/2608.31016", "https://aclanthology.org/2026.acl-long.1472/", "https://arxiv.org/abs/2506.11440", "https://arxiv.org/abs/2606.07834", "https://arxiv.org/abs/2604.16383" ] }, { "slug": "privacy-agent-policy-as-code", "title": "We built an AI agent that follows HIPAA in code. Here is every rule, and what it refused", "date": "2026-11-02", "stream": "lab", "url": "/blog/privacy-agent-policy-as-code.html", "summary": "Arbiter now has an agent that answers questions about a patient's records under 23 written rules, each tied to a HIPAA section, a state law or one of our own measured failures. It passed 8 of 8 policy scenarios on a local model. The architecture, the refusals, and what code alone cannot make compliant.\n", "policy_rules": [ "HIPAA-PURPOSE", "HIPAA-PSYCHOTHERAPY-NOTES", "PART2-SUD-RECORDS", "CA-AB3030-DISCLOSURE", "NO-AUTOMATED-DETERMINATIONS", "F-NEGATION", "F-EVAL-POWER" ], "sources": [ "https://www.arbiterai.tech/data/privacy-agent-policy-as-code/rules.csv", "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164", "https://www.ecfr.gov/current/title-42/chapter-I/subchapter-A/part-2/subpart-C/section-2.31", "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-A/part-92/subpart-C/section-92.210", "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202320240AB3030", "https://www.arbiterai.tech/data/privacy-agent-policy-as-code/scenarios.csv", "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202320240SB1120" ] } ]