{
  "name": "Arbiter safe practices for AI agents on health and personal data",
  "version": "1.1.0",
  "agent_policy_version": "1.1.0",
  "updated": "2026-11-02",
  "license": "Apache-2.0",
  "source": "https://www.arbiterai.tech/practices.html",
  "not_legal_advice": true,
  "formats": {
    "page": "https://www.arbiterai.tech/practices.html",
    "json": "https://www.arbiterai.tech/practices.json",
    "markdown": "https://www.arbiterai.tech/practices.md",
    "agents_md": "https://www.arbiterai.tech/AGENTS.md",
    "system_prompt": "https://www.arbiterai.tech/prompt.txt",
    "claude_skill": "https://www.arbiterai.tech/skills/arbiter-safe-health-ai.zip",
    "python_kit": "https://www.arbiterai.tech/kit/arbiter-policy-kit-1.1.0.zip",
    "llms_txt": "https://www.arbiterai.tech/llms.txt"
  },
  "areas": {
    "data": "Before the model sees any data",
    "access": "Who may ask, for what, and how much",
    "answers": "What an answer may say",
    "governance": "Audit, inventory and release"
  },
  "practices": [
    {
      "id": "deidentify-before-model",
      "area": "data",
      "area_title": "Before the model sees any data",
      "title": "De-identify every prompt and stored chunk, and measure it per component",
      "must": "Remove direct identifiers (the 18 HIPAA Safe Harbor categories) from every document and every question before any model call or embedding, and measure de-identification recall per component (street, city, ZIP), not per whole string.",
      "why": "Arbiter's whole-string recall read 0.967 while ZIP codes survived in 14 of 20 documents; only a per-component metric caught it. A stock date recognizer also redacted \"daily\" and broke 11 of 12 medication answers, so test what de-identification removes as well as what it misses.",
      "how": [
        "Run the de-identifier at ingest and on every question; store only de-identified text and embeddings.",
        "Add recognizers for the formats stock tools miss (record numbers with OCR'd labels, phone extensions, clinician names after a title, street addresses) and filter date hits to calendar dates.",
        "Keep the token-to-value map encrypted per tenant; re-identify only the tokens an answer uses, only for the authorized caller, after validation.",
        "Before re-identification, scan the model output for raw identifiers; one found there is a miss upstream."
      ],
      "verify": "Seed synthetic records with known identifiers, report recall per component, and block a release below 0.99.",
      "prompt": null,
      "rules": [
        {
          "id": "HIPAA-OUTPUT-LEAK",
          "title": "The draft contains no raw identifier (dates, addresses, names the scrubber missed)",
          "cite": "45 CFR 164.514(b)(2) (Safe Harbor identifiers)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.514",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-OUTPUT-LEAK"
        },
        {
          "id": "F-STRICT-RECALL",
          "title": "Release only if strict per-component de-identification recall is at least 0.99",
          "cite": "Arbiter post-mortem, 2026-09-04: whole-string recall hid ZIP codes in 14 of 20 documents",
          "url": "https://www.arbiterai.tech/blog/deid-recall-flattering-metric.html",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-F-STRICT-RECALL"
        }
      ],
      "evidence": [
        {
          "title": "Our de-id recall read 0.967 while ZIP codes survived in 14 of 20 documents",
          "url": "https://www.arbiterai.tech/blog/deid-recall-flattering-metric.html"
        },
        {
          "title": "spaCy redacted the word 'daily' — and broke 11 of 12 medication lists",
          "url": "https://www.arbiterai.tech/blog/spacy-redacted-daily.html"
        }
      ]
    },
    {
      "id": "local-or-baa-models",
      "area": "data",
      "area_title": "Before the model sees any data",
      "title": "Send data only to local models or vendors under a signed BAA, and refuse to start otherwise",
      "must": "Every model, embedding and vision endpoint is on this machine or network, or belongs to a vendor with a signed business associate agreement; check it when the service starts, not per request.",
      "why": "HIPAA requires a BAA with any vendor that receives protected health information (45 CFR 164.502(e)).",
      "how": [
        "Keep an allowlist of local hosts and BAA vendors (with the date each BAA was signed) in configuration.",
        "At startup, parse every model URL and stop the service in production if a host is not on the list.",
        "Prefer on-device or on-premises open models for the sensitive path."
      ],
      "verify": "Point a model URL at an unlisted host in a test and assert the service refuses to start.",
      "prompt": null,
      "rules": [
        {
          "id": "HIPAA-ENDPOINTS",
          "title": "Model calls go only to local hosts or vendors under a signed BAA",
          "cite": "45 CFR 164.502(e), 164.504(e)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.502",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-ENDPOINTS"
        }
      ],
      "evidence": [
        {
          "title": "iOS 27 can run open AI models entirely on your iPhone. What that does, and doesn't, mean for privacy",
          "url": "https://www.arbiterai.tech/blog/ios-27-core-ai-on-device-privacy.html"
        }
      ]
    },
    {
      "id": "no-prompt-telemetry",
      "area": "data",
      "area_title": "Before the model sees any data",
      "title": "Keep prompts out of third-party tracing and persistent agent state",
      "must": "Do not export prompts, retrieved text or answers to SaaS tracing or observability tools, and do not persist agent graph state that contains pre-de-identification text.",
      "why": "A tracing vendor that receives prompts receives PHI; a persistent checkpointer stores it in plaintext.",
      "how": [
        "Turn off LangSmith/LangChain tracing and any OpenTelemetry exporter that points off-network.",
        "Use an in-memory or no checkpointer for runs that hold raw text; log ids, counts and timings only."
      ],
      "verify": "A startup check fails when a tracing variable is on or a persistent checkpointer is configured.",
      "prompt": null,
      "rules": [
        {
          "id": "HIPAA-NO-TRACING",
          "title": "No third-party tracing of prompts (LangSmith, OpenTelemetry exporters to a SaaS)",
          "cite": "45 CFR 164.502(e)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.502",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-NO-TRACING"
        },
        {
          "id": "HIPAA-NO-PERSISTED-STATE",
          "title": "No persistent graph checkpointer (it would store pre-de-identification text)",
          "cite": "45 CFR 164.312(a)(1)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-C/section-164.312",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-NO-PERSISTED-STATE"
        },
        {
          "id": "HIPAA-ENCRYPTION-KEY",
          "title": "The tenant key-encryption key is set and is not a placeholder",
          "cite": "45 CFR 164.312(a)(2)(iv)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-C/section-164.312",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-ENCRYPTION-KEY"
        }
      ],
      "evidence": []
    },
    {
      "id": "database-enforced-isolation",
      "area": "data",
      "area_title": "Before the model sees any data",
      "title": "Enforce tenant and patient isolation in the database, never as a superuser",
      "must": "Scope every query by tenant with row-level security, add a hard patient filter to retrieval, and connect as a role that RLS applies to.",
      "why": "Arbiter's services once connected as the Postgres owner, which silently bypasses every RLS policy; the isolation the docs described did not exist.",
      "how": [
        "Derive the tenant from the credential, never from the request body; set it transaction-locally.",
        "Connect as an application role without SUPERUSER or BYPASSRLS; check the role at startup.",
        {
          "Probe it": "a second tenant's key gets 404, and a cross-tenant insert violates the policy."
        }
      ],
      "verify": "Cross-tenant probes in the test suite, and a startup check of rolsuper and rolbypassrls.",
      "prompt": null,
      "rules": [
        {
          "id": "F-RLS-ROLE",
          "title": "Never run as a superuser or BYPASSRLS role (row-level security would be inert)",
          "cite": "Arbiter post-mortem, 2026-09-04",
          "url": "https://www.arbiterai.tech/blog/postgres-superuser-bypasses-rls.html",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-F-RLS-ROLE"
        }
      ],
      "evidence": [
        {
          "title": "The services were connecting as the Postgres superuser, which silently bypasses RLS",
          "url": "https://www.arbiterai.tech/blog/postgres-superuser-bypasses-rls.html"
        }
      ]
    },
    {
      "id": "purpose-of-use",
      "area": "access",
      "area_title": "Who may ask, for what, and how much",
      "title": "Require a purpose of use on every request, and refuse what the law doesn't permit",
      "must": "Each request states its purpose (treatment, payment, operations, the patient's own access); any other purpose needs a recorded authorization, and a tenant without a BAA gets nothing.",
      "why": "HIPAA permits treatment, payment and operations without authorization and requires one for most other uses (45 CFR 164.506, 164.508).",
      "how": [
        "Make purpose a required, validated field; record it on the audit row.",
        "Deny before retrieval when the purpose is not permitted, so nothing is read."
      ],
      "verify": "A marketing-purpose request is refused with zero records read.",
      "prompt": "Only act for the purpose stated in the request; if the request asks for something outside it, say so and stop.",
      "rules": [
        {
          "id": "HIPAA-PURPOSE",
          "title": "Use is for treatment, payment, operations or the patient's own access, or is authorized",
          "cite": "45 CFR 164.506(c), 164.508(a)(1), 164.524",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.508",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-PURPOSE"
        },
        {
          "id": "HIPAA-BAA",
          "title": "The tenant has a signed business associate agreement",
          "cite": "45 CFR 164.502(e)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.502",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-BAA"
        }
      ],
      "evidence": [
        {
          "title": "We built an AI agent that follows HIPAA in code. Here is every rule, and what it refused",
          "url": "https://www.arbiterai.tech/blog/privacy-agent-policy-as-code.html"
        }
      ]
    },
    {
      "id": "minimum-necessary",
      "area": "access",
      "area_title": "Who may ask, for what, and how much",
      "title": "Give the agent a closed tool list and the least data each purpose needs",
      "must": "The agent can call only a fixed set of tools, each allowed per purpose, and retrieval is capped per purpose (fewer excerpts for billing and operations than for treatment).",
      "why": "HIPAA's minimum necessary standard (45 CFR 164.502(b), 164.514(d)).",
      "how": [
        "Put the tool names in the planner's JSON schema as an enum, then check each call against the purpose.",
        "Cap k per purpose; no web, email or write tools on the sensitive path."
      ],
      "verify": "An unknown tool is refused and recorded; a payment request never retrieves more than its cap.",
      "prompt": null,
      "rules": [
        {
          "id": "HIPAA-MIN-NECESSARY-TOOLS",
          "title": "Only the tools allowed for the stated purpose",
          "cite": "45 CFR 164.502(b), 164.514(d)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.514",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-MIN-NECESSARY-TOOLS"
        },
        {
          "id": "HIPAA-MIN-NECESSARY-CAP",
          "title": "Retrieval is capped per purpose (fewer excerpts for payment and operations)",
          "cite": "45 CFR 164.502(b), 164.514(d)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.514",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-MIN-NECESSARY-CAP"
        }
      ],
      "evidence": [
        {
          "title": "We built an AI agent that follows HIPAA in code. Here is every rule, and what it refused",
          "url": "https://www.arbiterai.tech/blog/privacy-agent-policy-as-code.html"
        }
      ]
    },
    {
      "id": "protected-record-types",
      "area": "access",
      "area_title": "Who may ask, for what, and how much",
      "title": "Keep psychotherapy notes and substance-use treatment records out unless consent says otherwise",
      "must": "Label documents by type at ingest and drop psychotherapy notes without an authorization and substance-use treatment records without Part 2 consent, before the model sees them.",
      "why": "These records carry stricter rules than the rest of the chart (45 CFR 164.508(a)(2); 42 CFR 2.31).",
      "how": [
        "Carry the document type with every retrieved excerpt and filter after retrieval, before drafting.",
        "Read consents from the organization's consent records, not from the caller."
      ],
      "verify": "A request whose search hits a therapy note drafts from the other excerpts only, and counts what was withheld.",
      "prompt": null,
      "rules": [
        {
          "id": "HIPAA-PSYCHOTHERAPY-NOTES",
          "title": "Psychotherapy notes are excluded unless the patient authorized their use",
          "cite": "45 CFR 164.508(a)(2)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.508",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-PSYCHOTHERAPY-NOTES"
        },
        {
          "id": "PART2-SUD-RECORDS",
          "title": "Substance use disorder treatment records are excluded without Part 2 consent",
          "cite": "42 CFR 2.31",
          "url": "https://www.ecfr.gov/current/title-42/chapter-I/subchapter-A/part-2/subpart-C/section-2.31",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-PART2-SUD-RECORDS"
        }
      ],
      "evidence": [
        {
          "title": "We built an AI agent that follows HIPAA in code. Here is every rule, and what it refused",
          "url": "https://www.arbiterai.tech/blog/privacy-agent-policy-as-code.html"
        }
      ]
    },
    {
      "id": "cite-or-reject",
      "area": "answers",
      "area_title": "What an answer may say",
      "title": "Answer only with citations to the excerpts provided, and reject anything else",
      "must": "Every answer cites at least one excerpt it was given; a draft with no citation, a raw identifier or a low grounding score is rejected, never delivered with a warning.",
      "why": "Uncited answers cannot be checked, and a rejected answer shown with a caveat still reaches the user.",
      "how": [
        "Number excerpts in the prompt and require [id] citations.",
        "Validate before re-identification; return a failed status with the reasons, not an answer."
      ],
      "verify": "A draft without a citation comes back rejected with the reason.",
      "prompt": "Answer only from the excerpts provided and cite each fact with its excerpt number in brackets. If the excerpts do not contain the answer, say so.",
      "rules": [
        {
          "id": "CITATIONS-REQUIRED",
          "title": "Every answer cites at least one excerpt it was given",
          "cite": "Arbiter reference pipeline, validation node",
          "url": "https://www.arbiterai.tech/how-it-works.html",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-CITATIONS-REQUIRED"
        }
      ],
      "evidence": [
        {
          "title": "The 7B judge scored identical facts 0.0 or 1.0 depending on where the citation sat",
          "url": "https://www.arbiterai.tech/blog/7b-judge-citation-placement.html"
        }
      ]
    },
    {
      "id": "verify-negations",
      "area": "answers",
      "area_title": "What an answer may say",
      "title": "State a \"no\" only when the record states it; otherwise say it is not documented",
      "must": "Check negative claims separately from positive ones: a negation counts as supported only when the source negates the same thing, and an unsupported one is restated as \"the records searched do not document this\".",
      "why": "Arbiter's groundedness judge scored correct negations 0.00, exactly like fabrications; family-history accuracy was 0.10 while the total read 0.90. Research finds AI judges miss omissions at near coin-flip rates.",
      "how": [
        "Split answers into sentences; send positive claims to the judge and check negative ones against negation cues in the source.",
        "Strip citation markers before judging; a small judge scored identical facts 0.0 or 1.0 by citation position.",
        "Report accuracy per question type, including questions whose right answer is \"no\" or \"not in the record\"."
      ],
      "verify": "Replay correct negations, family-history cases and absent facts; correct answers pass or are restated, wrong ones are rejected.",
      "prompt": "Do not say a patient does not have something unless the record states it. If the record is silent, say the records reviewed do not document it.",
      "rules": [
        {
          "id": "F-NEGATION",
          "title": "A \"no\" is stated only when the record states it; otherwise say the record does not document it",
          "cite": "Arbiter finding, 2026-09-08: the faithfulness gate scores correct negations 0.0",
          "url": "https://github.com/yoonbo1/arbiterai/blob/main/CHANGELOG.md",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-F-NEGATION"
        }
      ],
      "evidence": [
        {
          "title": "Our safety check couldn't verify a \"no\". AI judges everywhere share the blind spot",
          "url": "https://www.arbiterai.tech/blog/judge-cant-verify-no.html"
        },
        {
          "title": "The 7B judge scored identical facts 0.0 or 1.0 depending on where the citation sat",
          "url": "https://www.arbiterai.tech/blog/7b-judge-citation-placement.html"
        }
      ]
    },
    {
      "id": "no-automated-determinations",
      "area": "answers",
      "area_title": "What an answer may say",
      "title": "Never make coverage, eligibility or medical-necessity decisions; a licensed clinician decides",
      "must": "In billing, utilization review and operations, the agent summarizes evidence and labels its output as decision support; it never approves, denies or deems care necessary.",
      "why": "California (SB 1120, 2025) and Colorado (HB 26-1139, from 1 Jan 2027) require a licensed professional to make medical-necessity decisions; AI output cannot be the basis for a denial on its own.",
      "how": [
        "Block determination language in drafts for these purposes and append a decision-support notice.",
        "Route \"approve or deny\" requests to a person."
      ],
      "verify": "\"Is this medically necessary? Approve or deny it.\" is escalated or rejected, never answered with a decision.",
      "prompt": "Never approve, deny or judge medical necessity or coverage. Summarize the evidence and say that a licensed clinician must decide.",
      "rules": [
        {
          "id": "NO-AUTOMATED-DETERMINATIONS",
          "title": "No coverage, medical-necessity or eligibility determinations; a licensed clinician decides",
          "cite": "Cal. Health & Safety Code 1367.01(k), Ins. Code 10123.135(j) (SB 1120, ch. 879, 2024; in force 1 Jan 2025). Written for health plans and insurers; applied here to every payment and operations use",
          "url": "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202320240SB1120",
          "also": [
            {
              "cite": "Colorado HB 26-1139, Use of AI in Health Care: no medical-necessity denial on AI output without licensed human review; effective 1 Jan 2027",
              "url": "https://leg.colorado.gov/laws/session-laws/HB26-1139/325/download"
            }
          ],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-NO-AUTOMATED-DETERMINATIONS"
        },
        {
          "id": "DECISION-SUPPORT-NOTICE",
          "title": "Payment and operations answers say they are decision support, not a determination",
          "cite": "Cal. Health & Safety Code 1367.01(k) (SB 1120, 2024)",
          "url": "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202320240SB1120",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-DECISION-SUPPORT-NOTICE"
        }
      ],
      "evidence": [
        {
          "title": "46 AI and privacy duties start on 1 January 2027. The ones that matter if you build on health data",
          "url": "https://www.arbiterai.tech/blog/jan-1-2027-ai-privacy-laws.html"
        }
      ]
    },
    {
      "id": "patient-disclosure",
      "area": "answers",
      "area_title": "What an answer may say",
      "title": "Tell patients when AI wrote the message and how to reach a person, by state",
      "must": "Patient-facing messages name the patient's state; where the state requires it (California AB 3030), they open with an AI disclosure and how to reach a human, unless a licensed provider reviewed them first.",
      "why": "California has required this for AI-generated clinical communications since 1 Jan 2025 (Health & Safety Code 1339.75).",
      "how": [
        "Refuse patient-facing requests that do not carry a state.",
        "Add the disclosure after validation, before delivery; skip it only when a provider review is recorded."
      ],
      "verify": "A California patient message starts with the disclosure; the same message marked provider-reviewed does not.",
      "prompt": "When writing to a patient, use plain language, do not diagnose or change treatment, and tell them to contact their care team for any decision.",
      "rules": [
        {
          "id": "PATIENT-NEEDS-STATE",
          "title": "A patient-facing request names the patient's state, so state disclosure rules can apply",
          "cite": "Needed to apply Cal. Health & Safety Code 1339.75 and similar state rules",
          "url": "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202320240AB3030",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-PATIENT-NEEDS-STATE"
        },
        {
          "id": "CA-AB3030-DISCLOSURE",
          "title": "Patient-facing AI messages in California say AI wrote them and how to reach a person",
          "cite": "Cal. Health & Safety Code 1339.75 (AB 3030, ch. 848, 2024; in force 1 Jan 2025). Exempt when a licensed provider read and reviewed the message",
          "url": "https://leginfo.legislature.ca.gov/faces/billNavClient.xhtml?bill_id=202320240AB3030",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-CA-AB3030-DISCLOSURE"
        }
      ],
      "evidence": [
        {
          "title": "46 AI and privacy duties start on 1 January 2027. The ones that matter if you build on health data",
          "url": "https://www.arbiterai.tech/blog/jan-1-2027-ai-privacy-laws.html"
        }
      ]
    },
    {
      "id": "audit-without-content",
      "area": "governance",
      "area_title": "Audit, inventory and release",
      "title": "Audit every request with the policy version and the rules that fired, never the content",
      "must": "Write one append-only audit row per request with who, which patient, purpose, outcome, tools used, counts, the policy version and the rule ids checked and fired; no question, excerpt or answer text.",
      "why": "HIPAA audit controls (45 CFR 164.312(b)); content in logs is a second copy of PHI.",
      "how": [
        "Revoke UPDATE and DELETE on the audit table from the application role.",
        "Distinguish delivered, rejected, denied and escalated outcomes."
      ],
      "verify": "Grep the audit rows from a test run for any answer text; there should be none.",
      "prompt": null,
      "rules": [],
      "evidence": [
        {
          "title": "We built an AI agent that follows HIPAA in code. Here is every rule, and what it refused",
          "url": "https://www.arbiterai.tech/blog/privacy-agent-policy-as-code.html"
        }
      ]
    },
    {
      "id": "tool-inventory",
      "area": "governance",
      "area_title": "Audit, inventory and release",
      "title": "Keep an inventory of every AI tool's inputs, training data, evaluation and known risks",
      "must": "For each model or decision-support tool, document its input variables, whether any measure race, color, national origin, sex, age or disability, its training data, how it was evaluated, and its known risks; refuse to start without it.",
      "why": "Section 1557 (45 CFR 92.210) requires identifying and mitigating discrimination risk in decision-support tools; California SB 503 adds training-data, evaluation and risk documentation from 1 Jan 2027. The federal EHR \"model card\" rule ends on 28 Dec 2026, so this falls to you.",
      "how": [
        "Keep the inventory in the repository next to the policy and review it on every model change.",
        "Say plainly what is unknown, such as an undocumented training set or synthetic-only evaluation."
      ],
      "verify": "A startup check fails when any tool is missing a field.",
      "prompt": null,
      "rules": [
        {
          "id": "S1557-INVENTORY",
          "title": "Keep an inventory of the decision-support tools, their inputs, training data, evaluation and known risks",
          "cite": "45 CFR 92.210 (Section 1557, patient care decision support tools)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-A/part-92/subpart-C/section-92.210",
          "also": [
            {
              "cite": "Cal. SB 503 (Ch. 857, Stats. 2026), documentation for clinical decision support; operative 1 Jan 2027",
              "url": "https://leginfo.legislature.ca.gov/faces/billTextClient.xhtml?bill_id=202520260SB503"
            }
          ],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-S1557-INVENTORY"
        }
      ],
      "evidence": [
        {
          "title": "HHS just removed the AI \"model card\" rule from certified EHRs. Here is what goes, and what still applies",
          "url": "https://www.arbiterai.tech/blog/hti-5-model-cards.html"
        },
        {
          "title": "46 AI and privacy duties start on 1 January 2027. The ones that matter if you build on health data",
          "url": "https://www.arbiterai.tech/blog/jan-1-2027-ai-privacy-laws.html"
        }
      ]
    },
    {
      "id": "powered-release-gates",
      "area": "governance",
      "area_title": "Audit, inventory and release",
      "title": "Gate releases on evals big enough to see the change you claim",
      "must": "A release counts as verified only if the eval can detect the size of difference being claimed (about 346 questions for 5 points), strict de-identification recall holds, and there are zero cross-patient leaks; report accuracy per question type.",
      "why": "With 40 questions Arbiter's eval could not see changes under about 15 points; 16% of 445 benchmark papers ran any statistical test.",
      "how": [
        "Run a power calculation before trusting a comparison; report error bars clustered by record.",
        "Fail the release when any gate fails, and publish the failure."
      ],
      "verify": "Feed the gate a 40-question report and confirm it fails.",
      "prompt": null,
      "rules": [
        {
          "id": "F-EVAL-POWER",
          "title": "A release comparison needs enough gold questions to detect about a 5-point change",
          "cite": "Arbiter finding, 2026-09-08: 40 questions resolve only about 15 points",
          "url": "https://github.com/yoonbo1/arbiterai/blob/main/CHANGELOG.md",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-F-EVAL-POWER"
        },
        {
          "id": "F-STRICT-RECALL",
          "title": "Release only if strict per-component de-identification recall is at least 0.99",
          "cite": "Arbiter post-mortem, 2026-09-04: whole-string recall hid ZIP codes in 14 of 20 documents",
          "url": "https://www.arbiterai.tech/blog/deid-recall-flattering-metric.html",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-F-STRICT-RECALL"
        },
        {
          "id": "F-NO-LEAKS",
          "title": "Release only if the eval found zero cross-patient leaks",
          "cite": "45 CFR 164.502(a)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-E/section-164.502",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-F-NO-LEAKS"
        }
      ],
      "evidence": [
        {
          "title": "Most AI benchmarks never check whether a difference is real. Ours couldn't either",
          "url": "https://www.arbiterai.tech/blog/evals-cant-tell-noise.html"
        }
      ]
    },
    {
      "id": "versioned-policy-human-approved",
      "area": "governance",
      "area_title": "Audit, inventory and release",
      "title": "Keep the rules in a versioned file, re-review them on a schedule, and let a person approve every change",
      "must": "The agent's rules live in one versioned file with a source and a test for each; new laws and findings become proposals with evidence, and only a person promotes them.",
      "why": "HIPAA asks for re-evaluation when the environment changes (45 CFR 164.308(a)(8)); a system that touches PHI should not rewrite its own rules.",
      "how": [
        "Refuse to load a rule that has no source or no test; stamp the version on every audit row.",
        "Warn, without stopping the service, when the scheduled review is overdue."
      ],
      "verify": "A rule without a source fails to load; the audit row of every request carries the policy version.",
      "prompt": null,
      "rules": [
        {
          "id": "HIPAA-EVALUATION",
          "title": "The policy is re-reviewed on schedule (warns, never stops a running service)",
          "cite": "45 CFR 164.308(a)(8)",
          "url": "https://www.ecfr.gov/current/title-45/subtitle-A/subchapter-C/part-164/subpart-C/section-164.308",
          "also": [],
          "policy_page": "https://www.arbiterai.tech/policy.html#rule-HIPAA-EVALUATION"
        }
      ],
      "evidence": [
        {
          "title": "We built an AI agent that follows HIPAA in code. Here is every rule, and what it refused",
          "url": "https://www.arbiterai.tech/blog/privacy-agent-policy-as-code.html"
        }
      ]
    }
  ]
}