{
  "addendum_to_run_id": "banking-docs-c9aef3140970",
  "run_id": "banking-values-a651d652e76a",
  "frozen_at": "2026-10-11T03:44:47.379784+00:00",
  "authority": "Founder rulings D12 and D23, 2026-10-10 (portfolio/elevation/ELEVATION_PLAN.md).",
  "purpose": "Test how much of the documentation gain a bare schema recovers when it is also given the distinct values of its category columns (direction, event_kind, status and the other text columns), and nothing else.",
  "treatment": {
    "arm": "values",
    "rule": "Every VARCHAR column in evals/schema.json whose name does not end in \"_id\": its distinct values, read from the evaluation database and sorted, with null appended when the column has NULLs.",
    "message_header": "Distinct values of every non-identifier text column in the database (complete lists, sorted; JSON null means the column has NULLs):\n",
    "domains_file": "documentation/value-domains.json",
    "domains": {
      "account_daily": {
        "currency": [
          "CAD",
          "USD"
        ]
      },
      "accounts": {
        "currency": [
          "CAD",
          "USD"
        ],
        "account_status": [
          "closed",
          "open"
        ]
      },
      "card_spend_daily": {
        "currency": [
          "CAD",
          "USD"
        ]
      },
      "cards": {
        "card_status": [
          "active",
          "cancelled",
          "frozen"
        ]
      },
      "categories": {
        "category_name": [
          "Meals",
          "Software",
          "Supplies",
          "Travel"
        ],
        "reporting_group": [
          "Operations",
          "Technology"
        ]
      },
      "transactions": {
        "event_kind": [
          "deposit",
          "purchase",
          "refund",
          "transfer",
          "withdrawal"
        ],
        "status": [
          "pending",
          "posted",
          "void"
        ],
        "direction": [
          "credit",
          "debit"
        ],
        "currency": [
          "CAD",
          "USD"
        ]
      }
    }
  },
  "value_domains_message_sha256": "9448cb1ccd46df9f9a6cca2d97130ad8220d907f8cf599f007c98341f1932ca0",
  "fixed": [
    "Model gpt-4.1-mini-2025-04-14; temperature 0; seed 7; 800-token output ceiling; json_object response format.",
    "The 40 frozen final questions, the common system message (protocol, allowed functions, physical schema) and the user message.",
    "Scoring: the frozen rubric in banking/scoring.py with the corrected validator in banking/correction.py, the same gold rows, exact integer comparison and the paired bootstrap (10,000 replicates, seed 7).",
    "The bare and documented arms are not re-collected; their run 1 responses are re-scored with the same corrected validator."
  ],
  "budget": {
    "cap_usd": 0.5,
    "project_budget_usd": 10,
    "spend_dir": "_private/spend-values",
    "rule": "Worst-case reservations for this arm, including failures and retries, may not exceed the cap; collection stops before any request that would exceed it."
  },
  "collection": "Questions in file order, one request each, no automatic retries, failures retained as outcomes. No preflight: the response format is unchanged from run 1.",
  "primary_result": "The corrected two-arm comparison (bare versus documented, run 1 responses) is the headline. The values arm is a declared added arm reported beside it.",
  "analysis": [
    "Correct answers per arm (all 40, analytical 36, clarification 4).",
    "Paired differences documented minus bare, values minus bare and documented minus values, each with the question-level paired percentile bootstrap (10,000 replicates, seed 7).",
    "Twin-clustered bootstrap beside it: questions whose text is identical after replacing CAD and USD with one placeholder form one cluster; every other question is its own cluster. Added after run 1 results were seen.",
    "Failure causes per incorrect analytical outcome: mechanical flags from the stored SQL plus hand labels recorded in evals/failure-labels.json after outcomes are seen."
  ],
  "outputs": {
    "cache": "cache-values/",
    "receipt": "evals/cache-manifest-values.json",
    "results": "results-three-arm.json"
  },
  "price_capture": {
    "path": "_private/research/openai-model-values-capture.txt",
    "sha256": "78cab23e43e747d199df10854f27a0bc8ebe1af2bdd66642fe34462e1f680537"
  },
  "questions_sha256": "044b9209a7b36866a74fa22aee75010bacd682193a38f4a5b47f96439da2242f",
  "data_hash": "f937f3c0db0b2037d97e2fe000b6522daba5e9f597c30e70e8d8d7c8a9f2da67",
  "config": {
    "model": "gpt-4.1-mini-2025-04-14",
    "settings": {
      "temperature": 0,
      "seed": 7,
      "max_tokens": 800,
      "response_format": {
        "type": "json_object"
      }
    },
    "prices_usd_per_million": {
      "input": 0.4,
      "output": 1.6,
      "cached_input": 0.1
    },
    "price_source": "https://developers.openai.com/api/docs/models/gpt-4.1-mini.md",
    "price_checked_at": "2026-10-11T03:43:14+00:00",
    "price_capture": "_private/research/openai-model-values-capture.txt",
    "price_capture_sha256": "78cab23e43e747d199df10854f27a0bc8ebe1af2bdd66642fe34462e1f680537",
    "budget_usd": 0.5,
    "project_budget_usd": 10,
    "retry_policy": "No automatic retries. Failed or interrupted requests keep their full cost reservation and outcome."
  },
  "records": [
    {
      "question_id": "B01",
      "arm": "values",
      "cache_key": "9d230c29b83e16ea5b9887ff910e6387c7b726a0e6861426ebe5c57325f46f94"
    },
    {
      "question_id": "B02",
      "arm": "values",
      "cache_key": "214db06d0368e698ffb715cc0246223650217fec23a751404b21511289bcfcd2"
    },
    {
      "question_id": "B03",
      "arm": "values",
      "cache_key": "4707a7ba255a899740eee904666ef6c73451c6ff5b11c82cd82d6a69b0dc155b"
    },
    {
      "question_id": "B04",
      "arm": "values",
      "cache_key": "f136c543dc2401fbfee75050b93f347257b8a2f9c5c9db7c916bef1c053a9273"
    },
    {
      "question_id": "B05",
      "arm": "values",
      "cache_key": "3a18c8ab7b6f0e5de72555e49c2c29a4a9bd83563e2342062b9774c55a3e3986"
    },
    {
      "question_id": "B06",
      "arm": "values",
      "cache_key": "da008c1ad3e89f530ceebf31f9a546847e6cc6f3c487b5c8d06b13d22bc8d69c"
    },
    {
      "question_id": "B07",
      "arm": "values",
      "cache_key": "bfe179abb32bcda8771ce02bdb29e72a2986fdbd1c54e72f23769c2ed3ff3b45"
    },
    {
      "question_id": "B08",
      "arm": "values",
      "cache_key": "1c431ab461c7bacbab95e25bf797a2bcaa2f2374bcab905f5f6ded4feb46a8f6"
    },
    {
      "question_id": "B09",
      "arm": "values",
      "cache_key": "c4ce31dadaf0822db14feecc1138988cb137c0162fe276ffc4bde8c329edaab3"
    },
    {
      "question_id": "B10",
      "arm": "values",
      "cache_key": "df2e9fb737f5c45bd8b4332495672bf2aad73133bf024e65eadd09b0d43fc361"
    },
    {
      "question_id": "B11",
      "arm": "values",
      "cache_key": "e2aac81bcf6b423609128651f65fbeb05037170470fdbd7c965313edfb9c0690"
    },
    {
      "question_id": "B12",
      "arm": "values",
      "cache_key": "110e224b166b9275a5e9725c22d98686a4a26c658e27d334d33592ccafc67514"
    },
    {
      "question_id": "B13",
      "arm": "values",
      "cache_key": "dc02a3add5b03983de9298c6c703057a9d421f8cf793a783245b7f8a7dc84c75"
    },
    {
      "question_id": "B14",
      "arm": "values",
      "cache_key": "a7ffc3c9b4dbb4817a235107e644fb9971152ffb7631636e1658341579d102cd"
    },
    {
      "question_id": "B15",
      "arm": "values",
      "cache_key": "3a31f150cb03b8866d8562c6beb0bdc2326030ca254bf6c8f5fe62d30e78b71f"
    },
    {
      "question_id": "B16",
      "arm": "values",
      "cache_key": "b37f1448f8d7c0f6134a3b86f79302680fb8e6e9f6bc30779ee3c5a3dbe5da2e"
    },
    {
      "question_id": "B17",
      "arm": "values",
      "cache_key": "411f3d80b401494162c1a9ba5457729b79de49cae2493d59fe455115a1741dff"
    },
    {
      "question_id": "B18",
      "arm": "values",
      "cache_key": "0c8f3a5b78c04695b5b06753d34e4cbf7302e396ab24d2fafa1aa5bc06ce8616"
    },
    {
      "question_id": "B19",
      "arm": "values",
      "cache_key": "cfd66de81f7445a2390f199ce5e8e31b6083d6f026dde74898fb8996967942e1"
    },
    {
      "question_id": "B20",
      "arm": "values",
      "cache_key": "51e1f60bbfdb9e4f27355222e9930d4f949a34055c27b8067bed6cc833c9f9df"
    },
    {
      "question_id": "B21",
      "arm": "values",
      "cache_key": "f3b0ab268e9878531448154f61dd9db6480faedbd71be44d04ca62c78e34bae1"
    },
    {
      "question_id": "B22",
      "arm": "values",
      "cache_key": "aa4897fa0e45659eebb0418a2b2fba79dd2ac9aca1565e80f1cc7447bd9e09e8"
    },
    {
      "question_id": "B23",
      "arm": "values",
      "cache_key": "5d290d62ce36f1b49ce85ac6fafd566793d565af6c4ed2bed8e63477b13d484b"
    },
    {
      "question_id": "B24",
      "arm": "values",
      "cache_key": "ec3d5737df93561a8d1629884396c5acdfec6d969c90e6849cfc46586c7a8e9d"
    },
    {
      "question_id": "B25",
      "arm": "values",
      "cache_key": "d29f2957d58d3f854b6170944bc52931677a49425568583a74a82c77209b303e"
    },
    {
      "question_id": "B26",
      "arm": "values",
      "cache_key": "5f9e7f84d8177605146c3735d9a5f7fed656da01e13740f95706bf1856c636aa"
    },
    {
      "question_id": "B27",
      "arm": "values",
      "cache_key": "de3b2f7464be1a0b18f6c136f5980d8387e09e2c9531d6bdbf47f538bb66b8a5"
    },
    {
      "question_id": "B28",
      "arm": "values",
      "cache_key": "d6e4b66ad09646a1e9bac138896bf00cb8d6d4a1e151706204901f6a2cffe503"
    },
    {
      "question_id": "B29",
      "arm": "values",
      "cache_key": "8cecbbd2388eb18289de4b77bc48cbfd58e4bb5fd10e9251ff168852a9410f11"
    },
    {
      "question_id": "B30",
      "arm": "values",
      "cache_key": "f7702f4a6086c6175bf929be5ebd35cddb037bfaf25f3cedc3a071295821baa9"
    },
    {
      "question_id": "B31",
      "arm": "values",
      "cache_key": "9797db3f0779096d288876cb64cf923aa4ea2601d39992b9796b0466b8721138"
    },
    {
      "question_id": "B32",
      "arm": "values",
      "cache_key": "291a6b9034ed87bc5bdc6689496cbf13a8de030b1b121fc6de169412e71708c2"
    },
    {
      "question_id": "B33",
      "arm": "values",
      "cache_key": "f6526c9bf1908e51588e2a969bca1301de942cda0500f56b9187d772c67ee4a0"
    },
    {
      "question_id": "B34",
      "arm": "values",
      "cache_key": "30f3900d032455d4ec202eddd7b29b8f1a395fd9e0cc4d156eac8f467e08d4c0"
    },
    {
      "question_id": "B35",
      "arm": "values",
      "cache_key": "1beaf8feac72f6db984e313051431462872d6afef8f3fd919736dfd4794be498"
    },
    {
      "question_id": "B36",
      "arm": "values",
      "cache_key": "39a54d1468766f7da0e55fd94d40225fccffddfa7ae42317d87228aeca0fd928"
    },
    {
      "question_id": "B37",
      "arm": "values",
      "cache_key": "263941893b9010c5d4b82a696698beef097dfd4299f4fbdcbb8c38a9a39bf0b9"
    },
    {
      "question_id": "B38",
      "arm": "values",
      "cache_key": "268d9ed34485d9ed4ea30d8f0d8985440418467bbff563aee559c1410b923ef2"
    },
    {
      "question_id": "B39",
      "arm": "values",
      "cache_key": "9abef31c23920aadc5fb72eaa03f1c8059da1c985e61e9119991d402feca1115"
    },
    {
      "question_id": "B40",
      "arm": "values",
      "cache_key": "6023d6d1b28d6fe8d28e76c714c49190656f78656b3af2a369f5480cb6ca0c5d"
    }
  ],
  "file_sha256": {
    "evals/questions.json": "82230aff754c1a6a7cd831a9ab9b4a60a18942c61945c369481c1803d14450b2",
    "evals/schema.json": "4e2d7e72aa5c907c88795491cbf4c63ca9e3211860d63e10be3ca9c3276cc65d",
    "evals/collection-config.json": "824988e085bdc26360095488291d99c7ebf7e68bd3f694fcaf6e39726da16f2b",
    "evals/collection-config-values.json": "d5a5a78418d49673c99784b82124f57f04425c01b76a6e94a6558019f29b8f1b",
    "documentation/value-domains.json": "7b15cff23fcc2e7482ab3828d034051b93c8883ebcff31f0c9668f0fbd4c8234",
    "evals/run-manifest.json": "e6746537101fdbdc7f7449945d58657cb6bdde70b8fbc0c5fa7b079c2eafc2ac",
    "evals/cache-manifest.json": "cfb27f99ec61670b22a140cf3a25053daea6bfa778292d60d6d988c01637451f",
    "banking/inputs.py": "e16ee63c01568145d31d64212dc3c19bcb681d11c265f1c98973eac788a1b130",
    "banking/scoring.py": "8dd14aa09820df04d0277b6c3e0055add001424be5733857192c1d47bc2c1f45",
    "banking/correction.py": "855644d812f7bb7ac1f1e9b9c13d026c547997c443ce755b0f58d8f0e39ed7a3",
    "banking/values_arm.py": "3cc03fcab0d2a035234eb1f52b89d60462cc27ed5fe7b470cf7232da6982e4b7",
    "collect.py": "d0dfc50c4d3aefbcfb678628488c4c9d3f1f72468462b1549441975ce8795148",
    "collect_values.py": "612f92d8aee27268dfb92c543071a4ff52fb7f0549bf2f4a29e3ee5fbf0ad8bd"
  }
}