{ "run_id": "banking-values-a651d652e76a", "addendum_sha256": "dcdc1663991b460346d47a7f7b941fa20611c87db037a51b692b19989a8311ae", "values_receipt_sha256": "fa45c375617c72aaaeca266e8322660d348d03d118d183a3998b4c80fb4ddb9b", "corrected_results_sha256": "78edc3d56c3892e853c7d7dc01d6dfc6e682125103ccb18546cedb9d6486642a", "arms": { "bare": "Bare schema", "values": "Schema plus allowed values", "documented": "Full documentation" }, "counts": { "bare": { "all": 27, "n": 40, "analytical": 27, "analytical_n": 36, "clarification": 0, "clarification_n": 4, "statuses": { "correct": 27, "unsupported_answer": 3, "wrong_answer": 9, "wrong_clarification": 1 } }, "documented": { "all": 33, "n": 40, "analytical": 33, "analytical_n": 36, "clarification": 0, "clarification_n": 4, "statuses": { "correct": 33, "execution_error": 2, "invalid_response": 1, "unsupported_answer": 3, "wrong_answer": 1 } }, "values": { "all": 31, "n": 40, "analytical": 31, "analytical_n": 36, "clarification": 0, "clarification_n": 4, "statuses": { "collection_error": 1, "correct": 31, "query_denied": 1, "unsupported_answer": 2, "wrong_answer": 4, "wrong_clarification": 1 } } }, "comparisons": [ { "baseline": "bare", "treatment": "documented", "all": { "n": 40, "baseline_correct": 27, "treatment_correct": 33, "baseline_accuracy": 0.675, "treatment_accuracy": 0.825, "difference": 0.1499999999999999, "interval": [ 0.0, 0.3 ], "improved": 8, "regressed": 2 }, "analytical": { "n": 36, "baseline_correct": 27, "treatment_correct": 33, "baseline_accuracy": 0.75, "treatment_accuracy": 0.9166666666666666, "difference": 0.16666666666666663, "interval": [ 0.0, 0.3333333333333333 ], "improved": 8, "regressed": 2 }, "clarification": { "n": 4, "baseline_correct": 0, "treatment_correct": 0, "baseline_accuracy": 0.0, "treatment_accuracy": 0.0, "difference": 0.0, "interval": [ 0.0, 0.0 ], "improved": 0, "regressed": 0 }, "twin_clustered": { "all": { "n": 40, "clusters": 32, "largest_cluster": 2, "difference": 0.15, "interval": [ 0.0, 0.3142857142857143 ] }, "analytical": { "n": 36, "clusters": 28, "largest_cluster": 2, "difference": 0.16666666666666666, "interval": [ 0.0, 0.35135135135135137 ] } } }, { "baseline": "bare", "treatment": "values", "all": { "n": 40, "baseline_correct": 27, "treatment_correct": 31, "baseline_accuracy": 0.675, "treatment_accuracy": 0.775, "difference": 0.09999999999999998, "interval": [ 0.0, 0.225 ], "improved": 5, "regressed": 1 }, "analytical": { "n": 36, "baseline_correct": 27, "treatment_correct": 31, "baseline_accuracy": 0.75, "treatment_accuracy": 0.8611111111111112, "difference": 0.11111111111111116, "interval": [ 0.0, 0.25 ], "improved": 5, "regressed": 1 }, "clarification": { "n": 4, "baseline_correct": 0, "treatment_correct": 0, "baseline_accuracy": 0.0, "treatment_accuracy": 0.0, "difference": 0.0, "interval": [ 0.0, 0.0 ], "improved": 0, "regressed": 0 }, "twin_clustered": { "all": { "n": 40, "clusters": 32, "largest_cluster": 2, "difference": 0.1, "interval": [ -0.02564102564102564, 0.24324324324324326 ] }, "analytical": { "n": 36, "clusters": 28, "largest_cluster": 2, "difference": 0.1111111111111111, "interval": [ -0.027777777777777776, 0.2702702702702703 ] } } }, { "baseline": "values", "treatment": "documented", "all": { "n": 40, "baseline_correct": 31, "treatment_correct": 33, "baseline_accuracy": 0.775, "treatment_accuracy": 0.825, "difference": 0.04999999999999993, "interval": [ -0.075, 0.175 ], "improved": 4, "regressed": 2 }, "analytical": { "n": 36, "baseline_correct": 31, "treatment_correct": 33, "baseline_accuracy": 0.8611111111111112, "treatment_accuracy": 0.9166666666666666, "difference": 0.05555555555555547, "interval": [ -0.08333333333333333, 0.19444444444444445 ], "improved": 4, "regressed": 2 }, "clarification": { "n": 4, "baseline_correct": 0, "treatment_correct": 0, "baseline_accuracy": 0.0, "treatment_accuracy": 0.0, "difference": 0.0, "interval": [ 0.0, 0.0 ], "improved": 0, "regressed": 0 }, "twin_clustered": { "all": { "n": 40, "clusters": 32, "largest_cluster": 2, "difference": 0.05, "interval": [ -0.075, 0.175 ] }, "analytical": { "n": 36, "clusters": 28, "largest_cluster": 2, "difference": 0.05555555555555555, "interval": [ -0.07894736842105263, 0.1875 ] } } } ], "twin_rule": "Questions whose text is identical after replacing CAD and USD with one placeholder form one cluster; every other question is its own cluster.", "twin_groups": [ [ "B01", "B02" ], [ "B03" ], [ "B04", "B05" ], [ "B06" ], [ "B07", "B08" ], [ "B09", "B10" ], [ "B11", "B12" ], [ "B13" ], [ "B14" ], [ "B15" ], [ "B16" ], [ "B17" ], [ "B18" ], [ "B19" ], [ "B20", "B21" ], [ "B22" ], [ "B23", "B24" ], [ "B25" ], [ "B26" ], [ "B27", "B28" ], [ "B29" ], [ "B30" ], [ "B31" ], [ "B32" ], [ "B33" ], [ "B34" ], [ "B35" ], [ "B36" ], [ "B37" ], [ "B38" ], [ "B39" ], [ "B40" ] ], "failure_causes": { "rows": [ { "question_id": "B06", "arm": "bare", "status": "wrong_answer", "causes": [ "occurred_at_period" ], "reason": "Counted purchase events by the month of occurred_at instead of the posted spend date.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": true } }, { "question_id": "B19", "arm": "bare", "status": "wrong_answer", "causes": [ "occurred_at_period", "semantics" ], "reason": "Treated purchases as credits and refunds as debits, so the join matched nothing, and filtered refunds by occurred_at.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": true } }, { "question_id": "B20", "arm": "bare", "status": "wrong_answer", "causes": [ "enum_value_guess", "occurred_at_period" ], "reason": "Filtered direction = 'inbound', a value that does not exist, and used occurred_at for the period.", "flags": { "enum_literals_outside_domain": [ [ "direction", "inbound" ] ], "occurred_at_not_in_gold": true } }, { "question_id": "B21", "arm": "bare", "status": "wrong_answer", "causes": [ "enum_value_guess", "occurred_at_period" ], "reason": "Filtered direction = 'inbound', a value that does not exist, and used occurred_at for the period.", "flags": { "enum_literals_outside_domain": [ [ "direction", "inbound" ] ], "occurred_at_not_in_gold": true } }, { "question_id": "B22", "arm": "bare", "status": "wrong_answer", "causes": [ "enum_value_guess", "occurred_at_period" ], "reason": "Filtered direction = 'outbound', a value that does not exist, and used occurred_at for the period.", "flags": { "enum_literals_outside_domain": [ [ "direction", "outbound" ] ], "occurred_at_not_in_gold": true } }, { "question_id": "B23", "arm": "bare", "status": "wrong_answer", "causes": [ "enum_value_guess", "occurred_at_period" ], "reason": "Used direction 'inbound' and event_kind 'external', neither of which exists, and used occurred_at for the period.", "flags": { "enum_literals_outside_domain": [ [ "direction", "inbound" ], [ "event_kind", "external" ] ], "occurred_at_not_in_gold": true } }, { "question_id": "B23", "arm": "documented", "status": "execution_error", "causes": [ "sql_error" ], "reason": "Joined accounts with USING, then filtered an ambiguous currency column; the query failed.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": false } }, { "question_id": "B24", "arm": "bare", "status": "wrong_answer", "causes": [ "enum_value_guess", "occurred_at_period" ], "reason": "Used direction 'inbound' and event_kind 'external', neither of which exists, and used occurred_at for the period.", "flags": { "enum_literals_outside_domain": [ [ "direction", "inbound" ], [ "event_kind", "external" ] ], "occurred_at_not_in_gold": true } }, { "question_id": "B25", "arm": "documented", "status": "execution_error", "causes": [ "sql_error" ], "reason": "Referenced a column through an alias that was not defined in that scope; the query failed.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": false } }, { "question_id": "B26", "arm": "bare", "status": "wrong_answer", "causes": [ "enum_value_guess" ], "reason": "Filtered status = 'completed'; the posted value is 'posted', so no rows matched.", "flags": { "enum_literals_outside_domain": [ [ "status", "completed" ] ], "occurred_at_not_in_gold": false } }, { "question_id": "B35", "arm": "documented", "status": "wrong_answer", "causes": [ "grain" ], "reason": "Joined cards before card spend, so an account with two cards and no July purchases was listed twice.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": false } }, { "question_id": "B36", "arm": "bare", "status": "wrong_answer", "causes": [ "grain" ], "reason": "Summed opening and closing balances over all 31 days of July instead of taking July 1 and July 31.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": false } }, { "question_id": "B06", "arm": "values", "status": "wrong_answer", "causes": [ "occurred_at_period" ], "reason": "Counted purchase events by the month of occurred_at instead of the posted spend date.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": true } }, { "question_id": "B17", "arm": "values", "status": "query_denied", "causes": [ "sql_error" ], "reason": "Used UNION ALL, which the prompt does not allow; the validator denied the query.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": false } }, { "question_id": "B23", "arm": "values", "status": "wrong_answer", "causes": [ "occurred_at_period", "semantics" ], "reason": "Counted deposits, withdrawals and transfers as external cash flow, left out purchases and refunds, and used occurred_at for the period.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": true } }, { "question_id": "B24", "arm": "values", "status": "wrong_answer", "causes": [ "occurred_at_period", "semantics" ], "reason": "Counted only deposits and withdrawals as external cash flow, left out purchases and refunds, and used occurred_at for the period.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": true } }, { "question_id": "B36", "arm": "values", "status": "wrong_answer", "causes": [ "grain" ], "reason": "Took the minimum opening and maximum closing daily balance across accounts instead of summing July 1 opening and July 31 closing.", "flags": { "enum_literals_outside_domain": [], "occurred_at_not_in_gold": false } } ], "by_cause": { "enum_value_guess": { "bare": 6, "values": 0, "documented": 0 }, "occurred_at_period": { "bare": 7, "values": 3, "documented": 0 }, "grain": { "bare": 1, "values": 1, "documented": 1 }, "semantics": { "bare": 1, "values": 2, "documented": 0 }, "sql_error": { "bare": 0, "values": 1, "documented": 2 } }, "incorrect_analytical": { "bare": 9, "values": 5, "documented": 3 }, "note": "Hand labels for every incorrect analytical answer, assigned after the outcomes were seen (founder ruling D12). enum_value_guess and occurred_at_period must agree with flags computed from the stored SQL, and sql_error with the outcome status; grain and semantics are judgement labels." }, "acquisition": { "response_count": 39, "error_count": 1, "known_cost_usd": 0.0220276, "unknown_cost_requests": 1, "reserved_usd": 0.1443176, "input_tokens": 42801, "output_tokens": 3067, "original_latency_seconds": 40.231148, "collected_from": "2026-10-11T03:45:15.149156+00:00", "collected_to": "2026-10-11T03:48:46.402092+00:00" }, "values_outcomes": [ { "question_id": "B01", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT SUM(purchased_minor) FROM card_spend_daily WHERE currency = 'CAD' AND spend_date >= DATE '2026-07-01' AND spend_date < DATE '2026-08-01'", "rows": [ [ 1342646 ] ] }, { "question_id": "B02", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT SUM(purchased_minor) FROM card_spend_daily WHERE currency = 'USD' AND spend_date >= DATE '2026-07-01' AND spend_date < DATE '2026-08-01'", "rows": [ [ 580559 ] ] }, { "question_id": "B03", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(refunded_minor), 0) AS refunded_amount_minor FROM card_spend_daily WHERE currency = 'CAD' AND spend_date >= DATE '2026-07-01' AND spend_date < DATE '2026-08-01'", "rows": [ [ 64654 ] ] }, { "question_id": "B04", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(net_spend_minor), 0) AS net_card_spend_cad_july_2026 FROM card_spend_daily WHERE currency = 'CAD' AND spend_date >= DATE '2026-07-01' AND spend_date < DATE '2026-08-01'", "rows": [ [ 1277992 ] ] }, { "question_id": "B05", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(net_spend_minor), 0) AS net_card_spend_usd_july_2026 FROM card_spend_daily WHERE currency = 'USD' AND spend_date >= DATE '2026-07-01' AND spend_date < DATE '2026-08-01'", "rows": [ [ 554332 ] ] }, { "question_id": "B06", "arm": "values", "kind": "analytical", "correct": false, "status": "wrong_answer", "sql": "SELECT COUNT(*) FROM transactions WHERE event_kind = 'purchase' AND currency = 'CAD' AND EXTRACT(YEAR FROM occurred_at) = 2026 AND EXTRACT(MONTH FROM occurred_at) = 7", "rows": [ [ 342 ] ] }, { "question_id": "B07", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(c.category_name, 'Uncategorized') AS category_label, SUM(cs.net_spend_minor) AS net_minor_units\nFROM card_spend_daily cs\nLEFT JOIN categories c ON cs.category_id = c.category_id\nWHERE cs.currency = 'CAD' AND EXTRACT(YEAR FROM cs.spend_date) = 2026 AND EXTRACT(MONTH FROM cs.spend_date) = 7\nGROUP BY category_label\nHAVING SUM(cs.net_spend_minor) != 0", "rows": [ [ "Software", 224575 ], [ "Travel", 249615 ], [ "Supplies", 219068 ], [ "Meals", 293557 ], [ "Uncategorized", 291177 ] ] }, { "question_id": "B08", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(c.category_name, 'Uncategorized') AS category_label, SUM(cs.net_spend_minor) AS net_minor_units\nFROM card_spend_daily cs\nLEFT JOIN categories c ON cs.category_id = c.category_id\nWHERE cs.currency = 'USD' AND EXTRACT(YEAR FROM cs.spend_date) = 2026 AND EXTRACT(MONTH FROM cs.spend_date) = 7\nGROUP BY category_label\nHAVING SUM(cs.net_spend_minor) != 0", "rows": [ [ "Meals", 111588 ], [ "Travel", 117967 ], [ "Software", 101312 ], [ "Supplies", 94490 ], [ "Uncategorized", 128975 ] ] }, { "question_id": "B09", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT a.organization_id, SUM(csd.net_spend_minor) AS net_spend_minor\nFROM card_spend_daily csd\nJOIN accounts a ON csd.account_id = a.account_id\nWHERE csd.currency = 'CAD'\n AND csd.spend_date >= DATE '2026-07-01'\n AND csd.spend_date < DATE '2026-08-01'\nGROUP BY a.organization_id\nHAVING SUM(csd.net_spend_minor) != 0", "rows": [ [ "org_01", 230735 ], [ "org_02", 203006 ], [ "org_03", 253869 ], [ "org_04", 250691 ], [ "org_05", 243071 ], [ "org_06", 96620 ] ] }, { "question_id": "B10", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT a.organization_id, SUM(csd.net_spend_minor) AS net_spend_minor\nFROM card_spend_daily csd\nJOIN accounts a ON csd.account_id = a.account_id\nWHERE csd.currency = 'USD'\n AND csd.spend_date >= DATE '2026-07-01'\n AND csd.spend_date < DATE '2026-08-01'\nGROUP BY a.organization_id\nHAVING SUM(csd.net_spend_minor) != 0", "rows": [ [ "org_01", 106600 ], [ "org_02", 127490 ], [ "org_03", 88313 ], [ "org_05", 127957 ], [ "org_04", 103972 ] ] }, { "question_id": "B11", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT a.account_id, COALESCE(SUM(csd.net_spend_minor), 0) AS net_spend_minor\nFROM accounts a\nLEFT JOIN card_spend_daily csd ON a.account_id = csd.account_id\n AND csd.currency = 'CAD'\n AND csd.spend_date >= DATE '2026-07-01' AND csd.spend_date <= DATE '2026-07-31'\nWHERE a.currency = 'CAD'\nGROUP BY a.account_id", "rows": [ [ "acct_001", 129813 ], [ "acct_002", 100922 ], [ "acct_004", 106104 ], [ "acct_005", 96902 ], [ "acct_007", 130636 ], [ "acct_008", 123233 ], [ "acct_011", 126577 ], [ "acct_013", 120981 ], [ "acct_014", 122090 ], [ "acct_016", 96620 ], [ "acct_010", 124114 ], [ "acct_019", 0 ], [ "acct_017", 0 ] ] }, { "question_id": "B12", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT a.account_id, COALESCE(SUM(csd.net_spend_minor), 0) AS net_spend_minor\nFROM accounts a\nLEFT JOIN card_spend_daily csd ON a.account_id = csd.account_id\n AND csd.currency = 'USD'\n AND csd.spend_date >= DATE '2026-07-01' AND csd.spend_date <= DATE '2026-07-31'\nWHERE a.currency = 'USD'\nGROUP BY a.account_id", "rows": [ [ "acct_003", 106600 ], [ "acct_006", 127490 ], [ "acct_009", 88313 ], [ "acct_015", 127957 ], [ "acct_012", 103972 ], [ "acct_018", 0 ] ] }, { "question_id": "B13", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT STRFTIME(spend_date, '%Y-%m-%d') AS date, SUM(net_spend_minor) AS net_spend_minor\nFROM card_spend_daily\nWHERE currency = 'CAD' AND spend_date >= DATE '2026-07-01' AND spend_date < DATE '2026-08-01'\nGROUP BY spend_date\nORDER BY spend_date ASC", "rows": [ [ "2026-07-01", 66420 ], [ "2026-07-02", 48706 ], [ "2026-07-03", 29351 ], [ "2026-07-04", 43817 ], [ "2026-07-05", 25257 ], [ "2026-07-06", 47767 ], [ "2026-07-07", 54913 ], [ "2026-07-08", 37942 ], [ "2026-07-09", 37747 ], [ "2026-07-10", 42269 ], [ "2026-07-11", 58518 ], [ "2026-07-12", 27667 ], [ "2026-07-13", 52979 ], [ "2026-07-14", 49789 ], [ "2026-07-15", 25343 ], [ "2026-07-16", 29279 ], [ "2026-07-17", 41628 ], [ "2026-07-18", 46129 ], [ "2026-07-19", 34962 ], [ "2026-07-20", 43742 ], [ "2026-07-21", 40948 ], [ "2026-07-22", 45552 ], [ "2026-07-23", 26826 ], [ "2026-07-24", 49751 ], [ "2026-07-25", 30147 ], [ "2026-07-26", 34592 ], [ "2026-07-27", 61314 ], [ "2026-07-28", 38322 ], [ "2026-07-29", 37432 ], [ "2026-07-30", 30821 ], [ "2026-07-31", 38062 ] ] }, { "question_id": "B14", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT csd.account_id, SUM(csd.net_spend_minor) AS net_minor_units\nFROM card_spend_daily csd\nJOIN accounts a ON csd.account_id = a.account_id\nWHERE csd.currency = 'CAD' AND a.currency = 'CAD' AND csd.spend_date >= DATE '2026-07-01' AND csd.spend_date < DATE '2026-08-01'\nGROUP BY csd.account_id\nORDER BY net_minor_units DESC, csd.account_id ASC\nLIMIT 3", "rows": [ [ "acct_007", 130636 ], [ "acct_001", 129813 ], [ "acct_011", 126577 ] ] }, { "question_id": "B15", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(c.category_name, 'Uncategorized') AS category_label, SUM(cs.net_spend_minor) AS net_minor_units\nFROM card_spend_daily cs\nLEFT JOIN categories c ON cs.category_id = c.category_id\nWHERE cs.currency = 'CAD' AND EXTRACT(YEAR FROM cs.spend_date) = 2026 AND EXTRACT(MONTH FROM cs.spend_date) = 7\nGROUP BY category_label\nORDER BY net_minor_units DESC, category_label ASC\nLIMIT 3", "rows": [ [ "Meals", 293557 ], [ "Uncategorized", 291177 ], [ "Travel", 249615 ] ] }, { "question_id": "B16", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT a.account_id, COUNT(c.card_id) AS active_card_count\nFROM accounts a\nLEFT JOIN cards c ON a.account_id = c.account_id AND c.card_status = 'active'\nWHERE a.currency = 'CAD'\nGROUP BY a.account_id", "rows": [ [ "acct_001", 2 ], [ "acct_002", 2 ], [ "acct_004", 2 ], [ "acct_005", 1 ], [ "acct_007", 2 ], [ "acct_008", 2 ], [ "acct_010", 1 ], [ "acct_011", 2 ], [ "acct_013", 2 ], [ "acct_014", 2 ], [ "acct_016", 2 ], [ "acct_017", 2 ], [ "acct_019", 0 ] ] }, { "question_id": "B17", "arm": "values", "kind": "analytical", "correct": false, "status": "query_denied", "sql": "SELECT currency, COUNT(*) AS active_card_count FROM cards JOIN accounts USING(account_id) WHERE card_status = 'active' GROUP BY currency UNION ALL SELECT currency, 0 FROM accounts WHERE currency NOT IN (SELECT currency FROM cards JOIN accounts USING(account_id) WHERE card_status = 'active') GROUP BY currency ORDER BY currency", "rows": null }, { "question_id": "B18", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "WITH RefundsInJuly AS (\n SELECT\n related_transaction_id AS original_transaction_id,\n SUM(amount_minor) AS total_refunded_july\n FROM transactions\n WHERE event_kind = 'refund'\n AND currency = 'CAD'\n AND status = 'posted'\n AND EXTRACT(YEAR FROM occurred_at) = 2026\n AND EXTRACT(MONTH FROM occurred_at) = 7\n GROUP BY related_transaction_id\n),\nOriginalPurchases AS (\n SELECT\n transaction_id,\n amount_minor AS original_purchase_amount\n FROM transactions\n WHERE event_kind = 'purchase'\n AND currency = 'CAD'\n AND status = 'posted'\n)\nSELECT\n o.transaction_id AS original_transaction_id,\n o.original_purchase_amount,\n r.total_refunded_july\nFROM OriginalPurchases o\nJOIN RefundsInJuly r ON o.transaction_id = r.original_transaction_id;", "rows": [ [ "txn_00007", 1520, 506 ], [ "txn_00015", 7754, 2584 ], [ "txn_00030", 8191, 2730 ], [ "txn_00041", 9577, 3192 ], [ "txn_00049", 1443, 481 ], [ "txn_00057", 4566, 1522 ], [ "txn_00065", 546, 182 ], [ "txn_00118", 5858, 1952 ], [ "txn_00126", 3656, 1218 ], [ "txn_00134", 6827, 2275 ], [ "txn_00145", 5990, 1996 ], [ "txn_00153", 3000, 1000 ], [ "txn_00212", 7369, 2456 ], [ "txn_00220", 6360, 2120 ], [ "txn_00228", 9820, 3273 ], [ "txn_00236", 5333, 1777 ], [ "txn_00247", 6544, 2181 ], [ "txn_00262", 5017, 1672 ], [ "txn_00270", 6727, 2242 ], [ "txn_00316", 960, 320 ], [ "txn_00324", 7040, 2346 ], [ "txn_00332", 3863, 1287 ], [ "txn_00340", 3740, 1246 ], [ "txn_00351", 5838, 1946 ], [ "txn_00359", 1907, 635 ], [ "txn_00367", 2869, 956 ], [ "txn_00418", 5596, 1865 ], [ "txn_00426", 5476, 1825 ], [ "txn_00434", 5894, 1964 ], [ "txn_00442", 4394, 1464 ], [ "txn_00453", 8460, 2820 ], [ "txn_00461", 7231, 2410 ], [ "txn_00469", 1055, 351 ], [ "txn_00477", 2564, 854 ], [ "txn_00522", 3059, 1019 ], [ "txn_00530", 3720, 1240 ], [ "txn_00538", 4208, 1402 ], [ "boundary_june_occurrence", 12345, 3345 ] ] }, { "question_id": "B19", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COUNT(DISTINCT t1.transaction_id) AS count_distinct_purchases_with_refund\nFROM transactions t1\nJOIN transactions t2 ON t1.transaction_id = t2.related_transaction_id\nWHERE t1.currency = 'CAD'\n AND t1.event_kind = 'purchase'\n AND t1.status = 'posted'\n AND t2.event_kind = 'refund'\n AND t2.status = 'posted'\n AND EXTRACT(YEAR FROM t2.occurred_at) = 2026\n AND EXTRACT(MONTH FROM t2.occurred_at) = 7", "rows": [ [ 38 ] ] }, { "question_id": "B20", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(amount_minor), 0) AS deposit_amount_minor FROM transactions WHERE event_kind = 'deposit' AND currency = 'CAD' AND occurred_at >= '2026-07-01' AND occurred_at < '2026-08-01' AND status = 'posted'", "rows": [ [ 769596 ] ] }, { "question_id": "B21", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(amount_minor), 0) AS deposit_amount_usd_july_2026 FROM transactions WHERE event_kind = 'deposit' AND currency = 'USD' AND occurred_at >= '2026-07-01' AND occurred_at < '2026-08-01' AND status = 'posted'", "rows": [ [ 320665 ] ] }, { "question_id": "B22", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(amount_minor), 0) AS withdrawal_amount_minor FROM transactions WHERE event_kind = 'withdrawal' AND currency = 'CAD' AND occurred_at >= '2026-07-01' AND occurred_at < '2026-08-01' AND status = 'posted'", "rows": [ [ 12084 ] ] }, { "question_id": "B23", "arm": "values", "kind": "analytical", "correct": false, "status": "wrong_answer", "sql": "SELECT COALESCE(SUM(CASE WHEN direction = 'credit' THEN amount_minor ELSE -amount_minor END), 0) AS net_external_cash_flow_minor FROM transactions WHERE currency = 'CAD' AND status = 'posted' AND occurred_at >= '2026-07-01' AND occurred_at < '2026-08-01' AND event_kind IN ('deposit', 'withdrawal', 'transfer')", "rows": [ [ 757512 ] ] }, { "question_id": "B24", "arm": "values", "kind": "analytical", "correct": false, "status": "wrong_answer", "sql": "SELECT COALESCE(SUM(CASE WHEN direction = 'credit' THEN amount_minor ELSE -amount_minor END), 0) AS net_external_cash_flow_usd_july_2026 FROM transactions WHERE currency = 'USD' AND status = 'posted' AND occurred_at >= '2026-07-01' AND occurred_at < '2026-08-01' AND event_kind IN ('deposit', 'withdrawal')", "rows": [ [ 315630 ] ] }, { "question_id": "B25", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(ABS(amount_minor)), 0) AS internal_transfer_volume_cad_july_2026 FROM (SELECT DISTINCT transfer_group_id, amount_minor FROM transactions WHERE currency = 'CAD' AND event_kind = 'transfer' AND status = 'posted' AND EXTRACT(YEAR FROM occurred_at) = 2026 AND EXTRACT(MONTH FROM occurred_at) = 7 AND transfer_group_id IS NOT NULL)", "rows": [ [ 13910 ] ] }, { "question_id": "B26", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT currency, COUNT(DISTINCT transfer_group_id) AS transfer_count\nFROM transactions\nWHERE event_kind = 'transfer'\n AND status = 'posted'\n AND EXTRACT(YEAR FROM occurred_at) = 2026\n AND EXTRACT(MONTH FROM occurred_at) = 7\nGROUP BY currency", "rows": [ [ "CAD", 4 ], [ "USD", 4 ] ] }, { "question_id": "B27", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT SUM(closing_minor) FROM account_daily WHERE balance_date = DATE '2026-07-31' AND currency = 'CAD'", "rows": [ [ 125676 ] ] }, { "question_id": "B28", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT SUM(closing_minor) FROM account_daily WHERE balance_date = DATE '2026-07-31' AND currency = 'USD'", "rows": [ [ 20113 ] ] }, { "question_id": "B29", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT SUM(opening_balance_minor) FROM accounts WHERE currency = 'CAD' AND account_id IN (SELECT account_id FROM account_daily WHERE balance_date = DATE '2026-07-01')", "rows": [ [ 646156 ] ] }, { "question_id": "B30", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT SUM(closing_minor) FROM account_daily WHERE balance_date = DATE '2026-07-15' AND currency = 'CAD'", "rows": [ [ 308471 ] ] }, { "question_id": "B31", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(purchased_minor), 0) FROM card_spend_daily WHERE spend_date = DATE '2026-07-01' AND currency = 'CAD'", "rows": [ [ 66420 ] ] }, { "question_id": "B32", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COALESCE(SUM(purchased_minor), 0) AS gross_cad_card_spend FROM card_spend_daily WHERE spend_date = DATE '2026-07-31' AND currency = 'CAD'", "rows": [ [ 38062 ] ] }, { "question_id": "B33", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COUNT(*) AS event_count, SUM(amount_minor) AS total_amount_minor FROM transactions WHERE currency = 'CAD' AND event_kind = 'purchase' AND status = 'pending' AND EXTRACT(YEAR FROM occurred_at) = 2026 AND EXTRACT(MONTH FROM occurred_at) = 7", "rows": [ [ 31, 146326 ] ] }, { "question_id": "B34", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT COUNT(*) AS event_count, SUM(amount_minor) AS total_amount_minor FROM transactions WHERE event_kind = 'purchase' AND status = 'void' AND currency = 'CAD' AND EXTRACT(YEAR FROM occurred_at) = 2026 AND EXTRACT(MONTH FROM occurred_at) = 7", "rows": [ [ 25, 145470 ] ] }, { "question_id": "B35", "arm": "values", "kind": "analytical", "correct": true, "status": "correct", "sql": "SELECT a.account_id\nFROM accounts a\nWHERE a.currency = 'CAD'\nAND a.account_id NOT IN (\n SELECT DISTINCT csd.account_id\n FROM card_spend_daily csd\n WHERE csd.currency = 'CAD'\n AND csd.spend_date >= DATE '2026-07-01'\n AND csd.spend_date < DATE '2026-08-01'\n AND csd.purchased_minor > 0\n)", "rows": [ [ "acct_017" ], [ "acct_019" ] ] }, { "question_id": "B36", "arm": "values", "kind": "analytical", "correct": false, "status": "wrong_answer", "sql": "SELECT currency,\n MIN(opening_minor) AS opening_balance,\n SUM(posted_credit_minor - posted_debit_minor) AS total_signed_posted_movement,\n MAX(closing_minor) AS closing_balance\nFROM account_daily\nWHERE balance_date BETWEEN DATE '2026-07-01' AND DATE '2026-07-31'\nGROUP BY currency", "rows": [ [ "CAD", -32784, -520480, 128248 ], [ "USD", -5139, -238702, 72958 ] ] }, { "question_id": "B37", "arm": "values", "kind": "clarification", "correct": false, "status": "collection_error", "sql": null, "rows": null }, { "question_id": "B38", "arm": "values", "kind": "clarification", "correct": false, "status": "unsupported_answer", "sql": "SELECT SUM(net_spend_minor) FROM card_spend_daily WHERE currency = 'CAD'", "rows": null }, { "question_id": "B39", "arm": "values", "kind": "clarification", "correct": false, "status": "unsupported_answer", "sql": "SELECT SUM(closing_minor) FROM account_daily WHERE balance_date = DATE '2026-07-31' AND currency = 'CAD'", "rows": null }, { "question_id": "B40", "arm": "values", "kind": "clarification", "correct": false, "status": "wrong_clarification", "sql": null, "rows": null, "missing": [ "time_window", "metric_definition" ] } ] }