{"figure":"replay-nights","title":"Run records the nightly replay read and could score","page":"https://www.orbytlabs.ai/research/agent-evals","url":"https://www.orbytlabs.ai/research/agent-evals/replay-nights.json","source":"app/(site)/research/agent-evals/incident-figures.generated.json (replays.series), written by scripts/research/method-figures/agent-evals.ts from the nightly receipts in docs/org/replays/","caveats":["A record the replay could not score has no verdict. The receipts say so themselves: excluded rows are not zero.","Dates are the receipts' file stamps in UTC, so a night's date can be the day after the evening it ran in Central time."],"not_shown":["Why each other record could not be scored. Each receipt gives its reasons in its not_scorable and coverage blocks."],"counted_on":"2026-11-20","cite_as":"Orbyt Labs, Run records the nightly replay read and could score (2026-11-20), https://www.orbytlabs.ai/research/agent-evals/replay-nights.json","first_receipt":"2026-09-30","newest_receipt":"2026-11-20","receipts":"49","policy_changed":"2026-10-08T21:57:33.000Z commit 840c7ce26f15 (docs/org/replay-policy.json)","columns":[{"key":"night","label":"The receipt's file stamp date in UTC, one row per calendar day, YYYY-MM-DD"},{"key":"receipt","label":"true when a receipt exists for the night"},{"key":"records_read","label":"Run records the replay read (the receipt's stats input rows)","unit":"records"},{"key":"records_scored","label":"Run records it could score (certainly unaffected plus possibly cut); empty when the receipt gives no verdict","unit":"records"},{"key":"verdict_observed","label":"false when the receipt carries no scored number (the replay reported not observed); empty with no receipt"},{"key":"policy_sha256_prefix","label":"The first twelve hex of the receipt's policy_sha256, the replay policy file it ran"}],"rows":[{"night":"2026-09-30","receipt":true,"records_read":123,"records_scored":9,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-01","receipt":true,"records_read":125,"records_scored":10,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-02","receipt":true,"records_read":131,"records_scored":10,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-03","receipt":true,"records_read":135,"records_scored":10,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-04","receipt":true,"records_read":139,"records_scored":11,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-05","receipt":true,"records_read":141,"records_scored":11,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-06","receipt":true,"records_read":143,"records_scored":11,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-07","receipt":true,"records_read":146,"records_scored":12,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-08","receipt":true,"records_read":149,"records_scored":12,"verdict_observed":true,"policy_sha256_prefix":"a14c644cb313"},{"night":"2026-10-09","receipt":true,"records_read":151,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-10","receipt":true,"records_read":153,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-11","receipt":true,"records_read":155,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-12","receipt":true,"records_read":158,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-13","receipt":true,"records_read":158,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-14","receipt":true,"records_read":163,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-15","receipt":true,"records_read":165,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-16","receipt":true,"records_read":167,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-17","receipt":true,"records_read":169,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-18","receipt":false,"records_read":null,"records_scored":null,"verdict_observed":null,"policy_sha256_prefix":null},{"night":"2026-10-19","receipt":true,"records_read":170,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-20","receipt":true,"records_read":173,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-21","receipt":true,"records_read":175,"records_scored":null,"verdict_observed":false,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-22","receipt":true,"records_read":178,"records_scored":1,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-23","receipt":true,"records_read":180,"records_scored":1,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-24","receipt":true,"records_read":183,"records_scored":1,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-25","receipt":true,"records_read":185,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-26","receipt":true,"records_read":188,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-27","receipt":true,"records_read":190,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-28","receipt":true,"records_read":192,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-29","receipt":true,"records_read":192,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-30","receipt":true,"records_read":195,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-10-31","receipt":true,"records_read":196,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-01","receipt":true,"records_read":199,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-02","receipt":true,"records_read":202,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-03","receipt":true,"records_read":206,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-04","receipt":true,"records_read":208,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-05","receipt":true,"records_read":208,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-06","receipt":false,"records_read":null,"records_scored":null,"verdict_observed":null,"policy_sha256_prefix":null},{"night":"2026-11-07","receipt":true,"records_read":212,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-08","receipt":true,"records_read":215,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-09","receipt":true,"records_read":217,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-10","receipt":true,"records_read":219,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-11","receipt":false,"records_read":null,"records_scored":null,"verdict_observed":null,"policy_sha256_prefix":null},{"night":"2026-11-12","receipt":true,"records_read":224,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-13","receipt":true,"records_read":225,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-14","receipt":true,"records_read":228,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-15","receipt":true,"records_read":230,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-16","receipt":true,"records_read":232,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-17","receipt":true,"records_read":234,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-18","receipt":true,"records_read":237,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-19","receipt":true,"records_read":238,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"},{"night":"2026-11-20","receipt":true,"records_read":241,"records_scored":2,"verdict_observed":true,"policy_sha256_prefix":"41f4626d40da"}]}