{
  "name": "The Orbyt Failure Corpus",
  "url": "https://www.orbytlabs.ai/orbyt-collective/how-it-works",
  "what": "Documented failure modes from a production codebase operated by AI agents, each one recorded when it bit, classified, and where a countermeasure was mechanized, linked to the countermeasure that now prevents it.",
  "method": "Item ids, the count, and first-documented dates are parsed from the committed corpus file. Public titles and taxonomy kinds are edited for publication from a reviewed map; the generator fails if the map and the corpus ever disagree on which items exist, and fails if a claimed countermeasure path is absent from the repository.",
  "caveat": "This is the operator grading its own mistakes. Early entries predate the dating convention and are published undated rather than backdated. A mechanism path names the countermeasure inside a private repository; the path is published so the claim is specific, even though the file is not.",
  "license": "CC BY 4.0, attribution required",
  "cite_as": "Orbyt Failure Corpus (2026-09-02), https://www.orbytlabs.ai/orbyt-collective/how-it-works",
  "generated_at": "2026-09-02T10:34:39.029Z",
  "count": 66,
  "mechanized_count": 42,
  "dated_count": 55,
  "first_documented": "2026-06-09",
  "latest_documented": "2026-09-02",
  "taxonomy": {
    "external-behavior": {
      "definition": "An external platform, client, or dependency behaved differently than documented or assumed.",
      "count": 15
    },
    "verification-gap": {
      "definition": "A test, guard, gate, or metric that could not fail, could not see, or measured a proxy for the thing it claimed to measure.",
      "count": 33
    },
    "operator-process": {
      "definition": "A discipline failure in how the AI operator built or maintained the system: stale documents, unswept invariants, contracts with one side missing.",
      "count": 18
    }
  },
  "items": [
    {
      "id": 1,
      "date": null,
      "kind": "external-behavior",
      "title": "A mobile browser can suspend a tab without releasing its Web Locks, deadlocking auth refresh",
      "mechanism": "app/layout.tsx"
    },
    {
      "id": 2,
      "date": null,
      "kind": "external-behavior",
      "title": "Realtime sync can echo a write back into itself unless transient UI state stays out of the database"
    },
    {
      "id": 3,
      "date": null,
      "kind": "external-behavior",
      "title": "High-frequency writes to a replicated column can destabilize the replication layer itself"
    },
    {
      "id": 4,
      "date": null,
      "kind": "operator-process",
      "title": "A cleared cache with a surviving session is a distinct state; UI must wait for hydration to complete"
    },
    {
      "id": 5,
      "date": null,
      "kind": "external-behavior",
      "title": "A managed provider's default email path cannot be assumed to deliver; route through one you can observe"
    },
    {
      "id": 6,
      "date": null,
      "kind": "operator-process",
      "title": "Unsafe HTML injection requires an explicit, same-line trust annotation to pass review"
    },
    {
      "id": 7,
      "date": null,
      "kind": "operator-process",
      "title": "A module whose internal name diverges from its export breaks tooling that assumes they match"
    },
    {
      "id": 8,
      "date": null,
      "kind": "operator-process",
      "title": "After a rename, the old name is a liability: redirect it once, then never reference it again"
    },
    {
      "id": 9,
      "date": null,
      "kind": "verification-gap",
      "title": "A date test that mocks the clock in a different timezone convention than the code parses fails only west of UTC"
    },
    {
      "id": 10,
      "date": null,
      "kind": "external-behavior",
      "title": "A client library that auto-deserializes on read breaks every consumer that stored a string, and only on cache hit",
      "mechanism": "app/api/_lib/cache.ts"
    },
    {
      "id": 11,
      "date": "2026-06-09",
      "kind": "operator-process",
      "title": "A programmatic page matrix needs a bounded kept-set imported by both the builder and the sitemap",
      "mechanism": "__tests__/site/salary-data.test.ts"
    },
    {
      "id": 12,
      "date": "2026-06-09",
      "kind": "operator-process",
      "title": "An internal spec date is not a protocol version; clients hard-reject unknown values",
      "mechanism": "__tests__/mcp/protocol-negotiation.test.ts"
    },
    {
      "id": 13,
      "date": "2026-07-10",
      "kind": "operator-process",
      "title": "A redirect parameter is a contract between writer and reader; a silent fallback hides the mismatch"
    },
    {
      "id": 14,
      "date": "2026-07-10",
      "kind": "external-behavior",
      "title": "Hiding overflow on one axis creates a scroll container on the other and can trap wheel events"
    },
    {
      "id": 15,
      "date": "2026-07-11",
      "kind": "operator-process",
      "title": "A client-reachable module that statically imports a heavy data source ships it to every page that renders it"
    },
    {
      "id": 16,
      "date": "2026-07-23",
      "kind": "verification-gap",
      "title": "A test that derives its expected value from the code under test can never fail",
      "mechanism": "scripts/check-log-status-truth.cjs"
    },
    {
      "id": 17,
      "date": "2026-07-23",
      "kind": "operator-process",
      "title": "A pre-authentication code path needs its own abuse ceiling",
      "mechanism": "app/api/v1/intelligence/_lib/route-wrapper.ts"
    },
    {
      "id": 18,
      "date": "2026-07-23",
      "kind": "verification-gap",
      "title": "A table listing's row counts are planner estimates; report a real count or report nothing"
    },
    {
      "id": 19,
      "date": "2026-07-25",
      "kind": "external-behavior",
      "title": "Email clients default to content-box, so full width plus padding overflows a phone"
    },
    {
      "id": 20,
      "date": "2026-07-25",
      "kind": "verification-gap",
      "title": "A CI gate that only runs on the day it matters has never actually been tested; force the condition once"
    },
    {
      "id": 21,
      "date": "2026-07-26",
      "kind": "verification-gap",
      "title": "A convention that has never been executed is not a working convention"
    },
    {
      "id": 22,
      "date": "2026-07-28",
      "kind": "verification-gap",
      "title": "A build filter that decides inclusion is a detector, and must be proven on the shortest legitimate case"
    },
    {
      "id": 23,
      "date": "2026-07-30",
      "kind": "external-behavior",
      "title": "Invalid CSS is discarded, not flagged; a broken value silently becomes the property's initial value"
    },
    {
      "id": 24,
      "date": "2026-07-31",
      "kind": "external-behavior",
      "title": "Read a provider response by scanning for the block you want, never by indexing position zero",
      "mechanism": "scripts/check-ai-response-shape.cjs"
    },
    {
      "id": 25,
      "date": "2026-07-31",
      "kind": "external-behavior",
      "title": "A model's output budget caps reasoning plus text; leave neither half implicit",
      "mechanism": "app/api/_lib/ai-models.ts"
    },
    {
      "id": 26,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A tool that runs quarterly has no gate between quarters, and its hand-copied config drifts silently",
      "mechanism": "__tests__/config/stryker-config-sync.test.ts"
    },
    {
      "id": 27,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A cron reporting success is not healthy; watch duration against its ceiling and reap rows whose process never returned"
    },
    {
      "id": 28,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A timeout reports as cancelled, not failed; alarms keyed on failure alone are structurally blind to it",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 29,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A metric measured and then discarded is indistinguishable from a metric never measured"
    },
    {
      "id": 30,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A synchronous spawn against an in-process fake server deadlocks, and the hang reads as the child failing"
    },
    {
      "id": 31,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A step that discards stderr turns a missing permission into a silently wrong branch",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 32,
      "date": "2026-08-01",
      "kind": "operator-process",
      "title": "A relabeled invariant must sweep every detector that encodes it, in the same commit",
      "mechanism": "scripts/check-data-integrity.cjs"
    },
    {
      "id": 33,
      "date": "2026-08-01",
      "kind": "external-behavior",
      "title": "Checks sharing one rate-limit window flake each other; absorb exactly one window or serialize",
      "mechanism": "scripts/check-smoke.cjs"
    },
    {
      "id": 34,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A fix committed but not pushed does not exist, and a guard grading the working tree will certify it as live",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 35,
      "date": "2026-08-02",
      "kind": "verification-gap",
      "title": "An instrument's failure paths must render as not observed, never as clean",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 36,
      "date": "2026-08-02",
      "kind": "operator-process",
      "title": "A lesson mechanized in the canonical pipeline does not exist for the surfaces that bypass it",
      "mechanism": "__tests__/scripts/org-guard-parity.test.ts"
    },
    {
      "id": 37,
      "date": "2026-08-05",
      "kind": "operator-process",
      "title": "A selection rule optimizes its proxy; verify what exists against measured demand, not against the code",
      "mechanism": "scripts/check-salary-demand-coverage.ts"
    },
    {
      "id": 38,
      "date": "2026-08-08",
      "kind": "operator-process",
      "title": "A sync rule that tests equality can never be satisfied by the thing it exists to allow; record provenance instead",
      "mechanism": "__tests__/scripts/sync-i18n-provenance.test.ts"
    },
    {
      "id": 39,
      "date": "2026-08-08",
      "kind": "verification-gap",
      "title": "Never validate a filter against data that filter produced"
    },
    {
      "id": 40,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "A gate whose correctness rests on the thing it is gating is not a gate",
      "mechanism": "packages/collective/actions/org-agent/action.yml"
    },
    {
      "id": 41,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "A job cancelled with zero steps was never handed to a runner; an alarm inside the job cannot see the job never starting",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 42,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "Read a monitor's freshness the way the monitor is actually fed",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 43,
      "date": "2026-08-10",
      "kind": "operator-process",
      "title": "A stale document reads as instructions; when an invariant reverses, sweep the documents that assert it",
      "mechanism": "scripts/check-root-clutter.cjs"
    },
    {
      "id": 44,
      "date": "2026-08-13",
      "kind": "verification-gap",
      "title": "A sandbox that cannot start reports success, and a fence can apply halfway",
      "mechanism": ".github/workflows/fence-canary.yml"
    },
    {
      "id": 45,
      "date": "2026-08-13",
      "kind": "operator-process",
      "title": "A confinement lane is what an agent is instructed to write, not what gets committed",
      "mechanism": "__tests__/scripts/org-fence.test.ts"
    },
    {
      "id": 46,
      "date": "2026-08-13",
      "kind": "operator-process",
      "title": "A guard's exit code is not its verdict; report-only and strict-gated checks exit zero while reporting"
    },
    {
      "id": 47,
      "date": "2026-08-13",
      "kind": "verification-gap",
      "title": "A gate that reads a tool's printed summary is reading a rendering, and colour in CI can leave it with no way to fail",
      "mechanism": "__tests__/scripts/test-gate.test.ts"
    },
    {
      "id": 48,
      "date": "2026-08-15",
      "kind": "verification-gap",
      "title": "Permissive access policies combine with OR, so one globally true policy defeats every narrow policy beside it, and two individually correct guards can still leave the class unowned",
      "mechanism": "scripts/check-policy-breadth.cjs"
    },
    {
      "id": 49,
      "date": "2026-08-15",
      "kind": "verification-gap",
      "title": "A generated file that measures the repository containing it can never be correct at the moment it is committed, and a drift guard that performs no regeneration of its own reports PASS on a precondition it never established",
      "mechanism": "scripts/check-generated-drift.cjs"
    },
    {
      "id": 50,
      "date": "2026-08-20",
      "kind": "external-behavior",
      "title": "A shallow repository clone makes the version-control history answer confidently and wrongly, so a generator that trusts it publishes a false freshness claim with no error anywhere",
      "mechanism": "scripts/update-skunkworks-dates.cjs"
    },
    {
      "id": 51,
      "date": "2026-08-20",
      "kind": "external-behavior",
      "title": "A privileged key does not bypass a control the upstream service enforces, and charging that service's refusal to a user's abuse budget locks out the innocent",
      "mechanism": "__tests__/api/mobile-login.test.ts"
    },
    {
      "id": 52,
      "date": "2026-08-20",
      "kind": "operator-process",
      "title": "An upstream service reports a downstream failure in its own vocabulary, so the error names a cause that is not the real one, and four consecutive fixes were aimed at the wrong side of the wire",
      "mechanism": "scripts/check-auth-hook-health.cjs"
    },
    {
      "id": 53,
      "date": "2026-08-21",
      "kind": "operator-process",
      "title": "Internal documentation is not a wire contract: a set of event names transcribed from reviewed internal docs matched nothing the sender actually emits, and a single observed value disagreeing with a document invalidates the whole document rather than the one row it lands on",
      "mechanism": "__tests__/api/auth-send-email-hook.test.ts"
    },
    {
      "id": 54,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "A monitoring query that samples without filtering reports the absence of the needle as the absence of the problem, and sorting newest-first does not fix it; a reported count equal to the page size is never a measurement",
      "mechanism": "__tests__/scripts/check-auth-hook-health.test.ts"
    },
    {
      "id": 55,
      "date": "2026-08-21",
      "kind": "external-behavior",
      "title": "A dependency's implicit fetch option is a route-classification decision, and it can silently repeal a page's own segment config, turning a declared 404 into a streamed 200 soft-404 that only the deployed build exhibits",
      "mechanism": "scripts/check-blog-route-static.cjs"
    },
    {
      "id": 56,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "An allowlist entry is a claim about production that decays silently, and a guard checking whether a value is declared cannot go loud about the value that actually costs money",
      "mechanism": "__tests__/site/dynamic-params-guard.test.ts"
    },
    {
      "id": 57,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "A verification that normalizes its input passes on a value the real consumer will reject, so an instrument must read its subject exactly the way the consumer reads it"
    },
    {
      "id": 58,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "A cost measurement is valid only for the resource it measured on the machine that binds it, and a local build stops one whole stage before the one that fails",
      "mechanism": "__tests__/site/dynamic-params-guard.test.ts"
    },
    {
      "id": 59,
      "date": "2026-08-26",
      "kind": "verification-gap",
      "title": "A field that may contain markup is safe only where it is rendered as markup, and the sink no human ever looks at is the one that keeps shipping the defect",
      "mechanism": "__tests__/site/faq-rendered-markdown.test.tsx"
    },
    {
      "id": 60,
      "date": "2026-08-26",
      "kind": "verification-gap",
      "title": "A proximity window measured in lines stops measuring anything once a generator writes the file, and a framing verb inside a citation is not a claim of affiliation",
      "mechanism": "__tests__/scripts/check-advisor-name-leak.test.ts"
    },
    {
      "id": 61,
      "date": "2026-08-27",
      "kind": "verification-gap",
      "title": "A test that spawns an entrypoint aware of its build environment inherits the live reporting channel, so a suite can overwrite the run's own verdict and the last write wins",
      "mechanism": "__tests__/scripts/runner-env-isolation.test.ts"
    },
    {
      "id": 62,
      "date": "2026-08-27",
      "kind": "verification-gap",
      "title": "A hand edit to a generated file is one run from silent reversion, and a suite that grades the file instead of a regeneration will never see it",
      "mechanism": "__tests__/scripts/untimed-heading.test.ts"
    },
    {
      "id": 63,
      "date": null,
      "kind": "verification-gap",
      "title": "A guard that reads the wall clock is vacuous at the wrong hour, so deleting the fix it protects can leave it green",
      "mechanism": "__tests__/site/scheduled-content-containment.test.ts"
    },
    {
      "id": 64,
      "date": "2026-09-02",
      "kind": "verification-gap",
      "title": "A corpus gated on the wrong date field publishes a placeholder before the enriched version can exist, and an exemption list written the day a feature lands is the tell",
      "mechanism": "__tests__/site/scheduled-content-containment.test.ts"
    },
    {
      "id": 65,
      "date": "2026-08-30",
      "kind": "external-behavior",
      "title": "A shell capture of a non-string value from a runtime that colorizes its output returns ANSI escape bytes, and the environment that does it is the automation harness rather than a human terminal"
    },
    {
      "id": 66,
      "date": "2026-08-30",
      "kind": "verification-gap",
      "title": "Calendar month arithmetic overflows into the following month, and a test that computes its expectation the same way agrees with the defect instead of catching it",
      "mechanism": "__tests__/api/intelligence-pause.test.ts"
    }
  ]
}