{
  "name": "The Orbyt Failure Corpus",
  "url": "https://www.orbytlabs.ai/orbyt-collective/how-it-works",
  "what": "Documented failure modes from a production codebase operated by AI agents, each one recorded when it bit, classified, and where a countermeasure was mechanized, linked to the countermeasure that now prevents it.",
  "method": "Item ids, the count, and first-documented dates are parsed from the committed corpus file. Public titles and taxonomy kinds are edited for publication from a reviewed map; the generator fails if the map and the corpus ever disagree on which items exist, and fails if a claimed countermeasure path is absent from the repository.",
  "caveat": "This is the operator grading its own mistakes. Early entries predate the dating convention and are published undated rather than backdated. A mechanism path names the countermeasure inside a private repository; the path is published so the claim is specific, even though the file is not.",
  "license": "CC BY 4.0, attribution required",
  "cite_as": "Orbyt Failure Corpus (2026-08-22), https://www.orbytlabs.ai/orbyt-collective/how-it-works",
  "generated_at": "2026-08-22T07:52:25.026Z",
  "count": 58,
  "mechanized_count": 35,
  "dated_count": 48,
  "first_documented": "2026-06-09",
  "latest_documented": "2026-08-21",
  "taxonomy": {
    "external-behavior": {
      "definition": "An external platform, client, or dependency behaved differently than documented or assumed.",
      "count": 14
    },
    "verification-gap": {
      "definition": "A test, guard, gate, or metric that could not fail, could not see, or measured a proxy for the thing it claimed to measure.",
      "count": 26
    },
    "operator-process": {
      "definition": "A discipline failure in how the AI operator built or maintained the system: stale documents, unswept invariants, contracts with one side missing.",
      "count": 18
    }
  },
  "items": [
    {
      "id": 1,
      "date": null,
      "kind": "external-behavior",
      "title": "A mobile browser can suspend a tab without releasing its Web Locks, deadlocking auth refresh",
      "mechanism": "app/layout.tsx"
    },
    {
      "id": 2,
      "date": null,
      "kind": "external-behavior",
      "title": "Realtime sync can echo a write back into itself unless transient UI state stays out of the database"
    },
    {
      "id": 3,
      "date": null,
      "kind": "external-behavior",
      "title": "High-frequency writes to a replicated column can destabilize the replication layer itself"
    },
    {
      "id": 4,
      "date": null,
      "kind": "operator-process",
      "title": "A cleared cache with a surviving session is a distinct state; UI must wait for hydration to complete"
    },
    {
      "id": 5,
      "date": null,
      "kind": "external-behavior",
      "title": "A managed provider's default email path cannot be assumed to deliver; route through one you can observe"
    },
    {
      "id": 6,
      "date": null,
      "kind": "operator-process",
      "title": "Unsafe HTML injection requires an explicit, same-line trust annotation to pass review"
    },
    {
      "id": 7,
      "date": null,
      "kind": "operator-process",
      "title": "A module whose internal name diverges from its export breaks tooling that assumes they match"
    },
    {
      "id": 8,
      "date": null,
      "kind": "operator-process",
      "title": "After a rename, the old name is a liability: redirect it once, then never reference it again"
    },
    {
      "id": 9,
      "date": null,
      "kind": "verification-gap",
      "title": "A date test that mocks the clock in a different timezone convention than the code parses fails only west of UTC"
    },
    {
      "id": 10,
      "date": null,
      "kind": "external-behavior",
      "title": "A client library that auto-deserializes on read breaks every consumer that stored a string, and only on cache hit",
      "mechanism": "app/api/_lib/cache.ts"
    },
    {
      "id": 11,
      "date": "2026-06-09",
      "kind": "operator-process",
      "title": "A programmatic page matrix needs a bounded kept-set imported by both the builder and the sitemap",
      "mechanism": "__tests__/site/salary-data.test.ts"
    },
    {
      "id": 12,
      "date": "2026-06-09",
      "kind": "operator-process",
      "title": "An internal spec date is not a protocol version; clients hard-reject unknown values",
      "mechanism": "__tests__/mcp/protocol-negotiation.test.ts"
    },
    {
      "id": 13,
      "date": "2026-07-10",
      "kind": "operator-process",
      "title": "A redirect parameter is a contract between writer and reader; a silent fallback hides the mismatch"
    },
    {
      "id": 14,
      "date": "2026-07-10",
      "kind": "external-behavior",
      "title": "Hiding overflow on one axis creates a scroll container on the other and can trap wheel events"
    },
    {
      "id": 15,
      "date": "2026-07-11",
      "kind": "operator-process",
      "title": "A client-reachable module that statically imports a heavy data source ships it to every page that renders it"
    },
    {
      "id": 16,
      "date": "2026-07-23",
      "kind": "verification-gap",
      "title": "A test that derives its expected value from the code under test can never fail",
      "mechanism": "scripts/check-log-status-truth.cjs"
    },
    {
      "id": 17,
      "date": "2026-07-23",
      "kind": "operator-process",
      "title": "A pre-authentication code path needs its own abuse ceiling",
      "mechanism": "app/api/v1/intelligence/_lib/route-wrapper.ts"
    },
    {
      "id": 18,
      "date": "2026-07-23",
      "kind": "verification-gap",
      "title": "A table listing's row counts are planner estimates; report a real count or report nothing"
    },
    {
      "id": 19,
      "date": "2026-07-25",
      "kind": "external-behavior",
      "title": "Email clients default to content-box, so full width plus padding overflows a phone"
    },
    {
      "id": 20,
      "date": "2026-07-25",
      "kind": "verification-gap",
      "title": "A CI gate that only runs on the day it matters has never actually been tested; force the condition once"
    },
    {
      "id": 21,
      "date": "2026-07-26",
      "kind": "verification-gap",
      "title": "A convention that has never been executed is not a working convention"
    },
    {
      "id": 22,
      "date": "2026-07-28",
      "kind": "verification-gap",
      "title": "A build filter that decides inclusion is a detector, and must be proven on the shortest legitimate case"
    },
    {
      "id": 23,
      "date": "2026-07-30",
      "kind": "external-behavior",
      "title": "Invalid CSS is discarded, not flagged; a broken value silently becomes the property's initial value"
    },
    {
      "id": 24,
      "date": "2026-07-31",
      "kind": "external-behavior",
      "title": "Read a provider response by scanning for the block you want, never by indexing position zero",
      "mechanism": "scripts/check-ai-response-shape.cjs"
    },
    {
      "id": 25,
      "date": "2026-07-31",
      "kind": "external-behavior",
      "title": "A model's output budget caps reasoning plus text; leave neither half implicit",
      "mechanism": "app/api/_lib/ai-models.ts"
    },
    {
      "id": 26,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A tool that runs quarterly has no gate between quarters, and its hand-copied config drifts silently",
      "mechanism": "__tests__/config/stryker-config-sync.test.ts"
    },
    {
      "id": 27,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A cron reporting success is not healthy; watch duration against its ceiling and reap rows whose process never returned"
    },
    {
      "id": 28,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A timeout reports as cancelled, not failed; alarms keyed on failure alone are structurally blind to it",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 29,
      "date": "2026-07-31",
      "kind": "verification-gap",
      "title": "A metric measured and then discarded is indistinguishable from a metric never measured"
    },
    {
      "id": 30,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A synchronous spawn against an in-process fake server deadlocks, and the hang reads as the child failing"
    },
    {
      "id": 31,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A step that discards stderr turns a missing permission into a silently wrong branch",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 32,
      "date": "2026-08-01",
      "kind": "operator-process",
      "title": "A relabeled invariant must sweep every detector that encodes it, in the same commit",
      "mechanism": "scripts/check-data-integrity.cjs"
    },
    {
      "id": 33,
      "date": "2026-08-01",
      "kind": "external-behavior",
      "title": "Checks sharing one rate-limit window flake each other; absorb exactly one window or serialize",
      "mechanism": "scripts/check-smoke.cjs"
    },
    {
      "id": 34,
      "date": "2026-08-01",
      "kind": "verification-gap",
      "title": "A fix committed but not pushed does not exist, and a guard grading the working tree will certify it as live",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 35,
      "date": "2026-08-02",
      "kind": "verification-gap",
      "title": "An instrument's failure paths must render as not observed, never as clean",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 36,
      "date": "2026-08-02",
      "kind": "operator-process",
      "title": "A lesson mechanized in the canonical pipeline does not exist for the surfaces that bypass it",
      "mechanism": "__tests__/scripts/org-guard-parity.test.ts"
    },
    {
      "id": 37,
      "date": "2026-08-05",
      "kind": "operator-process",
      "title": "A selection rule optimizes its proxy; verify what exists against measured demand, not against the code",
      "mechanism": "scripts/check-salary-demand-coverage.ts"
    },
    {
      "id": 38,
      "date": "2026-08-08",
      "kind": "operator-process",
      "title": "A sync rule that tests equality can never be satisfied by the thing it exists to allow; record provenance instead",
      "mechanism": "__tests__/scripts/sync-i18n-provenance.test.ts"
    },
    {
      "id": 39,
      "date": "2026-08-08",
      "kind": "verification-gap",
      "title": "Never validate a filter against data that filter produced"
    },
    {
      "id": 40,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "A gate whose correctness rests on the thing it is gating is not a gate",
      "mechanism": "packages/collective/actions/org-agent/action.yml"
    },
    {
      "id": 41,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "A job cancelled with zero steps was never handed to a runner; an alarm inside the job cannot see the job never starting",
      "mechanism": "scripts/check-ci-headroom.cjs"
    },
    {
      "id": 42,
      "date": "2026-08-10",
      "kind": "verification-gap",
      "title": "Read a monitor's freshness the way the monitor is actually fed",
      "mechanism": "__tests__/scripts/daily-pulse.test.ts"
    },
    {
      "id": 43,
      "date": "2026-08-10",
      "kind": "operator-process",
      "title": "A stale document reads as instructions; when an invariant reverses, sweep the documents that assert it",
      "mechanism": "scripts/check-root-clutter.cjs"
    },
    {
      "id": 44,
      "date": "2026-08-13",
      "kind": "verification-gap",
      "title": "A sandbox that cannot start reports success, and a fence can apply halfway",
      "mechanism": ".github/workflows/fence-canary.yml"
    },
    {
      "id": 45,
      "date": "2026-08-13",
      "kind": "operator-process",
      "title": "A confinement lane is what an agent is instructed to write, not what gets committed",
      "mechanism": "__tests__/scripts/org-fence.test.ts"
    },
    {
      "id": 46,
      "date": "2026-08-13",
      "kind": "operator-process",
      "title": "A guard's exit code is not its verdict; report-only and strict-gated checks exit zero while reporting"
    },
    {
      "id": 47,
      "date": "2026-08-13",
      "kind": "verification-gap",
      "title": "A gate that reads a tool's printed summary is reading a rendering, and colour in CI can leave it with no way to fail",
      "mechanism": "__tests__/scripts/test-gate.test.ts"
    },
    {
      "id": 48,
      "date": "2026-08-15",
      "kind": "verification-gap",
      "title": "Permissive access policies combine with OR, so one globally true policy defeats every narrow policy beside it, and two individually correct guards can still leave the class unowned",
      "mechanism": "scripts/check-policy-breadth.cjs"
    },
    {
      "id": 49,
      "date": "2026-08-15",
      "kind": "verification-gap",
      "title": "A generated file that measures the repository containing it can never be correct at the moment it is committed, and a drift guard that performs no regeneration of its own reports PASS on a precondition it never established",
      "mechanism": "scripts/check-generated-drift.cjs"
    },
    {
      "id": 50,
      "date": "2026-08-20",
      "kind": "external-behavior",
      "title": "A shallow repository clone makes the version-control history answer confidently and wrongly, so a generator that trusts it publishes a false freshness claim with no error anywhere",
      "mechanism": "scripts/update-skunkworks-dates.cjs"
    },
    {
      "id": 51,
      "date": "2026-08-20",
      "kind": "external-behavior",
      "title": "A privileged key does not bypass a control the upstream service enforces, and charging that service's refusal to a user's abuse budget locks out the innocent",
      "mechanism": "__tests__/api/mobile-login.test.ts"
    },
    {
      "id": 52,
      "date": "2026-08-20",
      "kind": "operator-process",
      "title": "An upstream service reports a downstream failure in its own vocabulary, so the error names a cause that is not the real one, and four consecutive fixes were aimed at the wrong side of the wire",
      "mechanism": "scripts/check-auth-hook-health.cjs"
    },
    {
      "id": 53,
      "date": "2026-08-21",
      "kind": "operator-process",
      "title": "Internal documentation is not a wire contract: a set of event names transcribed from reviewed internal docs matched nothing the sender actually emits, and a single observed value disagreeing with a document invalidates the whole document rather than the one row it lands on",
      "mechanism": "__tests__/api/auth-send-email-hook.test.ts"
    },
    {
      "id": 54,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "A monitoring query that samples without filtering reports the absence of the needle as the absence of the problem, and sorting newest-first does not fix it; a reported count equal to the page size is never a measurement",
      "mechanism": "__tests__/scripts/check-auth-hook-health.test.ts"
    },
    {
      "id": 55,
      "date": "2026-08-21",
      "kind": "external-behavior",
      "title": "A dependency's implicit fetch option is a route-classification decision, and it can silently repeal a page's own segment config, turning a declared 404 into a streamed 200 soft-404 that only the deployed build exhibits",
      "mechanism": "scripts/check-blog-route-static.cjs"
    },
    {
      "id": 56,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "An allowlist entry is a claim about production that decays silently, and a guard checking whether a value is declared cannot go loud about the value that actually costs money",
      "mechanism": "__tests__/site/dynamic-params-guard.test.ts"
    },
    {
      "id": 57,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "A verification that normalizes its input passes on a value the real consumer will reject, so an instrument must read its subject exactly the way the consumer reads it"
    },
    {
      "id": 58,
      "date": "2026-08-21",
      "kind": "verification-gap",
      "title": "A cost measurement is valid only for the resource it measured on the machine that binds it, and a local build stops one whole stage before the one that fails",
      "mechanism": "__tests__/site/dynamic-params-guard.test.ts"
    }
  ]
}