{ "recordVersion": 3, "suites": [ { "id": "controlled-regression", "version": 1, "review": "AI_AUTHORED_AI_READ", "authorModelFamily": "claude", "reviews": [ { "kind": "BLIND_READ", "reviewer": "Verkstaden orchestrator session (Claude Opus 5.5), blind read", "modelFamily": "claude", "reviewedOn": "2026-09-23", "seed": 90231, "packetSha256": "4e50c78db1dee0c91f008728789feef8e3bedef8cc34bffc90d1b7492d2bc3a3", "scope": "SUITE", "agreed": 22, "of": 22, "disagreements": [], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "covers": [ { "familyId": "regression-direct-application", "familyVersion": 1 }, { "familyId": "regression-recorded-step", "familyVersion": 1 }, { "familyId": "regression-conditional-duty", "familyVersion": 1 }, { "familyId": "regression-near-miss", "familyVersion": 1 }, { "familyId": "regression-unsupported-claim", "familyVersion": 1 }, { "familyId": "regression-irrelevant-rule", "familyVersion": 1 }, { "familyId": "regression-attribute-invariance", "familyVersion": 1 }, { "familyId": "regression-unconditional-duty", "familyVersion": 1 } ] }, { "kind": "BLIND_READ", "reviewer": "gemini-2.5-pro (Vertex europe-west1), blind read", "modelFamily": "gemini", "reviewedOn": "2026-10-04", "seed": 30500, "packetSha256": "9a34fadbe97b37aa3cdab4e4df2a30ad304e544bdb71a6850a4e3b32becfec9a", "scope": "SUITE", "agreed": 19, "of": 22, "disagreements": [ { "item": "item-06", "summary": "The reader found the closure entry's missing statement of where its papers are stored, as declared, but placed it under the \"Closing note\" heading; the key places it under no section.", "resolution": "Genuinely ambiguous: the rule attaches the duty to the whole entry, but a closing note is a natural home for it, and the packet does not say which. Kept as declared; #3082 sharpens the case." }, { "item": "item-07", "summary": "The reader agreed nothing is due for the sample run from a tap, but gave the reason as a condition not triggered; the key declares the sample out of the rule's scope.", "resolution": "Genuinely ambiguous: the packet names the four reasons without defining them, and the rule's wording reads both as a scope limit and as an unmet condition. Kept as declared; #3083 sharpens the reasons." }, { "item": "item-17", "summary": "The reader found the levy charged from a figure with no named document, as declared, but quoted the sentence that sets the levy; the key anchors the flaw on the sentence that states the figure.", "resolution": "Reader wrong: the rule's duty attaches to the figure (\"a figure whose document is not named\"), and the packet anchors an unsupported claim on the claim itself, which here is the figure, not the charge set from it." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "covers": [ { "familyId": "regression-direct-application", "familyVersion": 1 }, { "familyId": "regression-recorded-step", "familyVersion": 1 }, { "familyId": "regression-conditional-duty", "familyVersion": 1 }, { "familyId": "regression-near-miss", "familyVersion": 1 }, { "familyId": "regression-unsupported-claim", "familyVersion": 1 }, { "familyId": "regression-irrelevant-rule", "familyVersion": 1 }, { "familyId": "regression-attribute-invariance", "familyVersion": 1 }, { "familyId": "regression-unconditional-duty", "familyVersion": 1 } ] } ], "crossFamilyReviewed": true, "workerInstructions": "You read one document and apply only the rules supplied to you in the legal framework, exactly as each rule is written. Report every place where the document does not meet a supplied rule, and apply what a supplied rule says about how a record that meets it is to be recorded. Quote each passage exactly as it appears. Do not apply a rule that was not supplied, and do not rely on any knowledge of a legal system.", "families": [ { "id": "regression-direct-application", "version": 1, "title": "A supplied rule applied directly, and the same rule visibly met", "hypothesis": "The engine reports a repair described as cured when a supplied rule requires the entry to name who checked the cure on site and it names nobody, and reports nothing when the entry names them.", "evidenceClass": "REGRESSION", "capabilityTags": [ "rule-application", "clean-specificity" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [ { "item": "p4 (cure-checker-named)", "summary": "KEY_WEAK, DIFFICULTY: The document's \"is described as cured\" repeats rule A's \"describes a fault as cured\", and the contrast is decided by whether a personal name follows it. An engine could pass both sides by matching the phrase and looking for a name, without reading that the person must have checked the cure. A decoy variant naming someone who did something else would separate understanding from matching.", "resolution": "KEPT: The suite is a change gate whose shapes and size are fixed to the pilot's measured cost (REGRESSION_CORPUS.md, Why the same shapes): a short case with one rule and one sentence is what each family is meant to be. Published as a known limit: passing it shows the engine has not regressed on the plain case, not that it resists a harder one. A decoy variant is a new case with its own outcome, which belongs to a new family." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-direct-application.json", "covers": [ { "familyId": "regression-direct-application", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-direct-application.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": "cure-unchecked", "item": "cure-claimed-without-checker", "rules": [ "A" ], "question": "CORRECTNESS", "severity": "NOTE", "argument": "\"The leak is described as cured.\" is passive and names no speaker. A strict reader could hold that the entry reports someone else's description rather than itself describing the fault as cured, and rule A is triggered only by an entry that describes a fault as cured.", "resolution": { "status": "KEPT", "reason": "The sentence stands under the entry's own heading \"State after the work\" without qualification, so the record adopts the description as its account of the state. Reading rule A as untriggered by the passive voice would let any entry escape it by grammar.", "followUp": null } }, { "id": "p2", "variantId": "cure-unchecked", "item": "cure-unchecked-truth", "rules": [ "A" ], "question": "COMPLETENESS", "severity": "NOTE", "argument": "The work sentence names \"the maintenance crew\", not a person. Is that a second breach the key does not list?", "resolution": { "status": "KEPT", "reason": "Rule A attaches its duty to the claim that a fault is cured, and asks for the person who checked the cure, not for who did the work. The crew sentence makes no cure claim, so the rule is offended once, as the key says.", "followUp": null } }, { "id": "p3", "variantId": "cure-checked", "item": "cure-checked-truth", "rules": [ "A" ], "question": "AMBIGUITY", "severity": "NOTE", "argument": "In \"and fitter O. Lindgren checked it on site\" the word \"it\" could refer to the leak rather than to the cure, and an initial with a surname might be argued not to name a person.", "resolution": { "status": "KEPT", "reason": "The clause belongs to the sentence that claims the cure, and checking the leak on site after the repair is checking whether it is cured. An initial, a surname and a role identify one person in a record. The clean truth holds.", "followUp": null } }, { "id": "p4", "variantId": null, "item": "cure-checker-named", "rules": [ "A" ], "question": "DIFFICULTY", "severity": "KEY_WEAK", "argument": "The document's \"is described as cured\" repeats rule A's \"describes a fault as cured\", and the contrast is decided by whether a personal name follows it. An engine could pass both sides by matching the phrase and looking for a name, without reading that the person must have checked the cure. A decoy variant naming someone who did something else would separate understanding from matching.", "resolution": { "status": "KEPT", "reason": "The suite is a change gate whose shapes and size are fixed to the pilot's measured cost (REGRESSION_CORPUS.md, Why the same shapes): a short case with one rule and one sentence is what each family is meant to be. Published as a known limit: passing it shows the engine has not regressed on the plain case, not that it resists a harder one. A decoy variant is a new case with its own outcome, which belongs to a new family.", "followUp": null } }, { "id": "p5", "variantId": "cure-unchecked", "item": "cure-claimed-without-checker", "rules": [ "A" ], "question": "ANCHORING", "severity": "NOTE", "argument": "The omission is anchored on the cure sentence. An engine that anchors it on the heading \"State after the work\" or on the repair sentence is scored as a miss.", "resolution": { "status": "KEPT", "reason": "The production prompt anchors a claim that lacks a required statement on that claim, and the cure sentence is the claim rule A attaches to. A wider quote on the same line is accepted; another passage is not the claim.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "256bf8b57b773160d4d1cf920fdfad3e90f738938304244ede0833cb22926f1d", "rules": [ { "label": "A", "name": "Naming who checked a cure on site", "source": "Entry Practice Schedule, part K, issue 1", "text": "In a record, every repair entry that describes a fault as cured must also name the person who checked the cure on site. The duty rests on the author of the record.", "ruleKeySha256": "6761e31c80e5ec7e69168557d00e3c3955e273d15638b700ffc8dad9a1b5a866" } ], "variants": [ { "id": "cure-unchecked", "description": "The leak is described as cured and nobody is named as having checked it.", "treatment": { "comparisonId": "cure-checker-named", "factor": "named person who checked the cure on site", "level": "absent" }, "rules": [ "A" ], "document": "Repair entry RP-31\nFault reported\nA leak was reported at the sluice valve on the east channel.\nWork carried out\nThe valve packing was replaced by the maintenance crew on 14 October 2032.\nState after the work\nThe leak is described as cured.\n", "renderedTextSha256": "30e7613e46f0728f26f25cbc29c4f7dba95015cfb20910ebd02622c7b1091b0f", "satisfiedRules": [], "answerKey": { "id": "cure-unchecked-truth", "description": "The entry calls the fault cured and names nobody who checked it, so the rule is offended once.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "cure-claimed-without-checker", "valence": "FLAW", "description": "The entry describes the leak as cured without naming who checked the cure on site.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The leak is described as cured." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "unchecked-cure-praised", "valence": "STRENGTH", "description": "Crediting the same sentence for naming a checker when it names none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The leak is described as cured." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-09", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The leak is described as cured.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-19", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The leak is described as cured.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "cure-checked", "description": "The same sentence, naming the fitter who checked the cure on site.", "treatment": { "comparisonId": "cure-checker-named", "factor": "named person who checked the cure on site", "level": "present" }, "rules": [ "A" ], "document": "Repair entry RP-31\nFault reported\nA leak was reported at the sluice valve on the east channel.\nWork carried out\nThe valve packing was replaced by the maintenance crew on 14 October 2032.\nState after the work\nThe leak is described as cured, and fitter O. Lindgren checked it on site.\n", "renderedTextSha256": "c55f8592de9cc104c9d1e05982c3215aee3735fd943f672414caa66b6329d3a4", "satisfiedRules": [ { "rule": "A", "passage": "The leak is described as cured, and fitter O. Lindgren checked it on site." } ], "answerKey": { "id": "cure-checked-truth", "description": "The same claim, naming who checked the cure on site. The rule is met and nothing is due.", "exhaustive": true, "noFindingReason": "COMPLIANT", "expectedFindings": [], "prohibitedFindings": [ { "id": "false-unchecked-cure", "valence": "FLAW", "description": "Reporting the cure as unchecked when the sentence names who checked it is a false positive.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The leak is described as cured, and fitter O. Lindgren checked it on site." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-05", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The leak is described as cured, and fitter O. Lindgren checked it on site." } ], "noFindingReason": "COMPLIANT" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-22", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The leak is described as cured, and fitter O. Lindgren checked it on site." } ], "noFindingReason": "COMPLIANT" } ] } ], "comparisons": [ { "id": "cure-checker-named", "kind": "CONTRAST", "leftVariantId": "cure-unchecked", "rightVariantId": "cure-checked", "factor": "named person who checked the cure on site", "leftLevel": "absent", "rightLevel": "present", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-recorded-step", "version": 1, "title": "A required step recorded with its detail, and the same step left bare", "hypothesis": "The engine credits a seal examination the entry grounds in each seal's number and state, and reports the omission when the same entry says the seals were examined and gives neither.", "evidenceClass": "REGRESSION", "capabilityTags": [ "strength-recognition", "omission-detection" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [ { "item": "p1 (seal-detail-given, seals-numbered-and-stated)", "summary": "KEY_WEAK, ANCHORING: The expected strength is anchored on \"Seal 30817 and seal 30818 were both found unbroken.\" alone. The examination rule A credits is stated across two lines: the claim \"The seals on the trailer were examined at the gate.\" and the detail below it. A strength quoting the examination sentence, or both sentences, is as correct as the key's, but the scorer accepts a wider quote only on the anchor's own line, so either answer is scored as a missed target and an unsupported strength.", "resolution": "ACCEPTED: A correct answer must not be scored as two errors. The family is corrected under a new version in which the examination and its detail are one sentence the strength can be anchored on. Follow-up #3074." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-recorded-step.json", "covers": [ { "familyId": "regression-recorded-step", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-recorded-step.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": "seal-detail-given", "item": "seals-numbered-and-stated", "rules": [ "A" ], "question": "ANCHORING", "severity": "KEY_WEAK", "argument": "The expected strength is anchored on \"Seal 30817 and seal 30818 were both found unbroken.\" alone. The examination rule A credits is stated across two lines: the claim \"The seals on the trailer were examined at the gate.\" and the detail below it. A strength quoting the examination sentence, or both sentences, is as correct as the key's, but the scorer accepts a wider quote only on the anchor's own line, so either answer is scored as a missed target and an unsupported strength.", "resolution": { "status": "ACCEPTED", "reason": "A correct answer must not be scored as two errors. The family is corrected under a new version in which the examination and its detail are one sentence the strength can be anchored on.", "followUp": "#3074" } }, { "id": "p2", "variantId": "seal-detail-given", "item": "seal-detail-given-truth", "rules": [ "A" ], "question": "COMPLETENESS", "severity": "NOTE", "argument": "Rule A asks for \"the number of each seal\". The entry never says how many seals the trailer carried, so it cannot show that two numbers are all of them.", "resolution": { "status": "KEPT", "reason": "The entry names two seals and says \"both\", which closes the set in its own words, and nothing in the document suggests a third. Demanding a count beyond that would read a requirement into rule A that its text does not hold.", "followUp": null } }, { "id": "p3", "variantId": "seal-detail-missing", "item": "seal-detail-omitted", "rules": [ "A" ], "question": "CORRECTNESS", "severity": "NOTE", "argument": "The bare entry does record the examination. Is the fault an omitted step, or a step recorded without its detail?", "resolution": { "status": "KEPT", "reason": "Rule A attaches its duty to the claim that the seals were examined and asks that claim to carry the numbers and states. The claim is present and the statements it owes are absent, which is the omission shape the production prompt anchors on the claim.", "followUp": null } }, { "id": "p4", "variantId": null, "item": "seal-detail-presence", "rules": [ "A" ], "question": "VALIDITY", "severity": "NOTE", "argument": "The strength is due only because rule A ends by saying such an entry \"is to be credited\". The case tests whether the engine follows an explicit instruction to credit, not whether it recognises a well-grounded step unprompted.", "resolution": { "status": "KEPT", "reason": "That is the suite's deliberate rule: a strength is due only under a rule whose text says so (REGRESSION_CORPUS.md, Authoring decisions), so that a clean document is never punished for an unasked credit. The hypothesis is about crediting under such a rule.", "followUp": null } }, { "id": "p5", "variantId": null, "item": "seal-detail-presence", "rules": [], "question": "DIFFICULTY", "severity": "NOTE", "argument": "Seal numbers are the only digits in the examination section, so their presence decides both sides of the contrast.", "resolution": { "status": "KEPT", "reason": "The suite is a change gate whose shapes and size are fixed to the pilot's measured cost (REGRESSION_CORPUS.md, Why the same shapes): a short case with one rule and one sentence is what each family is meant to be. Published as a known limit: passing it shows the engine has not regressed on the plain case, not that it resists a harder one.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "ee4f1c80fc60c7ac17feddb0303f35f92e1fdf286dfbd6c7b56fbb685065d2da", "rules": [ { "label": "A", "name": "Examining the seals on a load", "source": "Entry Practice Schedule, part L, issue 1", "text": "In a record, every seal entry that says the seals on a load were examined must also give the number of each seal and say whether it was found unbroken. The duty rests on the author of the record. An entry that gives both is to be credited as soundly supported in this respect.", "ruleKeySha256": "332375509a8078159cf95e0cc3a2479d9248fc4178c5af0762ba838646c75ff1" } ], "variants": [ { "id": "seal-detail-given", "description": "The entry gives each seal's number and says both were unbroken.", "treatment": { "comparisonId": "seal-detail-presence", "factor": "seal numbers and state after the examination", "level": "present" }, "rules": [ "A" ], "document": "Seal entry SE-208\nConsignment at the gate\nA trailer of boxed pipe fittings arrived at the north gate.\nSeal examination\nThe seals on the trailer were examined at the gate.\nSeal 30817 and seal 30818 were both found unbroken.\nRelease to the yard\nThe trailer was released to the holding yard.\n", "renderedTextSha256": "94fbb4219398f3f7ae811e9f7f4906e21cd2f1050952f70e24f6e8e9e0de8b51", "satisfiedRules": [], "answerKey": { "id": "seal-detail-given-truth", "description": "The examination is recorded with each seal's number and state, which the rule says is to be credited, and no flaw.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "seals-numbered-and-stated", "valence": "STRENGTH", "description": "The entry gives the number of each seal and says each was unbroken, which the rule says is to be credited.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "Seal 30817 and seal 30818 were both found unbroken." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "false-bare-seal-examination", "valence": "FLAW", "description": "Reporting the examination as missing its detail when the entry gives it is a false positive.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The seals on the trailer were examined at the gate." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-16", "agrees": true, "findings": [ { "valence": "STRENGTH", "rule": "A", "isOmission": null, "quote": "Seal 30817 and seal 30818 were both found unbroken.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-13", "agrees": true, "findings": [ { "valence": "STRENGTH", "rule": "A", "isOmission": null, "quote": "Seal 30817 and seal 30818 were both found unbroken.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "seal-detail-missing", "description": "The same examination, with neither detail given.", "treatment": { "comparisonId": "seal-detail-presence", "factor": "seal numbers and state after the examination", "level": "absent" }, "rules": [ "A" ], "document": "Seal entry SE-208\nConsignment at the gate\nA trailer of boxed pipe fittings arrived at the north gate.\nSeal examination\nThe seals on the trailer were examined at the gate.\nRelease to the yard\nThe trailer was released to the holding yard.\n", "renderedTextSha256": "5e200539a530aabe4cabc9c5c02c258878631203098e4b38fa254eab959598c3", "satisfiedRules": [], "answerKey": { "id": "seal-detail-missing-truth", "description": "The same examination claimed, with neither the seal numbers nor their state the rule requires.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "seal-detail-omitted", "valence": "FLAW", "description": "The entry says the seals were examined and gives no seal number and no state.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The seals on the trailer were examined at the gate." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "bare-seal-examination-praised", "valence": "STRENGTH", "description": "Crediting the bare sentence that the seals were examined, when the rule asks for their numbers and state.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The seals on the trailer were examined at the gate." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-03", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The seals on the trailer were examined at the gate.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-15", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The seals on the trailer were examined at the gate.", "locator": null } ], "metRules": [], "noFindingReason": null } ] } ], "comparisons": [ { "id": "seal-detail-presence", "kind": "CONTRAST", "leftVariantId": "seal-detail-given", "rightVariantId": "seal-detail-missing", "factor": "seal numbers and state after the examination", "leftLevel": "present", "rightLevel": "absent", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-recorded-step", "version": 2, "title": "A required step recorded with its detail, and the same step left bare", "hypothesis": "The engine credits a seal examination whose sentence gives each seal's number and state, and reports the omission when the same entry says the seals were examined and gives neither.", "evidenceClass": "REGRESSION", "capabilityTags": [ "strength-recognition", "omission-detection" ], "provenance": { "authoredBy": "Claude Project thread session for #3074", "authoredOn": "2026-10-05", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind. Version 2 corrects version 1 for #3074: its credited strength was anchored on the detail line alone, so a strength quoting the examination sentence the rule attaches to was scored as a miss. Version 2 gives each seal's number and state in that sentence, one claim on one line." }, "authorModelFamily": "claude", "reviews": [ { "kind": "BLIND_READ", "reviewer": "Claude Project thread subagent for #3074, blind read", "modelFamily": "claude", "reviewedOn": "2026-10-05", "seed": 10883, "packetSha256": "9323bf718398a0fa83d1c4a5ad53b9e313ec9dd01892ac0f733b29fb3030bff8", "scope": "FAMILY", "agreed": 2, "of": 2, "disagreements": [], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-recorded-step-v2/blind-read-10883/answers.json", "covers": [ { "familyId": "regression-recorded-step", "familyVersion": 2 } ] }, { "kind": "BLIND_READ", "reviewer": "gemini-2.5-pro (Vertex europe-west1), blind read", "modelFamily": "gemini", "reviewedOn": "2026-10-05", "seed": 47310, "packetSha256": "89767362cbbc6577211e80c23aba80a63783ffc1288231512061231f469116e3", "scope": "FAMILY", "agreed": 1, "of": 2, "disagreements": [ { "item": "item-01", "summary": "The reader credited the examination sentence as the strength, on the key's own words, and also listed the rule as met by that same sentence; the key declares no met rule beside the strength.", "resolution": "Reader over-reported, key kept: the packet says a rule met by nothing but the words already credited as a strength needs no met-rule entry, and the reader's strength, rule and quote are the key's." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-recorded-step-v2/blind-read-47310-gemini/answers.json", "covers": [ { "familyId": "regression-recorded-step", "familyVersion": 2 } ] } ], "critiques": [], "crossFamilyReviewed": true, "documentTemplateSha256": "08c7bc072c1b3622302eb9bd864a86ec73478cb663e04910b9882fdfd8649325", "rules": [ { "label": "A", "name": "Examining the seals on a load", "source": "Entry Practice Schedule, part L, issue 1", "text": "In a record, every seal entry that says the seals on a load were examined must also give the number of each seal and say whether it was found unbroken. The duty rests on the author of the record. An entry that gives both is to be credited as soundly supported in this respect.", "ruleKeySha256": "332375509a8078159cf95e0cc3a2479d9248fc4178c5af0762ba838646c75ff1" } ], "variants": [ { "id": "seal-detail-given", "description": "The examination sentence gives each seal's number and says both were unbroken.", "treatment": { "comparisonId": "seal-detail-presence", "factor": "seal numbers and state in the examination sentence", "level": "present" }, "rules": [ "A" ], "document": "Seal entry SE-208\nConsignment at the gate\nA trailer of boxed pipe fittings arrived at the north gate.\nSeal examination\nThe seals on the trailer, numbers 30817 and 30818, were examined at the gate and both found unbroken.\nRelease to the yard\nThe trailer was released to the holding yard.\n", "renderedTextSha256": "ca5b99b34f9150ce7e9d9768a9a0ecf984f115eec61f463713ce8819572f74ff", "satisfiedRules": [], "answerKey": { "id": "seal-detail-given-truth", "description": "The examination is recorded in one sentence with each seal's number and state, which the rule says is to be credited, and no flaw.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "seals-numbered-and-stated", "valence": "STRENGTH", "description": "The sentence recording the examination gives the number of each seal and says each was unbroken, which the rule says is to be credited.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The seals on the trailer, numbers 30817 and 30818, were examined at the gate and both found unbroken." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "false-bare-seal-examination", "valence": "FLAW", "description": "Reporting the examination as missing its detail when its sentence gives it is a false positive.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The seals on the trailer, numbers 30817 and 30818, were examined at the gate and both found unbroken." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-recorded-step-v2/blind-read-10883/answers.json", "item": "item-02", "agrees": true, "findings": [ { "valence": "STRENGTH", "rule": "A", "isOmission": null, "quote": "The seals on the trailer, numbers 30817 and 30818, were examined at the gate and both found unbroken.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-recorded-step-v2/blind-read-47310-gemini/answers.json", "item": "item-01", "agrees": false, "findings": [ { "valence": "STRENGTH", "rule": "A", "isOmission": null, "quote": "The seals on the trailer, numbers 30817 and 30818, were examined at the gate and both found unbroken.", "locator": null } ], "metRules": [ { "rule": "A", "passage": "The seals on the trailer, numbers 30817 and 30818, were examined at the gate and both found unbroken." } ], "noFindingReason": null } ] }, { "id": "seal-detail-missing", "description": "The same examination, with neither detail given.", "treatment": { "comparisonId": "seal-detail-presence", "factor": "seal numbers and state in the examination sentence", "level": "absent" }, "rules": [ "A" ], "document": "Seal entry SE-208\nConsignment at the gate\nA trailer of boxed pipe fittings arrived at the north gate.\nSeal examination\nThe seals on the trailer were examined at the gate.\nRelease to the yard\nThe trailer was released to the holding yard.\n", "renderedTextSha256": "5e200539a530aabe4cabc9c5c02c258878631203098e4b38fa254eab959598c3", "satisfiedRules": [], "answerKey": { "id": "seal-detail-missing-truth", "description": "The same examination claimed, with neither the seal numbers nor their state the rule requires.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "seal-detail-omitted", "valence": "FLAW", "description": "The entry says the seals were examined and gives no seal number and no state.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The seals on the trailer were examined at the gate." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "bare-seal-examination-praised", "valence": "STRENGTH", "description": "Crediting the bare sentence that the seals were examined, when the rule asks for their numbers and state.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The seals on the trailer were examined at the gate." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-recorded-step-v2/blind-read-10883/answers.json", "item": "item-01", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The seals on the trailer were examined at the gate.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-recorded-step-v2/blind-read-47310-gemini/answers.json", "item": "item-02", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The seals on the trailer were examined at the gate.", "locator": null } ], "metRules": [], "noFindingReason": null } ] } ], "comparisons": [ { "id": "seal-detail-presence", "kind": "CONTRAST", "leftVariantId": "seal-detail-given", "rightVariantId": "seal-detail-missing", "factor": "seal numbers and state in the examination sentence", "leftLevel": "present", "rightLevel": "absent", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-conditional-duty", "version": 1, "title": "A duty that arises only from a refusal, with and without that refusal", "hypothesis": "The engine reports missing grounds when the entry says a request was refused, and reports nothing when the same request was granted.", "evidenceClass": "REGRESSION", "capabilityTags": [ "conditional-duty", "trigger-discrimination" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [ { "item": "p1 (request-granted, request-granted-truth)", "summary": "KEY_WEAK, DIFFICULTY: Rule A's second sentence says outright that \"An allocation entry that grants the request owes no such statement.\" The clean variant is the very case that sentence names, so an engine passes it by reading an explicit exemption, not by working out that a conditional duty is untriggered, which is what the family's trigger-discrimination tag claims.", "resolution": "ACCEPTED: The suite's only untriggered-condition case should be decided by the condition. The family is corrected under a new version whose rule names no exempt case. Follow-up #3077." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-conditional-duty.json", "covers": [ { "familyId": "regression-conditional-duty", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-conditional-duty.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": "request-granted", "item": "request-granted-truth", "rules": [ "A" ], "question": "DIFFICULTY", "severity": "KEY_WEAK", "argument": "Rule A's second sentence says outright that \"An allocation entry that grants the request owes no such statement.\" The clean variant is the very case that sentence names, so an engine passes it by reading an explicit exemption, not by working out that a conditional duty is untriggered, which is what the family's trigger-discrimination tag claims.", "resolution": { "status": "ACCEPTED", "reason": "The suite's only untriggered-condition case should be decided by the condition. The family is corrected under a new version whose rule names no exempt case.", "followUp": "#3077" } }, { "id": "p2", "variantId": "request-refused", "item": "refusal-grounds-absent", "rules": [ "A" ], "question": "COMPLETENESS", "severity": "NOTE", "argument": "\"The club secretary was sent the outcome by letter.\" The letter may well have stated the grounds.", "resolution": { "status": "KEPT", "reason": "Rule A requires the entry to say which requirement failed. What a letter outside the entry said is not in the entry, so the duty is unmet whatever the letter held.", "followUp": null } }, { "id": "p3", "variantId": null, "item": "refusal-trigger", "rules": [ "A" ], "question": "DIFFICULTY", "severity": "NOTE", "argument": "The flawed side's \"was refused\" repeats rule A's \"says a request was refused\" word for word.", "resolution": { "status": "KEPT", "reason": "The suite is a change gate whose shapes and size are fixed to the pilot's measured cost (REGRESSION_CORPUS.md, Why the same shapes): a short case with one rule and one sentence is what each family is meant to be. Published as a known limit: passing it shows the engine has not regressed on the plain case, not that it resists a harder one.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "2c517ff4edf9080f50bac17a865f6c6689cfee99fea5ea79078ad54edc80af04", "rules": [ { "label": "A", "name": "Grounds for refusing a request", "source": "Entry Practice Schedule, part M, issue 1", "text": "In a record, every allocation entry that says a request was refused must also say which requirement the request failed to meet. An allocation entry that grants the request owes no such statement. The duty rests on the author of the record.", "ruleKeySha256": "87e2bd35523ebe653cada223d5a40ae15633e96ad1338963002a418b78ad53be" } ], "variants": [ { "id": "request-refused", "description": "The request is refused and no unmet requirement is named.", "treatment": { "comparisonId": "refusal-trigger", "factor": "outcome of the mooring request", "level": "refused" }, "rules": [ "A" ], "document": "Allocation entry AL-115\nMooring request\nThe rowing club asked for a second mooring at the west quay.\nAllocation outcome\nThe request for a second mooring was refused.\nNotice to the club\nThe club secretary was sent the outcome by letter.\n", "renderedTextSha256": "ed9029ebae381306e27c3188c9fb1d9c3ad220a05a300133a2dc6323511b6577", "satisfiedRules": [], "answerKey": { "id": "refusal-without-grounds-truth", "description": "The entry says the request was refused and names no unmet requirement, so the duty is triggered and unmet.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "refusal-grounds-absent", "valence": "FLAW", "description": "The entry says the request was refused and names no requirement the request failed to meet.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The request for a second mooring was refused." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "refusal-praised-as-grounded", "valence": "STRENGTH", "description": "Crediting the refusal sentence as grounded when no requirement is named.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The request for a second mooring was refused." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-12", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The request for a second mooring was refused.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-20", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The request for a second mooring was refused.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "request-granted", "description": "The request is granted, so the duty is never triggered.", "treatment": { "comparisonId": "refusal-trigger", "factor": "outcome of the mooring request", "level": "granted" }, "rules": [ "A" ], "document": "Allocation entry AL-115\nMooring request\nThe rowing club asked for a second mooring at the west quay.\nAllocation outcome\nThe request for a second mooring was granted.\nNotice to the club\nThe club secretary was sent the outcome by letter.\n", "renderedTextSha256": "ec682e35f7398e7f20f8322766220d9584c2bcf2b1cd7a8810852b0698877180", "satisfiedRules": [], "answerKey": { "id": "request-granted-truth", "description": "The request was granted, so the duty never arises and nothing is due.", "exhaustive": true, "noFindingReason": "CONDITION_NOT_TRIGGERED", "expectedFindings": [], "prohibitedFindings": [ { "id": "grounds-demanded-for-a-grant", "valence": "FLAW", "description": "Demanding grounds for a refusal the entry does not state is the rule applied past its trigger.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The request for a second mooring was granted." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-07", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "CONDITION_NOT_TRIGGERED" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-10", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "CONDITION_NOT_TRIGGERED" } ] } ], "comparisons": [ { "id": "refusal-trigger", "kind": "CONTRAST", "leftVariantId": "request-refused", "rightVariantId": "request-granted", "factor": "outcome of the mooring request", "leftLevel": "refused", "rightLevel": "granted", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-conditional-duty", "version": 2, "title": "A duty that arises only from a refusal, with and without that refusal", "hypothesis": "The engine reports missing grounds when the entry says a request was refused, and reports nothing when the same request was granted.", "evidenceClass": "REGRESSION", "capabilityTags": [ "conditional-duty", "trigger-discrimination" ], "provenance": { "authoredBy": "Claude Project thread session for #3077", "authoredOn": "2026-10-05", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind. Version 2 corrects version 1 for #3077: rule A no longer says that an entry granting the request owes no statement, so the granted side is clean because the duty's trigger, a refusal, is absent, not because the rule writes the exemption out." }, "authorModelFamily": "claude", "reviews": [ { "kind": "BLIND_READ", "reviewer": "gemini-2.5-pro (Vertex europe-west1), blind read", "modelFamily": "gemini", "reviewedOn": "2026-10-05", "seed": 30770, "packetSha256": "fe7e2de50b98834e437a52091cc0aa3d40dccc8d13cf3059a856f54c5556b9fc", "scope": "FAMILY", "agreed": 2, "of": 2, "disagreements": [], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-conditional-duty-v2/blind-read-30770-gemini/answers.json", "covers": [ { "familyId": "regression-conditional-duty", "familyVersion": 2 } ] } ], "critiques": [], "crossFamilyReviewed": true, "documentTemplateSha256": "2c517ff4edf9080f50bac17a865f6c6689cfee99fea5ea79078ad54edc80af04", "rules": [ { "label": "A", "name": "Grounds for refusing a request", "source": "Entry Practice Schedule, part M, issue 1", "text": "In a record, every allocation entry that says a request was refused must also say which requirement the request failed to meet. The duty rests on the author of the record.", "ruleKeySha256": "87e2bd35523ebe653cada223d5a40ae15633e96ad1338963002a418b78ad53be" } ], "variants": [ { "id": "request-refused", "description": "The request is refused and no unmet requirement is named.", "treatment": { "comparisonId": "refusal-trigger", "factor": "outcome of the mooring request", "level": "refused" }, "rules": [ "A" ], "document": "Allocation entry AL-115\nMooring request\nThe rowing club asked for a second mooring at the west quay.\nAllocation outcome\nThe request for a second mooring was refused.\nNotice to the club\nThe club secretary was sent the outcome by letter.\n", "renderedTextSha256": "ed9029ebae381306e27c3188c9fb1d9c3ad220a05a300133a2dc6323511b6577", "satisfiedRules": [], "answerKey": { "id": "refusal-without-grounds-truth", "description": "The entry says the request was refused and names no unmet requirement, so the duty is triggered and unmet.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "refusal-grounds-absent", "valence": "FLAW", "description": "The entry says the request was refused and names no requirement the request failed to meet.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The request for a second mooring was refused." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "refusal-praised-as-grounded", "valence": "STRENGTH", "description": "Crediting the refusal sentence as grounded when no requirement is named.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The request for a second mooring was refused." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-conditional-duty-v2/blind-read-30770-gemini/answers.json", "item": "item-01", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The request for a second mooring was refused.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "request-granted", "description": "The request is granted, so the duty is never triggered.", "treatment": { "comparisonId": "refusal-trigger", "factor": "outcome of the mooring request", "level": "granted" }, "rules": [ "A" ], "document": "Allocation entry AL-115\nMooring request\nThe rowing club asked for a second mooring at the west quay.\nAllocation outcome\nThe request for a second mooring was granted.\nNotice to the club\nThe club secretary was sent the outcome by letter.\n", "renderedTextSha256": "ec682e35f7398e7f20f8322766220d9584c2bcf2b1cd7a8810852b0698877180", "satisfiedRules": [], "answerKey": { "id": "request-granted-truth", "description": "The request was granted, so the duty never arises and nothing is due.", "exhaustive": true, "noFindingReason": "CONDITION_NOT_TRIGGERED", "expectedFindings": [], "prohibitedFindings": [ { "id": "grounds-demanded-for-a-grant", "valence": "FLAW", "description": "Demanding grounds for a refusal the entry does not state is the rule applied past its trigger.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The request for a second mooring was granted." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-conditional-duty-v2/blind-read-30770-gemini/answers.json", "item": "item-02", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "CONDITION_NOT_TRIGGERED" } ] } ], "comparisons": [ { "id": "refusal-trigger", "kind": "CONTRAST", "leftVariantId": "request-refused", "rightVariantId": "request-granted", "factor": "outcome of the mooring request", "leftLevel": "refused", "rightLevel": "granted", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-near-miss", "version": 1, "title": "A sample from open water without its depth, against a sample run from a tap", "hypothesis": "The engine reports a missing depth for a sample drawn from open water, and does not report one for a tap sample the same rule does not reach.", "evidenceClass": "REGRESSION", "capabilityTags": [ "scope-discrimination", "near-miss" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [ { "item": "p1 (tap-sample, tap-sample-truth)", "summary": "KEY_WEAK, VALIDITY: Rule A ends \"The duty reaches samples from open water only, and not samples run from a tap.\" The out-of-scope document says \"The sample was run from the tap in the pump house.\" The near miss is settled by matching the document against an exclusion the rule writes out word for word, so the case tests reading an exception clause, not the scope discrimination its hypothesis and its scope-discrimination tag claim.", "resolution": "ACCEPTED: The key is right, but the case does not test what it claims. The family is corrected under a new version whose rule states its scope by its trigger alone, over a source no careful reader could call open water. Follow-up #3076." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-near-miss.json", "covers": [ { "familyId": "regression-near-miss", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-near-miss.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": "tap-sample", "item": "tap-sample-truth", "rules": [ "A" ], "question": "VALIDITY", "severity": "KEY_WEAK", "argument": "Rule A ends \"The duty reaches samples from open water only, and not samples run from a tap.\" The out-of-scope document says \"The sample was run from the tap in the pump house.\" The near miss is settled by matching the document against an exclusion the rule writes out word for word, so the case tests reading an exception clause, not the scope discrimination its hypothesis and its scope-discrimination tag claim.", "resolution": { "status": "ACCEPTED", "reason": "The key is right, but the case does not test what it claims. The family is corrected under a new version whose rule states its scope by its trigger alone, over a source no careful reader could call open water.", "followUp": "#3076" } }, { "id": "p2", "variantId": "open-water-sample-without-depth", "item": "sample-depth-absent", "rules": [ "A" ], "question": "CORRECTNESS", "severity": "NOTE", "argument": "Is \"the open water of the north basin\" open water, or a basin a reader might take for a tank?", "resolution": { "status": "KEPT", "reason": "The document says open water in so many words, and rule A uses the same words. The flaw is a breach of rule A as written.", "followUp": null } }, { "id": "p3", "variantId": null, "item": "sample-source", "rules": [ "A" ], "question": "DIFFICULTY", "severity": "NOTE", "argument": "The flawed side's \"drawn from the open water\" repeats rule A's \"drawn from open water\", so the trigger can be found by phrase.", "resolution": { "status": "KEPT", "reason": "The suite is a change gate whose shapes and size are fixed to the pilot's measured cost (REGRESSION_CORPUS.md, Why the same shapes): a short case with one rule and one sentence is what each family is meant to be. Published as a known limit: passing it shows the engine has not regressed on the plain case, not that it resists a harder one.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "093b8e7c4ad697b18361c7126b1254d29148feaa047811246c02527c157762ed", "rules": [ { "label": "A", "name": "Depth of a sample drawn from open water", "source": "Entry Practice Schedule, part N, issue 1", "text": "In a record, every sampling entry that reports a sample drawn from open water must also give the depth at which it was drawn. The duty reaches samples from open water only, and not samples run from a tap. The duty rests on the author of the record.", "ruleKeySha256": "49c365fd01cfc065fb6716bdd33a26cf382c86ad7c19ea77fe2f8d043ca8aebf" } ], "variants": [ { "id": "open-water-sample-without-depth", "description": "A sample drawn from the open water of the north basin, with no depth.", "treatment": { "comparisonId": "sample-source", "factor": "where the sample was taken from", "level": "open water" }, "rules": [ "A" ], "document": "Sampling entry SA-062\nPurpose of sampling\nWater was sampled for the quarterly clarity check.\nWhere the sample came from\nThe sample was drawn from the open water of the north basin.\nDispatch of the sample\nThe sample was sent for analysis the same afternoon.\n", "renderedTextSha256": "4b6e1ba5981fa552334cbc9ade274b0261adc658b3317cb17099ed4f06f46865", "satisfiedRules": [], "answerKey": { "id": "open-water-sample-truth", "description": "A sample was drawn from open water and the depth the rule requires is absent.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "sample-depth-absent", "valence": "FLAW", "description": "The entry reports a sample drawn from open water and gives no depth.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The sample was drawn from the open water of the north basin." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "open-water-sample-praised-as-complete", "valence": "STRENGTH", "description": "Crediting the sample sentence for giving a depth when it gives none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The sample was drawn from the open water of the north basin." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-06", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The sample was drawn from the open water of the north basin.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-03", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The sample was drawn from the open water of the north basin.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "tap-sample", "description": "The same section, carrying a sample run from a tap.", "treatment": { "comparisonId": "sample-source", "factor": "where the sample was taken from", "level": "tap" }, "rules": [ "A" ], "document": "Sampling entry SA-062\nPurpose of sampling\nWater was sampled for the quarterly clarity check.\nWhere the sample came from\nThe sample was run from the tap in the pump house.\nDispatch of the sample\nThe sample was sent for analysis the same afternoon.\n", "renderedTextSha256": "5a9ffd8316a8ae34635812ca87e793c72047f6160c5d361616ac1d96ef31cdb7", "satisfiedRules": [], "answerKey": { "id": "tap-sample-truth", "description": "The same section carries a sample run from a tap, which the rule does not reach.", "exhaustive": true, "noFindingReason": "OUT_OF_SCOPE", "expectedFindings": [], "prohibitedFindings": [ { "id": "depth-demanded-for-a-tap-sample", "valence": "FLAW", "description": "Demanding a depth for a sample run from a tap is the rule stretched past its scope.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The sample was run from the tap in the pump house." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-15", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "OUT_OF_SCOPE" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-07", "agrees": false, "findings": [], "metRules": [], "noFindingReason": "CONDITION_NOT_TRIGGERED" } ] } ], "comparisons": [ { "id": "sample-source", "kind": "CONTRAST", "leftVariantId": "open-water-sample-without-depth", "rightVariantId": "tap-sample", "factor": "where the sample was taken from", "leftLevel": "open water", "rightLevel": "tap", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-near-miss", "version": 2, "title": "A sample from open water without its depth, against a sample drawn from a reference flask", "hypothesis": "The engine reports a missing depth for a sample drawn from open water, and does not report one for a sample drawn from a reference flask, which the rule's trigger does not reach although the rule never names it.", "evidenceClass": "REGRESSION", "capabilityTags": [ "scope-discrimination", "near-miss" ], "provenance": { "authoredBy": "Claude Project thread session for #3076", "authoredOn": "2026-10-05", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind. Version 2 corrects version 1 for #3076: version 1's rule wrote out the tap its clean side runs a sample from, so scope was matched against an exclusion clause rather than judged. Version 2's rule states its trigger alone, and the clean side's sample comes from a reference flask, which is plainly not open water." }, "authorModelFamily": "claude", "reviews": [ { "kind": "BLIND_READ", "reviewer": "Claude Project thread worker for #3076 (Claude Opus 5.5), blind read", "modelFamily": "claude", "reviewedOn": "2026-10-05", "seed": 30762, "packetSha256": "bccb4a7b52c9121b362ea4169066a621419f4f301ef81db400e8ab6beec68445", "scope": "FAMILY", "agreed": 1, "of": 2, "disagreements": [ { "item": "item-01", "summary": "The reader agreed nothing is due for the sample drawn from a reference flask, but gave the reason as a condition not triggered; the key declares the sample out of the rule's scope.", "resolution": "Genuinely ambiguous, as on version 1's tap sample: the packet names the four reasons without defining them, and a rule whose reach is its trigger reads both as a scope limit and as an unmet condition. Both readers, Claude and Gemini, read it as the condition; so did a Claude reader of a draft whose rule stated its reach in a sentence of its own. The wording cannot settle it; the packet's definitions can. Kept as declared; #3083 defines the reasons." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-near-miss-v2/blind-read-30762/answers.json", "covers": [ { "familyId": "regression-near-miss", "familyVersion": 2 } ] }, { "kind": "BLIND_READ", "reviewer": "gemini-2.5-pro (Vertex europe-west1), blind read", "modelFamily": "gemini", "reviewedOn": "2026-10-05", "seed": 30763, "packetSha256": "bccb4a7b52c9121b362ea4169066a621419f4f301ef81db400e8ab6beec68445", "scope": "FAMILY", "agreed": 1, "of": 2, "disagreements": [ { "item": "item-01", "summary": "The reader agreed nothing is due for the sample drawn from a reference flask, but gave the reason as a condition not triggered; the key declares the sample out of the rule's scope.", "resolution": "Genuinely ambiguous, as on version 1's tap sample: the packet names the four reasons without defining them, and a rule whose reach is its trigger reads both as a scope limit and as an unmet condition. Both readers, Claude and Gemini, read it as the condition; so did a Claude reader of a draft whose rule stated its reach in a sentence of its own. The wording cannot settle it; the packet's definitions can. Kept as declared; #3083 defines the reasons." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-near-miss-v2/blind-read-30763-gemini/answers.json", "covers": [ { "familyId": "regression-near-miss", "familyVersion": 2 } ] } ], "critiques": [], "crossFamilyReviewed": true, "documentTemplateSha256": "093b8e7c4ad697b18361c7126b1254d29148feaa047811246c02527c157762ed", "rules": [ { "label": "A", "name": "Depth of a sample drawn from open water", "source": "Entry Practice Schedule, part N, issue 1", "text": "In a record, every sampling entry that reports a sample drawn from open water must also give the depth at which it was drawn. The duty rests on the author of the record.", "ruleKeySha256": "49c365fd01cfc065fb6716bdd33a26cf382c86ad7c19ea77fe2f8d043ca8aebf" } ], "variants": [ { "id": "open-water-sample-without-depth", "description": "A sample drawn from the open water of the north basin, with no depth.", "treatment": { "comparisonId": "sample-source", "factor": "where the sample was taken from", "level": "open water" }, "rules": [ "A" ], "document": "Sampling entry SA-062\nPurpose of sampling\nWater was sampled for the quarterly clarity check.\nWhere the sample came from\nThe sample was drawn from the open water of the north basin.\nDispatch of the sample\nThe sample was sent for analysis the same afternoon.\n", "renderedTextSha256": "4b6e1ba5981fa552334cbc9ade274b0261adc658b3317cb17099ed4f06f46865", "satisfiedRules": [], "answerKey": { "id": "open-water-sample-truth", "description": "A sample was drawn from open water and the depth the rule requires is absent.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "sample-depth-absent", "valence": "FLAW", "description": "The entry reports a sample drawn from open water and gives no depth.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The sample was drawn from the open water of the north basin." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "open-water-sample-praised-as-complete", "valence": "STRENGTH", "description": "Crediting the sample sentence for giving a depth when it gives none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The sample was drawn from the open water of the north basin." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-near-miss-v2/blind-read-30762/answers.json", "item": "item-02", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The sample was drawn from the open water of the north basin.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-near-miss-v2/blind-read-30763-gemini/answers.json", "item": "item-02", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The sample was drawn from the open water of the north basin.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "tap-sample", "description": "The same section, carrying a sample drawn from a reference flask.", "treatment": { "comparisonId": "sample-source", "factor": "where the sample was taken from", "level": "reference flask" }, "rules": [ "A" ], "document": "Sampling entry SA-062\nPurpose of sampling\nWater was sampled for the quarterly clarity check.\nWhere the sample came from\nThe sample was drawn from the capped reference flask kept in the harbour office.\nDispatch of the sample\nThe sample was sent for analysis the same afternoon.\n", "renderedTextSha256": "1360098a3a0493d7a32aca4da29e03a54ab8c6d61945047c5e3046ecc1fae6b8", "satisfiedRules": [], "answerKey": { "id": "flask-sample-truth", "description": "The same section carries a sample drawn from a reference flask, which is not open water, so the rule does not reach it.", "exhaustive": true, "noFindingReason": "OUT_OF_SCOPE", "expectedFindings": [], "prohibitedFindings": [ { "id": "depth-demanded-for-a-flask-sample", "valence": "FLAW", "description": "Demanding a depth for a sample drawn from a reference flask is the rule stretched past its scope.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The sample was drawn from the capped reference flask kept in the harbour office." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-near-miss-v2/blind-read-30762/answers.json", "item": "item-01", "agrees": false, "findings": [], "metRules": [], "noFindingReason": "CONDITION_NOT_TRIGGERED" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-near-miss-v2/blind-read-30763-gemini/answers.json", "item": "item-01", "agrees": false, "findings": [], "metRules": [], "noFindingReason": "CONDITION_NOT_TRIGGERED" } ] } ], "comparisons": [ { "id": "sample-source", "kind": "CONTRAST", "leftVariantId": "open-water-sample-without-depth", "rightVariantId": "tap-sample", "factor": "where the sample was taken from", "leftLevel": "open water", "rightLevel": "reference flask", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-unsupported-claim", "version": 1, "title": "A relied-on figure with no named document, against a passage the copy omits", "hypothesis": "The engine reports a charge set from a figure that names no document when a supplied rule makes that unsupported, and reports nothing when the copy does not carry the passage that would decide it.", "evidenceClass": "REGRESSION", "capabilityTags": [ "substantiation", "insufficient-evidence" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [ { "item": "p1 (figure-without-document, charged-figure-without-document)", "summary": "KEY_WEAK, ANCHORING: Rule A's duty attaches to an entry that \"sets a charge from a figure\". That act is its own line, \"The levy was set from that tonnage.\", while the key anchors the omission on the line above, \"The gross tonnage of the ship was 1,840.\" An omission anchored on the setting of the charge is as correct as the key's, but the scorer accepts a wider quote only on the anchor's own line, so it is scored as a miss and an unexpected flaw.", "resolution": "ACCEPTED: A correct answer must not be scored as two errors. The family is corrected under a new version in which the figure and the charge set from it are one sentence. Follow-up #3075." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-unsupported-claim.json", "covers": [ { "familyId": "regression-unsupported-claim", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-unsupported-claim.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": "figure-without-document", "item": "charged-figure-without-document", "rules": [ "A" ], "question": "ANCHORING", "severity": "KEY_WEAK", "argument": "Rule A's duty attaches to an entry that \"sets a charge from a figure\". That act is its own line, \"The levy was set from that tonnage.\", while the key anchors the omission on the line above, \"The gross tonnage of the ship was 1,840.\" An omission anchored on the setting of the charge is as correct as the key's, but the scorer accepts a wider quote only on the anchor's own line, so it is scored as a miss and an unexpected flaw.", "resolution": { "status": "ACCEPTED", "reason": "A correct answer must not be scored as two errors. The family is corrected under a new version in which the figure and the charge set from it are one sentence.", "followUp": "#3075" } }, { "id": "p2", "variantId": "basis-withheld", "item": "withheld-basis-truth", "rules": [ "A" ], "question": "AMBIGUITY", "severity": "NOTE", "argument": "The withheld variant still says \"The harbour levy is recorded as payable in full.\" A careful reader could hold that the entry sets a charge and, as supplied, names no document for anything, so a flaw is due whatever the missing lines say.", "resolution": { "status": "KEPT", "reason": "Rule A is triggered by a charge set from a figure. With the basis withheld, neither whether a figure was used nor whether its document was named can be seen, and the marker says the copy, not the entry, lacks the lines. Insufficient evidence is the strict reading.", "followUp": null } }, { "id": "p3", "variantId": "basis-withheld", "item": "withheld-basis-truth", "rules": [], "question": "DIFFICULTY", "severity": "NOTE", "argument": "The bracketed line announces the gap in plain words, so the case tests obeying a marked gap rather than noticing that the decisive passage is missing.", "resolution": { "status": "KEPT", "reason": "A copy with withheld lines marks them, as redacted records do; an unmarked gap is a different capability and a different family.", "followUp": null } }, { "id": "p4", "variantId": null, "item": null, "rules": [ "A" ], "question": "VALIDITY", "severity": "NOTE", "argument": "The family carries the outcome unsupported-factual-claim, but structurally it is a claim-anchored omission under a rule that says what support is required, the same shape as the cure and refusal families. It does not test spotting an unsupported claim no rule describes.", "resolution": { "status": "KEPT", "reason": "The engine audits a document against supplied rules, so a claim is unsupported in the sense a supplied rule defines. The family's rule is what makes the figure unsupported; that is what the outcome names.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "1e2f522c3a8a20b3c8c8b90f93181b5e09a2fc9c59e8813d37cb50f566d3dd3f", "rules": [ { "label": "A", "name": "The document a charged figure was read from", "source": "Entry Practice Schedule, part P, issue 1", "text": "In a record, every levy entry that sets a charge from a figure must name the document the figure was read from. A figure whose document is not named gives the charge no support. The duty rests on the author of the record.", "ruleKeySha256": "fa3928d90b453dd2d705189e5d07ab64c11337803e039b0f60e0ab3268b5a7d6" } ], "variants": [ { "id": "figure-without-document", "description": "The levy is set from a tonnage the entry does not source.", "treatment": { "comparisonId": "charge-basis-availability", "factor": "the passage the rule would be applied to", "level": "present and unsourced" }, "rules": [ "A" ], "document": "Levy entry LV-440\nShip calling\nThe cargo ship Marta Holm called at the south quay.\nBasis of the charge\nThe gross tonnage of the ship was 1,840.\nThe levy was set from that tonnage.\nLevy result\nThe harbour levy is recorded as payable in full.\n", "renderedTextSha256": "98180581ad9bfe09a7cc9923acff073b7a099337f002b09c98ceb3b4b76ea369", "satisfiedRules": [], "answerKey": { "id": "unsourced-figure-truth", "description": "The charge is set from a stated figure and the entry names no document for it.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "charged-figure-without-document", "valence": "FLAW", "description": "The entry sets the levy from a figure and names no document it was read from.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The gross tonnage of the ship was 1,840." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "unsourced-figure-praised", "valence": "STRENGTH", "description": "Crediting the figure for naming its document when it names none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The gross tonnage of the ship was 1,840." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-02", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The gross tonnage of the ship was 1,840.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-17", "agrees": false, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The levy was set from that tonnage.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "basis-withheld", "description": "The same entry, whose copy does not carry that section.", "treatment": { "comparisonId": "charge-basis-availability", "factor": "the passage the rule would be applied to", "level": "not in the copy" }, "rules": [ "A" ], "document": "Levy entry LV-440\nShip calling\nThe cargo ship Marta Holm called at the south quay.\nBasis of the charge\n[The lines of this section are not in the copy supplied.]\nLevy result\nThe harbour levy is recorded as payable in full.\n", "renderedTextSha256": "ecbc0f6164232e4c9288ab383293ee2a7d46bf613acaa8d4febac0cb897b9455", "satisfiedRules": [], "answerKey": { "id": "withheld-basis-truth", "description": "The section the rule would be applied to is not in the copy, so the document cannot decide it.", "exhaustive": true, "noFindingReason": "INSUFFICIENT_EVIDENCE", "expectedFindings": [], "prohibitedFindings": [ { "id": "unsourced-figure-asserted-from-absent-text", "valence": "FLAW", "description": "Reporting an unsupported figure from a passage the copy does not carry is a finding without a document.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "[The lines of this section are not in the copy supplied.]" }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-08", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "INSUFFICIENT_EVIDENCE" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-08", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "INSUFFICIENT_EVIDENCE" } ] } ], "comparisons": [ { "id": "charge-basis-availability", "kind": "CONTRAST", "leftVariantId": "figure-without-document", "rightVariantId": "basis-withheld", "factor": "the passage the rule would be applied to", "leftLevel": "present and unsourced", "rightLevel": "not in the copy", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-unsupported-claim", "version": 2, "title": "A relied-on figure with no named document, against a passage the copy omits", "hypothesis": "The engine reports a charge set from a figure that names no document when a supplied rule makes that unsupported, and reports nothing when the copy does not carry the passage that would decide it.", "evidenceClass": "REGRESSION", "capabilityTags": [ "substantiation", "insufficient-evidence" ], "provenance": { "authoredBy": "Claude Project thread session for #3075", "authoredOn": "2026-10-05", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind. Version 2 corrects version 1 for #3075: an adversarial critique showed that an omission anchored on the sentence setting the levy, as correct as the key's, was scored as a miss. Version 2 states the figure and the charge set from it as one sentence." }, "authorModelFamily": "claude", "reviews": [ { "kind": "BLIND_READ", "reviewer": "Claude Project thread subagent for #3075 (Claude Opus 5.5), blind read", "modelFamily": "claude", "reviewedOn": "2026-10-05", "seed": 3075, "packetSha256": "64c33b3fa5357afc2585490495543ab872b55963d34cf4876add92f51c3d02a2", "scope": "FAMILY", "agreed": 2, "of": 2, "disagreements": [], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unsupported-claim-v2/blind-read-3075/answers.json", "covers": [ { "familyId": "regression-unsupported-claim", "familyVersion": 2 } ] }, { "kind": "BLIND_READ", "reviewer": "gemini-2.5-pro (Vertex europe-west1), blind read", "modelFamily": "gemini", "reviewedOn": "2026-10-05", "seed": 3075, "packetSha256": "64c33b3fa5357afc2585490495543ab872b55963d34cf4876add92f51c3d02a2", "scope": "FAMILY", "agreed": 2, "of": 2, "disagreements": [], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unsupported-claim-v2/blind-read-3075-gemini/answers.json", "covers": [ { "familyId": "regression-unsupported-claim", "familyVersion": 2 } ] } ], "critiques": [], "crossFamilyReviewed": true, "documentTemplateSha256": "1e2f522c3a8a20b3c8c8b90f93181b5e09a2fc9c59e8813d37cb50f566d3dd3f", "rules": [ { "label": "A", "name": "The document a charged figure was read from", "source": "Entry Practice Schedule, part P, issue 1", "text": "In a record, every levy entry that sets a charge from a figure must name the document the figure was read from. A figure whose document is not named gives the charge no support. The duty rests on the author of the record.", "ruleKeySha256": "fa3928d90b453dd2d705189e5d07ab64c11337803e039b0f60e0ab3268b5a7d6" } ], "variants": [ { "id": "figure-without-document", "description": "The levy is set from a tonnage the entry does not source.", "treatment": { "comparisonId": "charge-basis-availability", "factor": "the passage the rule would be applied to", "level": "present and unsourced" }, "rules": [ "A" ], "document": "Levy entry LV-440\nShip calling\nThe cargo ship Marta Holm called at the south quay.\nBasis of the charge\nThe levy was set from the ship's gross tonnage of 1,840.\nLevy result\nThe harbour levy is recorded as payable in full.\n", "renderedTextSha256": "f899d21aad385570091cdd0c546c16e4900b804ab86a7b49f106ea35b7a7b92c", "satisfiedRules": [], "answerKey": { "id": "unsourced-figure-truth", "description": "The charge is set from a stated figure and the entry names no document for it.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "charged-figure-without-document", "valence": "FLAW", "description": "The entry sets the levy from a figure and names no document it was read from.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The levy was set from the ship's gross tonnage of 1,840." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "unsourced-figure-praised", "valence": "STRENGTH", "description": "Crediting the figure for naming its document when it names none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The levy was set from the ship's gross tonnage of 1,840." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unsupported-claim-v2/blind-read-3075/answers.json", "item": "item-02", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The levy was set from the ship's gross tonnage of 1,840.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unsupported-claim-v2/blind-read-3075-gemini/answers.json", "item": "item-02", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The levy was set from the ship's gross tonnage of 1,840.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "basis-withheld", "description": "The same entry, whose copy does not carry that section.", "treatment": { "comparisonId": "charge-basis-availability", "factor": "the passage the rule would be applied to", "level": "not in the copy" }, "rules": [ "A" ], "document": "Levy entry LV-440\nShip calling\nThe cargo ship Marta Holm called at the south quay.\nBasis of the charge\n[The lines of this section are not in the copy supplied.]\nLevy result\nThe harbour levy is recorded as payable in full.\n", "renderedTextSha256": "ecbc0f6164232e4c9288ab383293ee2a7d46bf613acaa8d4febac0cb897b9455", "satisfiedRules": [], "answerKey": { "id": "withheld-basis-truth", "description": "The section the rule would be applied to is not in the copy, so the document cannot decide it.", "exhaustive": true, "noFindingReason": "INSUFFICIENT_EVIDENCE", "expectedFindings": [], "prohibitedFindings": [ { "id": "unsourced-figure-asserted-from-absent-text", "valence": "FLAW", "description": "Reporting an unsupported figure from a passage the copy does not carry is a finding without a document.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "[The lines of this section are not in the copy supplied.]" }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unsupported-claim-v2/blind-read-3075/answers.json", "item": "item-01", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "INSUFFICIENT_EVIDENCE" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unsupported-claim-v2/blind-read-3075-gemini/answers.json", "item": "item-01", "agrees": true, "findings": [], "metRules": [], "noFindingReason": "INSUFFICIENT_EVIDENCE" } ] } ], "comparisons": [ { "id": "charge-basis-availability", "kind": "CONTRAST", "leftVariantId": "figure-without-document", "rightVariantId": "basis-withheld", "factor": "the passage the rule would be applied to", "leftLevel": "present and unsourced", "rightLevel": "not in the copy", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-irrelevant-rule", "version": 1, "title": "The same entry read against one rule, and against that rule plus one it cannot offend", "hypothesis": "Adding a rule the document cannot offend leaves the engine's structured result unchanged.", "evidenceClass": "REGRESSION", "capabilityTags": [ "distractor-resistance", "rule-selection" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-irrelevant-rule.json", "covers": [ { "familyId": "regression-irrelevant-rule", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-irrelevant-rule.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": null, "item": "tow-line-rule-added", "rules": [ "A", "B" ], "question": "DIFFICULTY", "severity": "NOTE", "argument": "Rule B is triggered by a mention of a tow line, and the entry contains none, so the distractor can be dismissed by searching for the words. A distractor whose trigger is nearly met would be harder.", "resolution": { "status": "KEPT", "reason": "Rule B is near in subject (a dredger moved between berths is commonly moved under tow) and shares rule A's source label, so neither its topic nor its citation marks it as foreign. The hypothesis claims only that a rule the document cannot offend changes nothing, and that is what the pair tests.", "followUp": null } }, { "id": "p2", "variantId": null, "item": "tow-line-rule-added", "rules": [ "A", "B" ], "question": "VALIDITY", "severity": "NOTE", "argument": "Does the invariant pair change only the rule selection?", "resolution": { "status": "KEPT", "reason": "Both variants render the same bytes (one rendered SHA-256) and share one answer key; only the supplied rules differ, with rule B under the same authority type and source as rule A.", "followUp": null } }, { "id": "p3", "variantId": "move-rule-with-tow-line-rule", "item": "move-order-unnamed-truth", "rules": [ "A", "B" ], "question": "COMPLETENESS", "severity": "NOTE", "argument": "\"The crew of the dredger stayed aboard overnight.\" and \"The inner berth is recorded as free.\" are further claims. Could either owe something under rule A or rule B?", "resolution": { "status": "KEPT", "reason": "Rule A is triggered by a vessel being moved and rule B by a tow line being mentioned. Neither sentence does either, so the move sentence is the only breach.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "37f8abf88ff53803c9d7418c3bb80fcb85cf87d9cfd52d77765a008c8ec72172", "rules": [ { "label": "A", "name": "Who ordered a vessel moved", "source": "Entry Practice Schedule, part R, issue 1", "text": "In a record, every berthing entry that says a vessel was moved must also name the person who ordered the move. The duty rests on the author of the record.", "ruleKeySha256": "4b5e9ea4cdcacf53a9ae72bae55be530610c5818ca8ef2dbbcfee44e1eaa5441" }, { "label": "B", "name": "The rated load of a tow line", "source": "Entry Practice Schedule, part R, issue 1", "text": "In a record, every berthing entry that mentions a tow line must give the rated load of that line. The duty rests on the author of the record.", "ruleKeySha256": "5447e2e1fbdbd6908017eb7b1f781b1c18fd5d38d125888e0c7676d78ead10f7" } ], "variants": [ { "id": "move-rule-only", "description": "The entry read against the move rule alone.", "treatment": { "comparisonId": "tow-line-rule-added", "factor": "a rule the document cannot offend", "level": "not supplied" }, "rules": [ "A" ], "document": "Berthing entry BE-019\nMovement between berths\nThe dredger Kestrel was moved from the inner berth to the outer berth.\nCrew arrangements\nThe crew of the dredger stayed aboard overnight.\nBerth status\nThe inner berth is recorded as free.\n", "renderedTextSha256": "37f8abf88ff53803c9d7418c3bb80fcb85cf87d9cfd52d77765a008c8ec72172", "satisfiedRules": [], "answerKey": { "id": "move-order-unnamed-truth", "description": "The entry names nobody who ordered the move, which offends the first rule once. Exhaustive truth, so any finding drawn from the added rule counts as unexpected.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "move-without-ordering-person", "valence": "FLAW", "description": "The entry says the dredger was moved and names nobody who ordered the move.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The dredger Kestrel was moved from the inner berth to the outer berth." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "move-praised-as-ordered", "valence": "STRENGTH", "description": "Crediting the move sentence for naming who ordered it when it names nobody.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The dredger Kestrel was moved from the inner berth to the outer berth." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-10", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The dredger Kestrel was moved from the inner berth to the outer berth.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-12", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The dredger Kestrel was moved from the inner berth to the outer berth.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "move-rule-with-tow-line-rule", "description": "The same entry, read against the move rule and one about tow lines it mentions none of.", "treatment": { "comparisonId": "tow-line-rule-added", "factor": "a rule the document cannot offend", "level": "supplied" }, "rules": [ "A", "B" ], "document": "Berthing entry BE-019\nMovement between berths\nThe dredger Kestrel was moved from the inner berth to the outer berth.\nCrew arrangements\nThe crew of the dredger stayed aboard overnight.\nBerth status\nThe inner berth is recorded as free.\n", "renderedTextSha256": "37f8abf88ff53803c9d7418c3bb80fcb85cf87d9cfd52d77765a008c8ec72172", "satisfiedRules": [], "answerKey": { "id": "move-order-unnamed-truth", "description": "The entry names nobody who ordered the move, which offends the first rule once. Exhaustive truth, so any finding drawn from the added rule counts as unexpected.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "move-without-ordering-person", "valence": "FLAW", "description": "The entry says the dredger was moved and names nobody who ordered the move.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The dredger Kestrel was moved from the inner berth to the outer berth." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "move-praised-as-ordered", "valence": "STRENGTH", "description": "Crediting the move sentence for naming who ordered it when it names nobody.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The dredger Kestrel was moved from the inner berth to the outer berth." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-18", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The dredger Kestrel was moved from the inner berth to the outer berth.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-14", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The dredger Kestrel was moved from the inner berth to the outer berth.", "locator": null } ], "metRules": [], "noFindingReason": null } ] } ], "comparisons": [ { "id": "tow-line-rule-added", "kind": "INVARIANT", "leftVariantId": "move-rule-only", "rightVariantId": "move-rule-with-tow-line-rule", "factor": "a rule the document cannot offend", "leftLevel": "not supplied", "rightLevel": "supplied", "changed": "RULE_SELECTION" } ] }, { "id": "regression-attribute-invariance", "version": 1, "title": "One attribute of the licence holder moved, over a flawed and a clean entry", "hypothesis": "Moving only the courtesy title, or only the stated citizenship, leaves the engine's structured result unchanged, over an entry that offends the supplied rule and over one that meets it.", "evidenceClass": "REGRESSION", "capabilityTags": [ "attribute-invariance", "paired-execution" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [ { "item": "p1", "summary": "KEY_WEAK, VALIDITY: The citizenships are invented and the gender treatment moves only \"Ms\" to \"Mr\". An engine biased by real nationalities, or by names, would pass all four pairs. The pairs show that a pair executes and scores as a pair, not how the engine treats any group.", "resolution": "KEPT: REGRESSION_CORPUS.md says the pairs support no claim about demographic bias, and the catalog publishes that limit beside them. Real attributes need attested pairs and their own decision; they are not a correction of this key." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-attribute-invariance.json", "covers": [ { "familyId": "regression-attribute-invariance", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-attribute-invariance.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": null, "item": null, "rules": [], "question": "VALIDITY", "severity": "KEY_WEAK", "argument": "The citizenships are invented and the gender treatment moves only \"Ms\" to \"Mr\". An engine biased by real nationalities, or by names, would pass all four pairs. The pairs show that a pair executes and scores as a pair, not how the engine treats any group.", "resolution": { "status": "KEPT", "reason": "REGRESSION_CORPUS.md says the pairs support no claim about demographic bias, and the catalog publishes that limit beside them. Real attributes need attested pairs and their own decision; they are not a correction of this key.", "followUp": null } }, { "id": "p2", "variantId": null, "item": "licence-citizenship-pair-flawed-base", "rules": [ "A" ], "question": "VALIDITY", "severity": "NOTE", "argument": "The citizenship pairs hold the holder at \"Ms\" and the gender pairs hold the citizenship at Delvoria, so the two attributes are never crossed and an interaction between them would not show.", "resolution": { "status": "KEPT", "reason": "The suite is a change gate whose shapes and size are fixed to the pilot's measured cost (REGRESSION_CORPUS.md, Why the same shapes): a short case with one rule and one sentence is what each family is meant to be. Published as a known limit: passing it shows the engine has not regressed on the plain case, not that it resists a harder one. A crossed design needs more variants.", "followUp": null } }, { "id": "p3", "variantId": "licence-gender-flawed-feminine", "item": "renewal-without-deciding-officer", "rules": [ "A" ], "question": "COMPLETENESS", "severity": "NOTE", "argument": "\"The applicant asked to renew a permit to fish from the breakwater.\" Is the request a second decision that also owes an officer?", "resolution": { "status": "KEPT", "reason": "Rule A reaches a stated decision. An application is not a decision; the renewal is the only one, and it is offended once.", "followUp": null } }, { "id": "p4", "variantId": "licence-gender-clean-feminine", "item": "renewal-decider-named-truth", "rules": [ "A" ], "question": "DIFFICULTY", "severity": "NOTE", "argument": "The clean base says \"licensing officer D. Aske\", repeating rule A's word \"officer\".", "resolution": { "status": "KEPT", "reason": "The suite is a change gate whose shapes and size are fixed to the pilot's measured cost (REGRESSION_CORPUS.md, Why the same shapes): a short case with one rule and one sentence is what each family is meant to be. Published as a known limit: passing it shows the engine has not regressed on the plain case, not that it resists a harder one.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "2f20ee608dac3c2288b83757821258177ba22a18dc93b815e6c4cb58d42aa9a0", "rules": [ { "label": "A", "name": "Naming the officer behind a decision", "source": "Entry Practice Schedule, part S, issue 1", "text": "In a record, every licence entry that states a decision must also name the officer who made that decision. The duty rests on the author of the record.", "ruleKeySha256": "60b3f42fd2b335e9e6b87ec594e182bf83aab3cd8301b74b18b1fd319cb50c24" } ], "variants": [ { "id": "licence-gender-flawed-feminine", "description": "courtesy title of the licence holder set to \"Ms\", over entry LI-71.", "treatment": { "comparisonId": "licence-gender-pair-flawed-base", "factor": "courtesy title of the licence holder", "level": "Ms" }, "rules": [ "A" ], "document": "Licence entry LI-71\nLicence holder\nApplicant: Ms T. Brennholt, a citizen of Delvoria.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\n", "renderedTextSha256": "ab1ae27520365399502256511fd864fdaa97b16753fa12b98493d34cc9926a0c", "satisfiedRules": [], "answerKey": { "id": "renewal-decider-unnamed-truth", "description": "The decision names no officer who made it, which offends the supplied rule exactly once.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "renewal-without-deciding-officer", "valence": "FLAW", "description": "The entry states the renewal decision and names no officer who made it.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "renewal-praised-as-attributed", "valence": "STRENGTH", "description": "Crediting the decision for naming its officer when it names none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-21", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-01", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "licence-gender-flawed-masculine", "description": "courtesy title of the licence holder set to \"Mr\", over entry LI-71.", "treatment": { "comparisonId": "licence-gender-pair-flawed-base", "factor": "courtesy title of the licence holder", "level": "Mr" }, "rules": [ "A" ], "document": "Licence entry LI-71\nLicence holder\nApplicant: Mr T. Brennholt, a citizen of Delvoria.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\n", "renderedTextSha256": "bafbd5ebc4fe53b989e98bcaf43169ebf67578069055974eae36c0a1d144d4bd", "satisfiedRules": [], "answerKey": { "id": "renewal-decider-unnamed-truth", "description": "The decision names no officer who made it, which offends the supplied rule exactly once.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "renewal-without-deciding-officer", "valence": "FLAW", "description": "The entry states the renewal decision and names no officer who made it.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "renewal-praised-as-attributed", "valence": "STRENGTH", "description": "Crediting the decision for naming its officer when it names none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-19", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-21", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "licence-gender-clean-feminine", "description": "courtesy title of the licence holder set to \"Ms\", over entry LI-72.", "treatment": { "comparisonId": "licence-gender-pair-clean-base", "factor": "courtesy title of the licence holder", "level": "Ms" }, "rules": [ "A" ], "document": "Licence entry LI-72\nLicence holder\nApplicant: Ms T. Brennholt, a citizen of Delvoria.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\nThe renewal was decided by licensing officer D. Aske.\n", "renderedTextSha256": "818c484f49e7bd56f65327d59cc899b85c974dca049215e0be31dcb53b8234e1", "satisfiedRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "answerKey": { "id": "renewal-decider-named-truth", "description": "The same decision, naming the officer the rule asks for. Nothing is due.", "exhaustive": true, "noFindingReason": "COMPLIANT", "expectedFindings": [], "prohibitedFindings": [ { "id": "false-unattributed-renewal", "valence": "FLAW", "description": "Reporting a missing officer when the entry names one is a false positive.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-01", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-02", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" } ] }, { "id": "licence-gender-clean-masculine", "description": "courtesy title of the licence holder set to \"Mr\", over entry LI-72.", "treatment": { "comparisonId": "licence-gender-pair-clean-base", "factor": "courtesy title of the licence holder", "level": "Mr" }, "rules": [ "A" ], "document": "Licence entry LI-72\nLicence holder\nApplicant: Mr T. Brennholt, a citizen of Delvoria.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\nThe renewal was decided by licensing officer D. Aske.\n", "renderedTextSha256": "02f4b84966f4debf6fa3c1e667ca5618ab6f6a4158d4c783b2cdee1edf3fe50f", "satisfiedRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "answerKey": { "id": "renewal-decider-named-truth", "description": "The same decision, naming the officer the rule asks for. Nothing is due.", "exhaustive": true, "noFindingReason": "COMPLIANT", "expectedFindings": [], "prohibitedFindings": [ { "id": "false-unattributed-renewal", "valence": "FLAW", "description": "Reporting a missing officer when the entry names one is a false positive.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-13", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-09", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" } ] }, { "id": "licence-citizenship-flawed-first", "description": "stated citizenship of the licence holder set to \"Delvoria\", over entry LI-73.", "treatment": { "comparisonId": "licence-citizenship-pair-flawed-base", "factor": "stated citizenship of the licence holder", "level": "Delvoria" }, "rules": [ "A" ], "document": "Licence entry LI-73\nLicence holder\nApplicant: Ms T. Brennholt, a citizen of Delvoria.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\n", "renderedTextSha256": "884e10bed2b48f3b95805f9cf3a56d228b9f76268c465fddbb7b2bd6448572a5", "satisfiedRules": [], "answerKey": { "id": "renewal-decider-unnamed-truth", "description": "The decision names no officer who made it, which offends the supplied rule exactly once.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "renewal-without-deciding-officer", "valence": "FLAW", "description": "The entry states the renewal decision and names no officer who made it.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "renewal-praised-as-attributed", "valence": "STRENGTH", "description": "Crediting the decision for naming its officer when it names none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-04", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-16", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "licence-citizenship-flawed-second", "description": "stated citizenship of the licence holder set to \"Kastmere\", over entry LI-73.", "treatment": { "comparisonId": "licence-citizenship-pair-flawed-base", "factor": "stated citizenship of the licence holder", "level": "Kastmere" }, "rules": [ "A" ], "document": "Licence entry LI-73\nLicence holder\nApplicant: Ms T. Brennholt, a citizen of Kastmere.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\n", "renderedTextSha256": "bd34e510d196246750117545b9fcb6b4e691cb8df8733b7c01e4994cba1d15c4", "satisfiedRules": [], "answerKey": { "id": "renewal-decider-unnamed-truth", "description": "The decision names no officer who made it, which offends the supplied rule exactly once.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "renewal-without-deciding-officer", "valence": "FLAW", "description": "The entry states the renewal decision and names no officer who made it.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "renewal-praised-as-attributed", "valence": "STRENGTH", "description": "Crediting the decision for naming its officer when it names none.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-20", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-05", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": "The fishing permit was renewed for one season.", "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "licence-citizenship-clean-first", "description": "stated citizenship of the licence holder set to \"Delvoria\", over entry LI-74.", "treatment": { "comparisonId": "licence-citizenship-pair-clean-base", "factor": "stated citizenship of the licence holder", "level": "Delvoria" }, "rules": [ "A" ], "document": "Licence entry LI-74\nLicence holder\nApplicant: Ms T. Brennholt, a citizen of Delvoria.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\nThe renewal was decided by licensing officer D. Aske.\n", "renderedTextSha256": "e053ebf3b9dabc1aacceb791736e3e987c58042d281356af79de0a3ca7bb3cb7", "satisfiedRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "answerKey": { "id": "renewal-decider-named-truth", "description": "The same decision, naming the officer the rule asks for. Nothing is due.", "exhaustive": true, "noFindingReason": "COMPLIANT", "expectedFindings": [], "prohibitedFindings": [ { "id": "false-unattributed-renewal", "valence": "FLAW", "description": "Reporting a missing officer when the entry names one is a false positive.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-14", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-18", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" } ] }, { "id": "licence-citizenship-clean-second", "description": "stated citizenship of the licence holder set to \"Kastmere\", over entry LI-74.", "treatment": { "comparisonId": "licence-citizenship-pair-clean-base", "factor": "stated citizenship of the licence holder", "level": "Kastmere" }, "rules": [ "A" ], "document": "Licence entry LI-74\nLicence holder\nApplicant: Ms T. Brennholt, a citizen of Kastmere.\nRenewal sought\nThe applicant asked to renew a permit to fish from the breakwater.\nRenewal outcome\nThe fishing permit was renewed for one season.\nThe renewal was decided by licensing officer D. Aske.\n", "renderedTextSha256": "5ae9c3823e551668f0efd5667c00445ab0c282b30b37159b9c1ff3cfa4fdd392", "satisfiedRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "answerKey": { "id": "renewal-decider-named-truth", "description": "The same decision, naming the officer the rule asks for. Nothing is due.", "exhaustive": true, "noFindingReason": "COMPLIANT", "expectedFindings": [], "prohibitedFindings": [ { "id": "false-unattributed-renewal", "valence": "FLAW", "description": "Reporting a missing officer when the entry names one is a false positive.", "isOmission": true, "evidence": { "kind": "QUOTE", "quote": "The fishing permit was renewed for one season." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-11", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-11", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The renewal was decided by licensing officer D. Aske." } ], "noFindingReason": "COMPLIANT" } ] } ], "comparisons": [ { "id": "licence-gender-pair-flawed-base", "kind": "INVARIANT", "leftVariantId": "licence-gender-flawed-feminine", "rightVariantId": "licence-gender-flawed-masculine", "factor": "courtesy title of the licence holder", "leftLevel": "Ms", "rightLevel": "Mr", "changed": "DOCUMENT_PASSAGE" }, { "id": "licence-gender-pair-clean-base", "kind": "INVARIANT", "leftVariantId": "licence-gender-clean-feminine", "rightVariantId": "licence-gender-clean-masculine", "factor": "courtesy title of the licence holder", "leftLevel": "Ms", "rightLevel": "Mr", "changed": "DOCUMENT_PASSAGE" }, { "id": "licence-citizenship-pair-flawed-base", "kind": "INVARIANT", "leftVariantId": "licence-citizenship-flawed-first", "rightVariantId": "licence-citizenship-flawed-second", "factor": "stated citizenship of the licence holder", "leftLevel": "Delvoria", "rightLevel": "Kastmere", "changed": "DOCUMENT_PASSAGE" }, { "id": "licence-citizenship-pair-clean-base", "kind": "INVARIANT", "leftVariantId": "licence-citizenship-clean-first", "rightVariantId": "licence-citizenship-clean-second", "factor": "stated citizenship of the licence holder", "leftLevel": "Delvoria", "rightLevel": "Kastmere", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-unconditional-duty", "version": 1, "title": "A duty owed by every entry of its kind, with and without the sentence that meets it", "hypothesis": "The engine reports a closure entry that says nothing about where its papers are stored as an omission with no passage to quote, and reports nothing when the same entry says where.", "evidenceClass": "REGRESSION", "capabilityTags": [ "omission-detection", "unconditional-duty" ], "provenance": { "authoredBy": "Verkstaden implementer (Claude Local), #2323", "authoredOn": "2026-09-23", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind." }, "authorModelFamily": "claude", "reviews": [ { "kind": "ADVERSARIAL_CRITIQUE", "reviewer": "Claude Project thread session for #3049, adversarial critique", "modelFamily": "claude", "reviewedOn": "2026-10-04", "seed": null, "packetSha256": null, "scope": "FAMILY", "agreed": null, "of": null, "disagreements": [], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-unconditional-duty.json", "covers": [ { "familyId": "regression-unconditional-duty", "familyVersion": 1 } ] } ], "critiques": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/critique/regression-unconditional-duty.json", "protocolVersion": 1, "points": [ { "id": "p1", "variantId": "papers-location-unstated", "item": "papers-location-absent", "rules": [ "A" ], "question": "ANCHORING", "severity": "NOTE", "argument": "The target has no quote. An engine that anchors the omission on the closing note, \"The matter needs no further work.\", where the statement would naturally sit, is scored as a miss and an unexpected flaw.", "resolution": { "status": "KEPT", "reason": "The production prompt tells the engine that a document lacking a required step altogether reports it as an omission with no quote. Anchoring it on an unrelated sentence departs from that contract, which the suite settled (#2081).", "followUp": null } }, { "id": "p2", "variantId": "papers-location-unstated", "item": "papers-location-unstated-truth", "rules": [ "A" ], "question": "CORRECTNESS", "severity": "NOTE", "argument": "A complaint closed by telephone may have no original papers at all, in which case there is nothing to say where they are.", "resolution": { "status": "KEPT", "reason": "Rule A is owed by every closure entry \"whatever its matter\", and the matter began as a complaint, which the entry records. If there were no papers, the entry would have to say so; it says nothing.", "followUp": null } }, { "id": "p3", "variantId": null, "item": "papers-location-presence", "rules": [ "A" ], "question": "DIFFICULTY", "severity": "NOTE", "argument": "Rule A's second sentence tells the reader the duty is owed \"without first mentioning papers at all\", so it states the very property the family tests.", "resolution": { "status": "KEPT", "reason": "Unlike an exemption naming a case, the sentence states the rule's scope as a real rule would and does not settle either variant: whether the entry says where the papers are still decides the contrast.", "followUp": null } } ] } ], "crossFamilyReviewed": true, "documentTemplateSha256": "8e8453d2c13349db47bc8a8625e3526784a24d3f3bd807ef19494c25263f6c55", "rules": [ { "label": "A", "name": "Where the original papers are stored", "source": "Entry Practice Schedule, part T, issue 1", "text": "In a record, every closure entry must say where the original papers behind it are stored. Each closure entry owes this whatever its matter, without first mentioning papers at all. The duty rests on the author of the record.", "ruleKeySha256": "9179015ff26a2a0761ed1900dcb9a8b4361adb49cc61403b42fc4ab93235e471" } ], "variants": [ { "id": "papers-location-unstated", "description": "The closure entry says nothing about where its papers are stored.", "treatment": { "comparisonId": "papers-location-presence", "factor": "stated place where the original papers are stored", "level": "absent" }, "rules": [ "A" ], "document": "Closure entry CL-507\nMatter closed\nThe complaint about night noise at the ferry ramp was closed.\nClosing steps\nThe complainant was told of the closure by telephone on 3 June 2032.\nClosing note\nThe matter needs no further work.\n", "renderedTextSha256": "692f4cad59486f1923b934ebe4f752574d84e41a3e016633146230e27f293c03", "satisfiedRules": [], "answerKey": { "id": "papers-location-unstated-truth", "description": "The entry names no place for any papers under any heading, so the duty is owed and unmet with no passage the omission can be anchored on.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "papers-location-absent", "valence": "FLAW", "description": "The closure entry does not say where its original papers are stored, which the rule requires of every closure entry.", "isOmission": true, "evidence": { "kind": "NO_QUOTE", "locator": null }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "closing-note-praised-as-locating-papers", "valence": "STRENGTH", "description": "Crediting the closing note for saying where the papers are stored when it says nothing of the kind.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The matter needs no further work." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-22", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": null, "locator": null } ], "metRules": [], "noFindingReason": null }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-06", "agrees": false, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": null, "locator": "Closing note" } ], "metRules": [], "noFindingReason": null } ] }, { "id": "papers-location-stated", "description": "The same entry, saying where the original papers are stored.", "treatment": { "comparisonId": "papers-location-presence", "factor": "stated place where the original papers are stored", "level": "present" }, "rules": [ "A" ], "document": "Closure entry CL-507\nMatter closed\nThe complaint about night noise at the ferry ramp was closed.\nClosing steps\nThe complainant was told of the closure by telephone on 3 June 2032.\nClosing note\nThe matter needs no further work.\nThe original papers are stored in cabinet 12 of the east office.\n", "renderedTextSha256": "2d2a5a773b541622b19e4da769ae33e2d6b79e5b2e3c75623bfa2c26768b3f2c", "satisfiedRules": [ { "rule": "A", "passage": "The original papers are stored in cabinet 12 of the east office." } ], "answerKey": { "id": "papers-location-stated-truth", "description": "The same entry, saying where the papers are stored. The rule carries no crediting clause, so nothing is due either way.", "exhaustive": true, "noFindingReason": "COMPLIANT", "expectedFindings": [], "prohibitedFindings": [ { "id": "false-unstated-papers-location", "valence": "FLAW", "description": "Reporting the papers' location as missing when the entry states it is a false positive, and an unlocated one is how this family's true target looks.", "isOmission": true, "evidence": { "kind": "NO_QUOTE", "locator": null }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-90231/answers.json", "item": "item-17", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The original papers are stored in cabinet 12 of the east office." } ], "noFindingReason": "COMPLIANT" }, { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/blind-read-30500-gemini/answers.json", "item": "item-04", "agrees": true, "findings": [], "metRules": [ { "rule": "A", "passage": "The original papers are stored in cabinet 12 of the east office." } ], "noFindingReason": "COMPLIANT" } ] } ], "comparisons": [ { "id": "papers-location-presence", "kind": "CONTRAST", "leftVariantId": "papers-location-unstated", "rightVariantId": "papers-location-stated", "factor": "stated place where the original papers are stored", "leftLevel": "absent", "rightLevel": "present", "changed": "DOCUMENT_PASSAGE" } ] }, { "id": "regression-unconditional-duty", "version": 2, "title": "A duty owed by every entry of its kind, with and without the sentence that meets it", "hypothesis": "The engine reports a closure entry that says nothing about where its papers are stored as an omission with no passage to quote, and reports nothing when the same entry says where.", "evidenceClass": "REGRESSION", "capabilityTags": [ "omission-detection", "unconditional-duty" ], "provenance": { "authoredBy": "Claude Project thread session for #3082", "authoredOn": "2026-10-05", "note": "Authored for #2323 as new REGRESSION material in the outcome shapes of the controlled-pilot suite, sharing none of its text. The material is fictitious so no rule can be answered from memory of a legal order, so the suite depends on no production record, and so one variable can be moved without a real document's incidental correlations moving with it. It is REGRESSION evidence whose declared truth a reader who did not author it has checked blind. Version 2 corrects version 1 for #3082: a blind reader placed the missing statement of where the papers are stored under the entry's closing note, a defensible home the key did not allow. Version 2 drops that section and gives the stated side's sentence a heading of its own." }, "authorModelFamily": "claude", "reviews": [ { "kind": "BLIND_READ", "reviewer": "gemini-2.5-pro (Vertex europe-west1), blind read", "modelFamily": "gemini", "reviewedOn": "2026-10-05", "seed": 220898, "packetSha256": "c83e9b1f8e8dcca275990498aedd610b8a4943b573aa885090431008ea491c14", "scope": "FAMILY", "agreed": 1, "of": 2, "disagreements": [ { "item": "item-02", "summary": "The reader found the duty met by the sentence that says where the papers are stored, as declared, but credited that sentence as a strength; the key declares the entry compliant and credits nothing.", "resolution": "Reader over-credits: doing what the rule requires, and no more, is compliance, not a strength. Kept as declared." } ], "evidence": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unconditional-duty-v2/blind-read-220898-gemini/answers.json", "covers": [ { "familyId": "regression-unconditional-duty", "familyVersion": 2 } ] } ], "critiques": [], "crossFamilyReviewed": true, "documentTemplateSha256": "aa450261fa4e8225c1db9e713ace8affc46b4c8bf412c4abd533b31e8276db0e", "rules": [ { "label": "A", "name": "Where the original papers are stored", "source": "Entry Practice Schedule, part T, issue 1", "text": "In a record, every closure entry must say where the original papers behind it are stored. Each closure entry owes this whatever its matter, without first mentioning papers at all. The duty rests on the author of the record.", "ruleKeySha256": "9179015ff26a2a0761ed1900dcb9a8b4361adb49cc61403b42fc4ab93235e471" } ], "variants": [ { "id": "papers-location-unstated", "description": "The closure entry says nothing about where its papers are stored.", "treatment": { "comparisonId": "papers-location-presence", "factor": "stated place where the original papers are stored", "level": "absent" }, "rules": [ "A" ], "document": "Closure entry CL-507\nMatter closed\nThe complaint about night noise at the ferry ramp was closed.\nComplainant informed\nThe complainant was told of the closure by telephone on 3 June 2032.\n", "renderedTextSha256": "1509e407bc17990d3ef50ffc20ccf600dc31a31c66aa255af6eae9b453b19620", "satisfiedRules": [], "answerKey": { "id": "papers-location-unstated-truth", "description": "The entry names no place for any papers under any heading, so the duty is owed and unmet with no passage the omission can be anchored on.", "exhaustive": true, "noFindingReason": null, "expectedFindings": [ { "id": "papers-location-absent", "valence": "FLAW", "description": "The closure entry does not say where its original papers are stored, which the rule requires of every closure entry.", "isOmission": true, "evidence": { "kind": "NO_QUOTE", "locator": null }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ], "prohibitedFindings": [ { "id": "notice-praised-as-locating-papers", "valence": "STRENGTH", "description": "Crediting the line on how the complainant was told for saying where the papers are stored when it says nothing of the kind.", "isOmission": null, "evidence": { "kind": "QUOTE", "quote": "The complainant was told of the closure by telephone on 3 June 2032." }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unconditional-duty-v2/blind-read-220898-gemini/answers.json", "item": "item-01", "agrees": true, "findings": [ { "valence": "FLAW", "rule": "A", "isOmission": true, "quote": null, "locator": null } ], "metRules": [], "noFindingReason": null } ] }, { "id": "papers-location-stated", "description": "The same entry, saying where the original papers are stored.", "treatment": { "comparisonId": "papers-location-presence", "factor": "stated place where the original papers are stored", "level": "present" }, "rules": [ "A" ], "document": "Closure entry CL-507\nMatter closed\nThe complaint about night noise at the ferry ramp was closed.\nComplainant informed\nThe complainant was told of the closure by telephone on 3 June 2032.\nOriginal papers\nThe original papers are stored in cabinet 12 of the east office.\n", "renderedTextSha256": "10959a33b3c8f0e43a1467090d89842940fd1cef080cae42008a8ce0c72d2aa0", "satisfiedRules": [ { "rule": "A", "passage": "The original papers are stored in cabinet 12 of the east office." } ], "answerKey": { "id": "papers-location-stated-truth", "description": "The same entry, saying where the papers are stored. The rule carries no crediting clause, so nothing is due either way.", "exhaustive": true, "noFindingReason": "COMPLIANT", "expectedFindings": [], "prohibitedFindings": [ { "id": "false-unstated-papers-location", "valence": "FLAW", "description": "Reporting the papers' location as missing when the entry states it is a false positive, and an unlocated one is how this family's true target looks.", "isOmission": true, "evidence": { "kind": "NO_QUOTE", "locator": null }, "primaryRules": [ "A" ], "requiredSupportingRules": [], "allowedSupportingRules": [], "stages": [ "WORKER", "DELIVERED" ] } ] }, "blindReads": [ { "review": "apps/backend/src/eval/reviews/controlled-regression-v1/regression-unconditional-duty-v2/blind-read-220898-gemini/answers.json", "item": "item-02", "agrees": false, "findings": [ { "valence": "STRENGTH", "rule": "A", "isOmission": null, "quote": "The original papers are stored in cabinet 12 of the east office.", "locator": null } ], "metRules": [], "noFindingReason": null } ] } ], "comparisons": [ { "id": "papers-location-presence", "kind": "CONTRAST", "leftVariantId": "papers-location-unstated", "rightVariantId": "papers-location-stated", "factor": "stated place where the original papers are stored", "leftLevel": "absent", "rightLevel": "present", "changed": "DOCUMENT_PASSAGE" } ] } ] } ] }