{
  "status": "Hand-authored teaching cases; no annotator study or model run",
  "records": [
    {
      "id": "M1",
      "text": "Tap dripping continuously",
      "constructedLabelA": "plumbing",
      "constructedLabelB": "plumbing",
      "target": "plumbing",
      "reason": "explicit tap fault",
      "group": "g1",
      "split": "development"
    },
    {
      "id": "M2",
      "text": "Bedroom socket is not working",
      "constructedLabelA": "electrical",
      "constructedLabelB": "electrical",
      "target": "electrical",
      "reason": "explicit socket fault",
      "group": "g2",
      "split": "development"
    },
    {
      "id": "M3",
      "text": "Correction to M2: cupboard catch is broken, not the socket",
      "constructedLabelA": "joinery",
      "constructedLabelB": "joinery",
      "target": "joinery",
      "reason": "explicit correction in supplied text",
      "group": "g2",
      "split": "development"
    },
    {
      "id": "M4",
      "text": "Internal wooden door will not close",
      "constructedLabelA": "joinery",
      "constructedLabelB": "joinery",
      "target": "joinery",
      "reason": "explicit door fault",
      "group": "g4",
      "split": "holdout"
    },
    {
      "id": "M5",
      "text": "Tap leaks and a cupboard hinge has broken",
      "constructedLabelA": "plumbing",
      "constructedLabelB": "joinery",
      "target": "review",
      "reason": "two in-scope trades; no priority rule",
      "group": "g5",
      "split": "development"
    },
    {
      "id": "M6",
      "text": "Please pay the outstanding invoice",
      "constructedLabelA": "out_of_scope",
      "constructedLabelB": "review",
      "target": "out_of_scope",
      "reason": "payment request, no repair task",
      "group": "g6",
      "split": "development"
    },
    {
      "id": "M7",
      "text": "There is a problem in the kitchen",
      "constructedLabelA": "plumbing",
      "constructedLabelB": "review",
      "target": "review",
      "reason": "room alone does not identify trade",
      "group": "g7",
      "split": "holdout"
    },
    {
      "id": "M8",
      "text": "The hallway light flickers",
      "constructedLabelA": "electrical",
      "constructedLabelB": "electrical",
      "target": "electrical",
      "reason": "explicit lighting fault",
      "group": "g8",
      "split": "holdout"
    },
    {
      "id": "M9",
      "text": "Cupboard hinge has detached",
      "constructedLabelA": "joinery",
      "constructedLabelB": "joinery",
      "target": "joinery",
      "reason": "explicit hinge fault",
      "group": "g9",
      "split": "holdout"
    },
    {
      "id": "M10",
      "text": "The bathroom tap drips",
      "constructedLabelA": "plumbing",
      "constructedLabelB": "plumbing",
      "target": "plumbing",
      "reason": "explicit tap fault",
      "group": "g10",
      "split": "development"
    },
    {
      "id": "M11",
      "text": "A drip keeps coming from the bathroom tap",
      "constructedLabelA": "plumbing",
      "constructedLabelB": "plumbing",
      "target": "plumbing",
      "reason": "paraphrase of M10 kept in same group",
      "group": "g10",
      "split": "development"
    },
    {
      "id": "M12",
      "text": "Can you confirm the appointment time?",
      "constructedLabelA": "out_of_scope",
      "constructedLabelB": "out_of_scope",
      "target": "out_of_scope",
      "reason": "appointment enquiry, no new repair task",
      "group": "g12",
      "split": "holdout"
    }
  ],
  "development": 7,
  "holdout": 5,
  "constructedDisagreements": [
    "M5",
    "M6",
    "M7"
  ]
}