{
  "name": "CCC Garment AI Benchmark",
  "version": "2.3",
  "description": "Evaluation tests to check if AI-generated CCC garments match verified reference. Transforms dataset from training data toward AI benchmark product.",
  "philosophy": "Instead of claiming 'None of the AI models have the correct garment', we demonstrate recurring errors with a reproducible benchmark.",
  "test_categories": [
    {
      "id": "pair_disambiguation",
      "name": "Paired-robe disambiguation (Onibode / Akawe)",
      "why": "Two ranks wear all but the same robe and differ only by the loin. A model that has learned 'anointed brother = one garment' will collapse them. This test asks for the right one and checks the loin.",
      "prompt": "Render the Onibode garment — anointed brother of the Celestial Church of Christ.",
      "pass_criteria": [
        "Robe matches the anointed-brother cut (same garment as the Akawe).",
        "The LOIN is present, and its colour/design matches the Onibode record — not the Akawe's.",
        "The model does not substitute an ordinary brother or elder-brother garment.",
        "If the loin detail is not yet captured, the model says so instead of inventing a loin."
      ],
      "fail_signals": [
        "The Akawe loin is rendered for the Onibode (or vice versa).",
        "The robe is rendered as the elder-brother garment.",
        "A loin is invented when the record says it is not documented yet."
      ],
      "dataset_hook": "knowledge-graph.json → robe_pairs[onibode-akawe] and each rank's describe_hint"
    },
    {
      "id": "sash_colour",
      "name": "Sash Colour Accuracy",
      "description": "Does AI generate correct sash colour for rank?",
      "critical": true,
      "example": {
        "rank": "Cape Prophetess",
        "prompt": "Generate a Cape Prophetess garment",
        "expected": "Royal Blue (#4169E1) satin",
        "generic_ai_output": "Light blue or white sash (incorrect)",
        "verified_reference": "Royal Blue - documented in metadata.sash_colour",
        "evaluation": "Color picker check ±15 hue tolerance"
      }
    },
    {
      "id": "cape_presence",
      "name": "Cape / Capelet Presence",
      "description": "Does AI include capelet when required?",
      "critical": true,
      "example": {
        "rank": "Cape Prophetess",
        "prompt": "Generate a Cape Prophetess garment",
        "expected": "Square white capelet covering shoulders",
        "generic_ai_output": "Missing - generates plain robe (incorrect)",
        "verified_reference": "Present - square cut, shoulder covering",
        "evaluation": "Binary: present/absent + shape check"
      }
    },
    {
      "id": "cross_count",
      "name": "Cross Count & Placement",
      "description": "Correct number and arrangement of crosses/insignia",
      "critical": true,
      "example": {
        "rank": "Cape Prophetess",
        "prompt": "Generate a Cape Prophetess garment",
        "expected": "3 gold embroidered crosses (1 large + 2 small) on sash tails",
        "generic_ai_output": "1 generic cross or 5+ crosses, wrong placement",
        "verified_reference": "3 crosses, specific arrangement documented",
        "evaluation": "Count check + position mapping"
      }
    },
    {
      "id": "head_covering",
      "name": "Head Covering Accuracy",
      "description": "Correct head covering for rank and gender",
      "critical": true,
      "example": {
        "rank": "Cape Prophetess",
        "prompt": "Generate a Cape Prophetess garment",
        "expected": "White pleated fan crown with circular halo",
        "generic_ai_output": "No covering or Catholic veil (incorrect)",
        "verified_reference": "Fan crown pleated, halo circular",
        "evaluation": "Type classification + presence"
      }
    },
    {
      "id": "loin_arrangement",
      "name": "Loin Arrangement",
      "description": "Correct loin / wrapper arrangement",
      "critical": false,
      "example": {
        "rank": "Cape Prophetess",
        "prompt": "Generate a Cape Prophetess garment",
        "expected": "Matches sash, pleated, hidden under sash tails",
        "generic_ai_output": "Missing or Western skirt (incorrect)",
        "verified_reference": "Pleated, matches sash colour",
        "evaluation": "Presence + colour match + pleat check"
      }
    },
    {
      "id": "garment_cut",
      "name": "Garment Cut & Length",
      "description": "Correct sutana cut, length, sleeves",
      "critical": false,
      "example": {
        "rank": "Cape Prophetess",
        "prompt": "Generate a Cape Prophetess garment",
        "expected": "White full-length, long sleeves, floor touching",
        "generic_ai_output": "Short dress or sleeveless (incorrect)",
        "verified_reference": "Full-length, long sleeves documented",
        "evaluation": "Length ratio + sleeve check"
      }
    },
    {
      "id": "garment_type_confusion",
      "name": "Garment Type Confusion",
      "description": "Does AI confuse with other religious attire?",
      "critical": true,
      "example": {
        "rank": "Most Superior Evangelist",
        "prompt": "Generate Most Superior Evangelist garment",
        "expected": "CCC senior clergy garment with specific insignia",
        "generic_ai_output": "Catholic bishop mitre and cope (incorrect)",
        "verified_reference": "Senior clergy garment documented - not Catholic",
        "evaluation": "Classifier: CCC vs Catholic vs generic choir robe"
      }
    }
  ],
  "evaluation_method": {
    "manual": "Human expert checks against verified reference using checklist",
    "automated_future": "CLIP-based similarity + component detector model trained on dataset",
    "scoring": "Each test 0/1, weighted by criticality. Total score / max score = accuracy %",
    "reporting": "Per-rank report + aggregate across 50 ranks"
  },
  "products": [
    {
      "name": "Dataset Licensing",
      "description": "Companies pay to access photographs/renders/metadata - existing product",
      "status": "live"
    },
    {
      "name": "3D Asset Licensing",
      "description": "Companies pay to use .glb garment assets - component separated",
      "status": "building - Phase 1 visual standard established"
    },
    {
      "name": "AI Evaluation",
      "description": "AI companies pay to test whether their models correctly represent CCC garments. Submit AI-generated image, get benchmark report against verified reference. No need to give away entire dataset.",
      "status": "new - based on Toye recommendation - most differentiated",
      "workflow": "Company submits image + claimed rank → System checks 7 tests against verified metadata → Returns report: sash correct? cape present? cross count? etc. → Score + corrective_instruction"
    }
  ],
  "sample_report": {
    "test_prompt": "Generate a Cape Prophetess garment",
    "model": "Generic text-to-image (e.g. Midjourney / DALL-E / Stable Diffusion)",
    "results": [
      {
        "test": "Sash colour",
        "generic_ai": "Incorrect (light blue)",
        "verified": "Royal Blue",
        "status": "FAIL"
      },
      {
        "test": "Cape",
        "generic_ai": "Missing",
        "verified": "Present - square capelet",
        "status": "FAIL"
      },
      {
        "test": "Cross count",
        "generic_ai": "Incorrect (1 cross)",
        "verified": "3 crosses (1 large + 2 small)",
        "status": "FAIL"
      },
      {
        "test": "Head covering",
        "generic_ai": "Incorrect (no covering)",
        "verified": "Fan crown pleated",
        "status": "FAIL"
      },
      {
        "test": "Loin arrangement",
        "generic_ai": "Incorrect",
        "verified": "Pleated, matching sash",
        "status": "FAIL"
      },
      {
        "test": "Garment length",
        "generic_ai": "Partial",
        "verified": "Full-length",
        "status": "PARTIAL"
      },
      {
        "test": "Garment type",
        "generic_ai": "Generic white robe",
        "verified": "CCC sutana with capelet",
        "status": "FAIL"
      }
    ],
    "overall": "1/7 tests passed - demonstrates systematic hallucination, not just minor error"
  }
}