quality_report.json

{
  "schema_version": 1,
  "comparison_type": "cross-runtime output agreement against pinned BF16 source",
  "passed": true,
  "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
  "embedding_model": {
    "repo_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
    "revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42",
    "pooling": "attention-mask mean pooling followed by L2 normalization",
    "maximum_tokens": 256
  },
  "thresholds": {
    "mean_semantic_similarity": 0.55,
    "per_case_severe_regression": 0.25
  },
  "baseline_valid": true,
  "candidate_functional": true,
  "semantic_gate_passed": true,
  "validation_inputs_match": true,
  "validation_inputs": {
    "prompts_sha256": "136a918e5fee962f2b52f8e520a0275fd5ec5569181e0f5fdf4910ee3c34d528",
    "tools_sha256": "86ae46ebdbb0324c9672eec87e6b7f7683b0eb9c8c7169dcabdf646cf9bab122",
    "image_sha256": "0b1ae6badbe19a6049305c36d165a34cdf36df00033842e265339b2b7f057295"
  },
  "mean_semantic_similarity": 0.9368221938610077,
  "exact_matches": 3,
  "comparisons": [
    {
      "id": "fact",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "math",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.6446280991735537,
      "semantic_similarity": 0.9139397144317627,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "code",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.5797101449275363,
      "semantic_similarity": 0.9799442291259766,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "multiturn",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "thinking_off",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.9695290858725761,
      "semantic_similarity": 0.9862716794013977,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "reasoning_low",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.6030150753768844,
      "semantic_similarity": 0.7996187210083008,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "reasoning_high",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.6644295302013423,
      "semantic_similarity": 0.9212576150894165,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "long_form",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.10660091047040972,
      "semantic_similarity": 0.8248775005340576,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "non_english",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.7034482758620689,
      "semantic_similarity": 0.942312479019165,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "json",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    }
  ],
  "functional_results": {
    "reference_text": {
      "passed": 10,
      "total": 10
    },
    "candidate_text": {
      "passed": 10,
      "total": 10
    },
    "reference_tools": {
      "passed": 5,
      "total": 5
    },
    "candidate_tools": {
      "passed": 5,
      "total": 5
    },
    "reference_vision": {
      "passed": 3,
      "total": 3
    },
    "candidate_vision": {
      "passed": 3,
      "total": 3
    }
  },
  "measurements": {
    "average_generation_tps": 14.54951526929673,
    "peak_memory_gb": 20.756765927,
    "artifact_bytes": 19467900026,
    "maximum_prompt_tokens_tested": 73,
    "loop_rate": 0.0
  },
  "warnings": [
    "Semantic similarity is a measured embedding-model proxy, not ground-truth accuracy.",
    "Raw-logit equality is unavailable across all target runtimes; exact functional gates and output agreement are used for portable release validation.",
    "Sequence agreement is lexical and is reported diagnostically, not used as semantic accuracy."
  ]
}