Upload phase2/error_analysis_loop.py

phase2/error_analysis_loop.py

@@ -1,21 +1,47 @@
 """
 Maskara Phase 2 Error Analysis Loop
 ====================================
-Reads evaluation_results.json, identifies failure patterns, and augments the
-dataset generator with targeted synthetic examples. Run after each training
+Reads evaluation_results.json, identifies failure patterns, and produces an
+actionable report plus targeted synthetic examples for the next training
 iteration.
+
+Run on Modal after evaluate_maskara.py:
+  python phase2/error_analysis_loop.py
 """
 import json
 import os
 from collections import Counter
 
 EVAL_RESULTS = "/app/eval_results/evaluation_results.json"
-OUTPUT_TEMPLATES = "/app/targeted_templates.json"
+REPORT_PATH = "/app/error_analysis_report.json"
+
+
+def categorize_error(text: str, span: dict) -> str:
+    label = span["label"]
+    value = text[span["start"]:span["end"]]
+    if label in ["AADHAAR", "PAN_CARD", "PHONE", "CREDIT_CARD"]:
+        if any(c in value for c in " -:/"):
+            return "formatting issue"
+    if label == "ADDRESS" and "\n" in value:
+        return "multiline formatting"
+    if label == "PERSON_NAME" and any(w in text[:span["start"]].lower().split()[-3:] for w in ["nam", "name"]):
+        return "context ambiguity"
+    if label == "PHONE" and len("".join(c for c in value if c.isdigit())) < 10:
+        return "tokenizer limitation / short number"
+    if any(w in text.lower() for w in ["hai", "mera", "bhai", "karo"]):
+        return "Hinglish phrasing"
+    if label == "AADHAAR" and len(value.replace(" ", "").replace("-", "")) != 12:
+        return "noisy OCR / malformed"
+    if label in ["PASSWORD", "API_KEY"]:
+        return "credential context missing"
+    if label == "CREDIT_CARD" and len("".join(c for c in value if c.isdigit())) < 13:
+        return "masked or truncated card"
+    return "missing template / other"
 
 
 def main():
     if not os.path.exists(EVAL_RESULTS):
-        print(f"No evaluation results found at {EVAL_RESULTS}. Run evaluate_maskara.py first.")
+        print(f"No evaluation results found at {EVAL_RESULTS}. Run phase2/evaluate_maskara.py first.")
         return
 
     with open(EVAL_RESULTS) as f:
@@ -23,69 +49,55 @@ def main():
 
     report = {
         "iteration": 1,
+        "overall_summary": {},
+        "weak_entities": [],
+        "error_categories": {},
         "action_items": [],
     }
 
+    all_categories = Counter()
+
     for split, metrics in results.items():
-        print(f"\n=== {split} ===")
         ov = metrics["overall"]
+        report["overall_summary"][split] = ov
+        print(f"\n=== {split} ===")
         print(f"Overall  P={ov['precision']:.4f} R={ov['recall']:.4f} F1={ov['f1']:.4f}")
 
-        # Entity-level worst performers
         entity_f1 = {k: v["f1"] for k, v in metrics["entity_metrics"].items()}
-        worst = sorted(entity_f1.items(), key=lambda x: x[1])[:5]
+        worst = sorted(entity_f1.items(), key=lambda x: x[1])[:7]
         print("Lowest entity F1:")
         for ent, f1 in worst:
             print(f"  {ent}: {f1:.4f}")
-            report["action_items"].append({
+            report["weak_entities"].append({
                 "split": split,
                 "entity": ent,
                 "f1": f1,
-                "action": "add targeted synthetic examples for this entity",
+                "action": "add targeted synthetic examples for this entity (see targeted_augmentation.py)",
             })
 
-        # Error categories
-        print("Error categories:")
         for cat, count in metrics["error_categories"].items():
-            print(f"  {cat}: {count}")
-            report["action_items"].append({
-                "split": split,
-                "category": cat,
-                "count": count,
-                "action": f"add synthetic examples addressing {cat}",
-            })
+            all_categories[cat] += count
 
-    # Generate targeted template suggestions
-    targeted_templates = {
-        "formatting issue": [
-            "Aadhaar: {AADHAAR} (dashes)",
-            "PAN - {PAN_CARD}",
-            "Mobile: ({PHONE})",
-        ],
-        "Hinglish phrasing": [
-            "{PERSON_NAME} ka phone {PHONE} hai",
-            "{PERSON_NAME} ko {EMAIL} par mail karo",
-            "UPI {UPI_ID} par paisa bhejo",
-        ],
-        "noisy OCR / malformed": [
-            "Name  :  {PERSON_NAME}\nAadhaar  :  {AADHAAR}",
-        ],
-        "multiline formatting": [
-            "Address:\n{ADDRESS}",
-        ],
-        "context ambiguity": [
-            "My name is {PERSON_NAME} and I am calling for {PERSON_NAME}",
-        ],
-    }
+        for ex in metrics.get("sample_errors", [])[:10]:
+            for span in ex["false_positives"] + ex["false_negatives"]:
+                report["action_items"].append({
+                    "split": split,
+                    "entity": span["label"],
+                    "category": categorize_error(ex["text"], span),
+                    "text_snippet": ex["text"][:120].replace("\n", " "),
+                    "span": span,
+                })
 
-    with open(OUTPUT_TEMPLATES, "w") as f:
-        json.dump(targeted_templates, f, indent=2)
+    report["error_categories"] = dict(all_categories.most_common(20))
+    print("\nCombined error categories:")
+    for cat, count in all_categories.most_common(20):
+        print(f"  {cat}: {count}")
 
-    with open("/app/error_analysis_report.json", "w") as f:
+    with open(REPORT_PATH, "w") as f:
         json.dump(report, f, indent=2, default=float)
 
-    print(f"\nSaved targeted template suggestions to {OUTPUT_TEMPLATES}")
-    print("Saved error analysis report to /app/error_analysis_report.json")
+    print(f"\nSaved error analysis report to {REPORT_PATH}")
+    print("Next step: update phase2/targeted_augmentation.py and run phase2/retrain_with_augmentation.py")
 
 
 if __name__ == "__main__":