audit.json

{
 "leakage": {
  "failures": [
   "held-out parents retro_29aad6b9d7d9f369 and retro_d74bf07d9615e5d2 share ('scaffold', 'C1CCC2(CC1)OCCO2')"
  ],
  "held_out_keys": 39694,
  "held_out_molecules": 11953,
  "held_out_parents": 3000,
  "train_tasks": 75224,
  "train_visible_reactions": 177822
 },
 "passed": false,
 "probes": {
  "failures": [],
  "mean_reward": {
   "duplicated_route": 0.48,
   "empty": 0.0,
   "false_stock_claim": 0.2,
   "first_step_only": 0.595,
   "oracle_one_route": 1.0
  },
  "tasks": 50
 },
 "schema": {
  "failures": [],
  "splits": {
   "dev": {
    "tasks": 1079,
    "variants": {
     "diversity": 1,
     "forbidden_class": 6,
     "max_depth": 56,
     "restricted_stock": 16,
     "standard": 1000
    }
   },
   "test_hard": {
    "tasks": 1119,
    "variants": {
     "forbidden_class": 6,
     "max_depth": 89,
     "restricted_stock": 24,
     "standard": 1000
    }
   },
   "test_id": {
    "tasks": 1083,
    "variants": {
     "diversity": 2,
     "forbidden_class": 1,
     "max_depth": 66,
     "restricted_stock": 14,
     "standard": 1000
    }
   },
   "train": {
    "tasks": 75224,
    "variants": {
     "diversity": 183,
     "forbidden_class": 190,
     "max_depth": 3659,
     "restricted_stock": 1164,
     "standard": 70028
    }
   }
  }
 },
 "solvability": {
  "checked": 5281,
  "failure_count": 0,
  "failures": [],
  "mean_oracle_reward": {
   "dev/diversity": 1.0,
   "dev/forbidden_class": 1.0,
   "dev/max_depth": 1.0,
   "dev/restricted_stock": 1.0,
   "dev/standard": 1.0,
   "test_hard/forbidden_class": 1.0,
   "test_hard/max_depth": 1.0,
   "test_hard/restricted_stock": 1.0,
   "test_hard/standard": 1.0,
   "test_id/diversity": 1.0,
   "test_id/forbidden_class": 1.0,
   "test_id/max_depth": 1.0,
   "test_id/restricted_stock": 1.0,
   "test_id/standard": 1.0,
   "train/diversity": 0.9929,
   "train/forbidden_class": 1.0,
   "train/max_depth": 1.0,
   "train/restricted_stock": 1.0,
   "train/standard": 1.0
  }
 }
}