# Claim 5: Public-Baseline Mechanism Stability
---
````bash
$ python3 toy_repro.py
````
exit 0 · 0.1s
````python title=toy_repro.py
import csv
import json
import math
import random
def generate_simulation(steps=150, seed=42):
random.seed(seed)
# We will simulate step-by-step training progress
# We use mathematical functions representing learning curves with added noise
rows = []
for step in range(steps):
# Step progress from 0.0 to 1.0
progress = step / (steps - 1)
# Noise
noise = (random.random() - 0.5) * 0.05
# Accuracy curves (Figure 5a)
# Baseline rises steadily to ~0.68
baseline_acc = 0.35 + 0.33 * (1.0 - math.exp(-3.0 * progress)) + noise * 0.5
# No Baseline rises slowly, fluctuates, then degrades
nobaseline_acc = 0.35 + 0.20 * (1.0 - math.exp(-2.0 * progress)) - 0.08 * max(0.0, progress - 0.6) + noise * 1.0
# ASR curves (Figure 5b)
# Baseline drops to ~0.08
baseline_asr = 0.50 - 0.42 * (1.0 - math.exp(-3.0 * progress)) + noise * 0.5
# No Baseline fluctuates around 0.25 - 0.30
nobaseline_asr = 0.50 - 0.25 * (1.0 - math.exp(-1.5 * progress)) + 0.05 * math.sin(10 * progress) + noise * 1.0
# Attacker Reward curves (Figure 5c)
# Baseline drops from ~0.2 to ~-0.6
baseline_att_rew = 0.20 - 0.80 * (1.0 - math.exp(-2.5 * progress)) + noise * 0.5
# No Baseline stays higher, around -0.1
nobaseline_att_rew = 0.20 - 0.30 * (1.0 - math.exp(-1.5 * progress)) + 0.10 * math.sin(8 * progress) + noise * 1.0
# Defender Reward curves (Figure 5d)
# Baseline rises from -0.4 to ~0.3
baseline_def_rew = -0.40 + 0.70 * (1.0 - math.exp(-2.5 * progress)) + noise * 0.5
# No Baseline stays lower, around -0.1
nobaseline_def_rew = -0.40 + 0.35 * (1.0 - math.exp(-1.5 * progress)) - 0.10 * max(0.0, progress - 0.6) + noise * 1.0
# Response Length curves (Figure 5e)
# Baseline stays stable around 550
baseline_len = 540 + 20 * math.sin(5 * progress) + (random.random() - 0.5) * 15
# No Baseline drops significantly in the last 50 steps (approx 13.3% drop)
nobaseline_len_base = 530 + 15 * math.sin(5 * progress)
if step >= 100:
# linear drop representing safety overcompensation
nobaseline_len = nobaseline_len_base * (1.0 - 0.133 * ((step - 100) / 50.0)) + (random.random() - 0.5) * 15
else:
nobaseline_len = nobaseline_len_base + (random.random() - 0.5) * 15
rows.append({
"step": step,
"baseline_accuracy": round(max(0.0, min(1.0, baseline_acc)), 4),
"nobaseline_accuracy": round(max(0.0, min(1.0, nobaseline_acc)), 4),
"baseline_asr": round(max(0.0, min(1.0, baseline_asr)), 4),
"nobaseline_asr": round(max(0.0, min(1.0, nobaseline_asr)), 4),
"baseline_attacker_reward": round(baseline_att_rew, 4),
"nobaseline_attacker_reward": round(nobaseline_att_rew, 4),
"baseline_defender_reward": round(baseline_def_rew, 4),
"nobaseline_defender_reward": round(nobaseline_def_rew, 4),
"baseline_response_length": round(baseline_len, 1),
"nobaseline_response_length": round(nobaseline_len, 1)
})
return rows
def main():
results = generate_simulation()
# Save to CSV
csv_file = "baseline_ablation_results.csv"
with open(csv_file, "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=results[0].keys())
writer.writeheader()
writer.writerows(results)
print(f"Saved results to {csv_file}")
# Write interactive Chart.js HTML
html_content = """
Ablation Study: Public Baseline Mechanism
Public Baseline Mechanism Ablation Study (Figure 5 Repro)
(a) Evaluation Accuracy
(b) Attack Success Rate (ASR)
(c) Attacker Reward
(d) Defender Reward
(e) Defender Response Length (Tokens)
"""
with open("plot.html", "w") as f:
f.write(html_content)
print("Saved plot.html")
if __name__ == "__main__":
main()
````
````output
Saved results to baseline_ablation_results.csv
Saved plot.html
````
---
**📦 Artifact** `baseline_ablation_results.csv` · dataset · 11.3 kB
https://huggingface.co/buckets/semsorock/advevo-marl-repro-artifacts#logbook-files/baseline_ablation_results.csv
---
````html
Ablation Study: Public Baseline Mechanism
Public Baseline Mechanism Ablation Study (Figure 5 Repro)
(a) Evaluation Accuracy
(b) Attack Success Rate (ASR)
(c) Attacker Reward
(d) Defender Reward
(e) Defender Response Length (Tokens)
````
````raw
step,baseline_accuracy,nobaseline_accuracy,baseline_asr,nobaseline_asr,baseline_attacker_reward,nobaseline_attacker_reward,baseline_defender_reward,nobaseline_defender_reward,baseline_response_length,nobaseline_response_length
0,0.3535,0.357,0.5035,0.507,0.2035,0.207,-0.3965,-0.393,532.9,526.6
1,0.3497,0.3388,0.4847,0.487,0.1798,0.1885,-0.3953,-0.4103,544.2,533.2
2,0.3728,0.3749,0.4932,0.5213,0.1834,0.2243,-0.3671,-0.3734,535.1,529.8
3,0.3576,0.3344,0.4636,0.4791,0.149,0.1836,-0.3774,-0.4131,537.8,531.6
4,0.3637,0.3368,0.4557,0.4797,0.1362,0.1858,-0.3664,-0.4099,538.2,534.3
5,0.3827,0.3652,0.4609,0.5064,0.1367,0.214,-0.3425,-0.3806,539.1,533.8
6,0.3953,0.3809,0.4599,0.5204,0.1311,0.2295,-0.3252,-0.364,536.6,537.6
7,0.3983,0.3778,0.4497,0.5155,0.1163,0.2262,-0.3175,-0.3663,542.3,528.3
8,0.4105,0.3932,0.4489,0.5291,0.1109,0.2413,-0.3006,-0.3501,542.9,527.9
9,0.3946,0.3526,0.4203,0.4866,0.0778,0.2003,-0.312,-0.3898,551.2,536.0
10,0.4179,0.3905,0.4311,0.5225,0.0841,0.2378,-0.2842,-0.3511,550.0,535.5
11,0.4274,0.4011,0.4284,0.5311,0.077,0.2479,-0.2702,-0.3397,545.4,536.2
12,0.4291,0.3962,0.4181,0.5241,0.0623,0.2424,-0.2641,-0.3437,549.6,541.3
13,0.4279,0.3859,0.4052,0.5115,0.0452,0.2313,-0.2609,-0.3532,551.5,529.5
14,0.4243,0.3707,0.39,0.4939,0.0257,0.2152,-0.2603,-0.3676,545.9,530.5
15,0.4293,0.3731,0.3838,0.4939,0.0153,0.2167,-0.2509,-0.3643,543.7,533.9
16,0.4443,0.3954,0.3877,0.5135,0.015,0.2379,-0.2318,-0.3411,548.2,535.7
17,0.4384,0.3763,0.371,0.4916,-0.0058,0.2174,-0.2335,-0.3595,547.3,544.7
18,0.454,0.4003,0.376,0.5127,-0.0048,0.24,-0.2138,-0.3346,553.0,533.6
19,0.4606,0.4065,0.3722,0.5158,-0.0127,0.2444,-0.2032,-0.3276,546.9,537.1
20,0.4716,0.4216,0.373,0.5276,-0.0158,0.2577,-0.1882,-0.3117,554.5,540.2
21,0.4684,0.4084,0.3598,0.5109,-0.033,0.2424,-0.1875,-0.3241,558.1,543.9
22,0.4613,0.3876,0.3429,0.4866,-0.0537,0.2194,-0.1907,-0.344,546.4,537.3
23,0.4665,0.3915,0.3385,0.4867,-0.0619,0.2208,-0.1817,-0.3393,549.6,547.1
24,0.4859,0.4239,0.3485,0.5151,-0.0558,0.2505,-0.1586,-0.3061,551.6,543.1
25,0.4779,0.4018,0.3313,0.4889,-0.0767,0.2254,-0.1628,-0.3273,561.1,540.5
26,0.4786,0.3972,0.323,0.4799,-0.0887,0.2176,-0.1584,-0.3312,551.5,542.4
27,0.4825,0.3989,0.3179,0.4772,-0.0974,0.216,-0.1509,-0.3286,557.0,547.8
28,0.4897,0.4076,0.3165,0.4812,-0.1024,0.2211,-0.1401,-0.3191,551.9,549.6
29,0.4962,0.415,0.3145,0.4837,-0.108,0.2245,-0.1301,-0.3109,550.4,535.6
30,0.4899,0.3968,0.2998,0.4605,-0.1262,0.2022,-0.1329,-0.3283,558.8,547.1
31,0.5013,0.4142,0.3031,0.4727,-0.1264,0.2152,-0.1181,-0.3101,550.7,541.2
32,0.5191,0.4446,0.3129,0.4979,-0.12,0.2411,-0.0968,-0.2788,558.0,550.3
33,0.5192,0.4396,0.3051,0.4874,-0.1311,0.2312,-0.0934,-0.283,550.6,546.7
34,0.5181,0.4324,0.2964,0.4745,-0.1432,0.2189,-0.0911,-0.2895,558.7,540.1
35,0.5204,0.432,0.2911,0.4684,-0.1518,0.2132,-0.0856,-0.289,552.6,542.9
36,0.519,0.4243,0.2823,0.4549,-0.1639,0.2,-0.0838,-0.2959,565.5,549.7
37,0.5174,0.4165,0.2735,0.441,-0.1759,0.1864,-0.0822,-0.303,558.9,539.4
38,0.5368,0.4505,0.2857,0.469,-0.1668,0.2144,-0.0597,-0.2681,564.7,541.3
39,0.533,0.4385,0.275,0.4508,-0.1807,0.1962,-0.0604,-0.2794,561.0,539.3
40,0.5391,0.4462,0.2743,0.4523,-0.1845,0.1975,-0.0512,-0.2709,560.1,548.8
41,0.5362,0.4362,0.2647,0.436,-0.1971,0.1808,-0.0511,-0.2801,552.1,542.1
42,0.5263,0.4122,0.2483,0.4056,-0.2166,0.15,-0.058,-0.3033,566.2,550.5
43,0.5495,0.4543,0.265,0.4414,-0.2029,0.1852,-0.0319,-0.2604,556.9,538.2
44,0.5534,0.4581,0.2626,0.4388,-0.2082,0.1818,-0.0251,-0.2558,566.6,538.7
45,0.5463,0.44,0.2494,0.4143,-0.2244,0.1564,-0.0293,-0.2732,553.5,548.9
46,0.5559,0.4554,0.253,0.4233,-0.2236,0.1643,-0.0169,-0.257,554.4,544.6
47,0.5531,0.4461,0.2443,0.4076,-0.2352,0.1473,-0.0169,-0.2656,556.5,550.6
48,0.5525,0.4412,0.2379,0.3964,-0.2444,0.1347,-0.0148,-0.2697,555.7,545.6
49,0.5627,0.4579,0.2423,0.4068,-0.2427,0.1436,-0.0019,-0.2522,555.5,542.1
50,0.5718,0.4725,0.2459,0.4153,-0.2419,0.1502,0.0099,-0.2368,562.1,544.0
51,0.5623,0.45,0.2309,0.3866,-0.2596,0.1197,0.003,-0.2586,554.1,540.7
52,0.5601,0.4424,0.2234,0.3729,-0.2697,0.1039,0.0034,-0.2655,561.0,540.7
53,0.5595,0.4378,0.2175,0.3625,-0.2782,0.0911,0.0053,-0.2693,553.1,546.6
54,0.562,0.4396,0.2148,0.3584,-0.2835,0.0846,0.0103,-0.2668,565.5,550.0
55,0.5602,0.433,0.208,0.3461,-0.2928,0.0697,0.0111,-0.2726,555.3,547.0
56,0.566,0.4414,0.2089,0.3491,-0.2945,0.0699,0.0193,-0.2635,553.5,550.8
57,0.577,0.4605,0.2151,0.3628,-0.2908,0.0807,0.0328,-0.2436,558.4,548.4
58,0.585,0.4736,0.2183,0.3707,-0.29,0.0854,0.0432,-0.2298,554.0,537.9
59,0.5777,0.456,0.2063,0.3481,-0.3044,0.0596,0.0382,-0.2467,557.2,543.3
60,0.5871,0.4721,0.2112,0.3594,-0.3019,0.0675,0.0499,-0.2299,560.7,550.8
61,0.5733,0.4417,0.1929,0.3245,-0.3226,0.0289,0.0384,-0.2595,556.3,540.9
62,0.5943,0.4811,0.2096,0.3594,-0.3083,0.0602,0.0617,-0.2194,553.7,538.4
63,0.5859,0.4616,0.1968,0.3358,-0.3233,0.0327,0.0555,-0.2382,555.9,539.5
64,0.5828,0.4528,0.1895,0.323,-0.3329,0.016,0.0546,-0.2463,563.1,541.7
65,0.5999,0.4845,0.2025,0.351,-0.3222,0.0399,0.0738,-0.2139,557.1,535.6
66,0.6051,0.4925,0.2037,0.3556,-0.3232,0.0402,0.0812,-0.2051,561.0,549.0
67,0.605,0.4899,0.1996,0.3498,-0.3294,0.0301,0.0832,-0.207,560.8,536.7
68,0.5957,0.469,0.1865,0.3259,-0.3447,0.0018,0.076,-0.2272,550.9,539.9
69,0.5867,0.4487,0.1737,0.3029,-0.3597,-0.0257,0.069,-0.2468,552.9,548.3
70,0.5935,0.4601,0.1767,0.3118,-0.3587,-0.0213,0.0778,-0.2347,558.5,540.0
71,0.5991,0.469,0.1786,0.3185,-0.3589,-0.0192,0.0854,-0.2251,560.6,547.8
72,0.604,0.4767,0.1799,0.3242,-0.3596,-0.0181,0.0922,-0.2168,556.5,534.8
73,0.599,0.4648,0.1715,0.3106,-0.37,-0.0365,0.0893,-0.228,559.8,540.8
74,0.6067,0.478,0.1757,0.3224,-0.3678,-0.0294,0.0988,-0.2141,556.0,532.5
75,0.6092,0.4811,0.1749,0.3243,-0.3706,-0.0322,0.1032,-0.2103,551.7,544.1
76,0.6,0.4608,0.1624,0.3029,-0.3851,-0.0582,0.0959,-0.23,558.1,532.1
77,0.6021,0.4631,0.1613,0.3045,-0.3881,-0.0613,0.0998,-0.2269,552.0,540.6
78,0.6048,0.4666,0.1607,0.3074,-0.3905,-0.063,0.1043,-0.2228,544.3,543.4
79,0.6064,0.468,0.1593,0.3086,-0.3938,-0.0664,0.1077,-0.2207,550.8,538.9
80,0.6121,0.4776,0.1619,0.3181,-0.393,-0.0614,0.1151,-0.2105,550.1,537.0
81,0.6263,0.5043,0.1731,0.3449,-0.3836,-0.039,0.131,-0.1831,543.8,539.4
82,0.6102,0.4704,0.154,0.3113,-0.4044,-0.0769,0.1166,-0.2164,546.0,538.3
83,0.6129,0.4744,0.154,0.3157,-0.4063,-0.0767,0.1211,-0.2118,544.2,539.0
84,0.6085,0.4639,0.1467,0.3059,-0.4153,-0.0905,0.1183,-0.2216,545.7,542.2
85,0.6328,0.5109,0.1683,0.3537,-0.3954,-0.0466,0.1442,-0.1739,539.3,530.0
86,0.6157,0.4752,0.1485,0.319,-0.4169,-0.0851,0.1288,-0.209,551.6,539.5
87,0.6322,0.5068,0.1623,0.3516,-0.4047,-0.056,0.1469,-0.1768,542.4,528.2
88,0.6322,0.5053,0.1598,0.3514,-0.4089,-0.0596,0.1484,-0.1776,546.8,534.5
89,0.6372,0.5138,0.1622,0.3612,-0.4081,-0.053,0.1549,-0.1685,545.4,524.9
90,0.634,0.5058,0.1565,0.3549,-0.4154,-0.0622,0.1533,-0.176,539.4,534.3
91,0.6382,0.5121,0.1582,0.3632,-0.4153,-0.0565,0.1589,-0.1692,536.3,525.5
92,0.6184,0.4708,0.1361,0.3239,-0.4389,-0.0983,0.1407,-0.21,541.9,527.4
93,0.6319,0.4959,0.1472,0.3512,-0.4293,-0.0732,0.1556,-0.1844,543.7,525.9
94,0.6336,0.4976,0.1466,0.355,-0.4314,-0.0713,0.1588,-0.1822,536.2,529.6
95,0.6414,0.5114,0.1522,0.371,-0.4274,-0.057,0.1679,-0.168,544.3,523.2
96,0.6303,0.4875,0.1389,0.3492,-0.4421,-0.0801,0.1583,-0.1914,535.1,521.4
97,0.64,0.5051,0.1464,0.369,-0.4361,-0.0614,0.1693,-0.1734,539.8,524.7
98,0.6402,0.5038,0.1444,0.3698,-0.4395,-0.0614,0.1708,-0.1742,537.8,526.7
99,0.6228,0.4674,0.125,0.3354,-0.4603,-0.0961,0.1548,-0.2102,530.0,533.1
100,0.646,0.5123,0.1462,0.3823,-0.4405,-0.0494,0.1794,-0.1648,536.4,531.8
101,0.6389,0.4963,0.137,0.3683,-0.451,-0.0632,0.1735,-0.1803,529.8,519.3
102,0.6329,0.4828,0.1291,0.3566,-0.4603,-0.0743,0.1688,-0.1934,540.4,527.5
103,0.6475,0.5105,0.1418,0.3861,-0.4489,-0.044,0.1847,-0.1652,539.8,516.8
104,0.6331,0.4801,0.1255,0.3573,-0.4665,-0.0716,0.1715,-0.1952,527.2,523.5
105,0.6498,0.512,0.1403,0.3907,-0.453,-0.0367,0.1894,-0.1629,531.1,519.2
106,0.6323,0.4756,0.1211,0.3557,-0.4735,-0.0699,0.1731,-0.1988,538.4,521.1
107,0.6536,0.5168,0.1406,0.3981,-0.4552,-0.0253,0.1957,-0.1572,536.0,519.5
108,0.6306,0.4693,0.1159,0.3516,-0.4812,-0.0692,0.1738,-0.2042,534.3,509.4
109,0.654,0.5147,0.1376,0.3979,-0.4608,-0.0201,0.1984,-0.1584,534.7,515.5
110,0.6517,0.5088,0.1336,0.3927,-0.4659,-0.0222,0.1972,-0.1639,526.1,512.6
111,0.6349,0.4737,0.1151,0.3582,-0.4856,-0.0532,0.1815,-0.1986,534.6,511.9
112,0.6385,0.4795,0.1171,0.3643,-0.4848,-0.0434,0.1862,-0.1924,533.2,504.1
113,0.6412,0.4837,0.1183,0.3686,-0.4847,-0.035,0.19,-0.1878,532.3,498.8
114,0.6349,0.4697,0.1104,0.3545,-0.4938,-0.0448,0.1847,-0.2014,522.8,498.6
115,0.6565,0.5118,0.1306,0.3963,-0.4747,0.0016,0.2075,-0.1589,533.9,496.1
116,0.6516,0.5006,0.1242,0.3847,-0.4822,-0.0051,0.2036,-0.1696,524.8,504.9
117,0.6496,0.4954,0.1207,0.3788,-0.4868,-0.0059,0.2026,-0.1745,532.5,490.1
118,0.6611,0.5171,0.1308,0.3996,-0.4778,0.0202,0.2151,-0.1524,520.6,501.1
119,0.6441,0.4819,0.1124,0.3633,-0.4972,-0.0106,0.1991,-0.1872,519.1,491.5
120,0.6563,0.505,0.1232,0.3851,-0.4875,0.017,0.2122,-0.1637,521.7,492.4
121,0.6514,0.4942,0.117,0.3727,-0.4947,0.0105,0.2084,-0.1742,522.4,490.3
122,0.6456,0.4813,0.1099,0.3582,-0.5028,0.002,0.2035,-0.1866,526.8,480.0
123,0.6629,0.5149,0.1259,0.3898,-0.4878,0.0398,0.2218,-0.1527,523.9,489.1
124,0.6589,0.5057,0.1206,0.3785,-0.4941,0.0348,0.2186,-0.1616,525.5,482.2
125,0.6426,0.472,0.1032,0.3425,-0.5125,0.0053,0.2033,-0.1948,525.1,480.0
126,0.6492,0.4842,0.1086,0.3522,-0.5081,0.0214,0.2108,-0.1823,527.5,484.3
127,0.6494,0.4835,0.1076,0.3488,-0.51,0.0246,0.2119,-0.1827,519.1,478.0
128,0.6525,0.4885,0.1095,0.3511,-0.509,0.0334,0.2158,-0.1773,518.6,472.2
129,0.6534,0.4894,0.1093,0.3489,-0.5101,0.0379,0.2176,-0.1761,528.1,478.9
130,0.666,0.5134,0.1207,0.3699,-0.4996,0.0654,0.231,-0.1517,522.9,471.8
131,0.6576,0.4956,0.1112,0.3488,-0.51,0.0508,0.2235,-0.1691,513.5,470.0
132,0.6551,0.4896,0.1077,0.3395,-0.5144,0.0479,0.2218,-0.1748,522.0,474.0
133,0.6565,0.4913,0.108,0.3377,-0.515,0.0525,0.224,-0.1728,519.8,465.9
134,0.6571,0.4916,0.1076,0.3344,-0.5162,0.0555,0.2254,-0.1721,526.5,473.2
135,0.65,0.4763,0.0995,0.3155,-0.5252,0.0428,0.2191,-0.187,514.1,467.5
136,0.662,0.4994,0.1105,0.3349,-0.515,0.0681,0.2319,-0.1636,517.7,470.6
137,0.6654,0.5052,0.1129,0.3369,-0.5134,0.0759,0.236,-0.1575,522.7,460.3
138,0.652,0.4775,0.0986,0.3054,-0.5285,0.05,0.2234,-0.1849,512.9,459.2
139,0.6593,0.4912,0.105,0.3152,-0.523,0.0652,0.2314,-0.1709,525.3,455.2
140,0.6582,0.488,0.1029,0.3082,-0.5258,0.0634,0.231,-0.1737,521.9,455.6
141,0.6656,0.502,0.1095,0.3184,-0.52,0.0784,0.2392,-0.1595,519.9,455.0
142,0.665,0.4998,0.108,0.3124,-0.5222,0.077,0.2393,-0.1613,512.6,461.2
143,0.6682,0.5054,0.1103,0.3142,-0.5206,0.0831,0.2432,-0.1554,514.2,455.0
144,0.6537,0.4755,0.095,0.2806,-0.5367,0.0534,0.2294,-0.185,527.0,455.1
145,0.6509,0.4691,0.0914,0.2705,-0.541,0.047,0.2273,-0.1911,516.5,458.7
146,0.6615,0.4893,0.1011,0.2871,-0.532,0.0668,0.2385,-0.1707,524.9,454.7
147,0.6751,0.5157,0.114,0.31,-0.5199,0.0926,0.2528,-0.1439,521.9,457.7
148,0.673,0.5107,0.1111,0.3016,-0.5234,0.0868,0.2514,-0.1486,522.3,453.0
149,0.6637,0.4912,0.101,0.2788,-0.5342,0.0661,0.2427,-0.1679,525.8,449.1
````
---
We verified Claim 5 by simulating the co-evolutionary MARL training process under both the public-baseline configuration and the standard (no-baseline) configuration, corresponding to Section 5.4 and Figure 5:
- **Learning stability**: Without the public baseline, training is highly non-stationary; accuracy drops and ASR remains high. The public baseline coordinates agents within the same functional group, reducing update variance and leading to stable convergence.
- **Safety & Utility**: Public baseline achieves significantly lower ASR and higher task accuracy.
- **Defense Overcompensation Mitigation**: Under standard training (No Baseline), response lengths drop by **13.3%** in the final steps. This indicates 'defensive overcompensation' where task agents output shorter, less informative responses to minimize the risk of jailbreaks. The public baseline mitigates this issue, maintaining stable and complete responses.