diff --git "a/hns/build_metadata/hns_f2_s1/spectral_edit_meta.json" "b/hns/build_metadata/hns_f2_s1/spectral_edit_meta.json" new file mode 100644--- /dev/null +++ "b/hns/build_metadata/hns_f2_s1/spectral_edit_meta.json" @@ -0,0 +1,15899 @@ +{ + "meta": { + "method": "posthoc_hns", + "source_lora": "/magicoder", + "scope": "all_modules", + "fast_steps": 2, + "stable_steps": 1, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "is_svd_reconstruction_control": false + }, + "summary": { + "num_modules": 252, + "mean_effective_rank_before": 10.281708520678572, + "mean_effective_rank_after": 15.923773571102778, + "adapter_fro_before": 2.9731058469475404, + "adapter_fro_after": 1.7949345031617032, + "adapter_fro_ratio": 0.6037237137065083, + "max_saved_update_relative_error": 0.0 + }, + "module_stats": { + "base_model.model.model.layers.0.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.759818682581951, + "effective_rank_before": 9.759818682581951, + "effective_rank_after": 15.987370711014664, + "nuclear_norm_before_full": 0.4271538918837905, + "nuclear_norm_before": 0.4271538918837905, + "nuclear_norm_after": 0.4271538918837905, + "fro_norm_before_full": 0.17418302820668183, + "fro_norm_after": 0.10687214195286754, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1507236212491989, + 0.05889657139778137, + 0.03420061990618706, + 0.029478706419467926, + 0.021824825555086136, + 0.015788547694683075, + 0.015325453132390976, + 0.013847226276993752, + 0.012921072542667389, + 0.012371138669550419, + 0.011854982934892178, + 0.011141561903059483, + 0.010628205724060535, + 0.010026084259152412, + 0.009069821797311306, + 0.009055452421307564 + ], + "sigma_after": [ + 0.02717591486675046, + 0.026416400986642052, + 0.028066059169317675, + 0.028602844072039762, + 0.027535756198478034, + 0.027180357918658436, + 0.027189326738702918, + 0.02714982457547553, + 0.027021909626845, + 0.02688742163626647, + 0.026711387906006856, + 0.02637416342043582, + 0.02605408532773526, + 0.025585427264281396, + 0.024609993740877068, + 0.024593018435277763 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.0.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 14.022500336778304, + "effective_rank_before": 14.022500336778304, + "effective_rank_after": 15.996159237450463, + "nuclear_norm_before_full": 0.5514505449682474, + "nuclear_norm_before": 0.5514505449682474, + "nuclear_norm_after": 0.5514505449682474, + "fro_norm_before_full": 0.15876089579015032, + "fro_norm_after": 0.13789557065982785, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.0912172794342041, + 0.07052672654390335, + 0.05303370580077171, + 0.039624299854040146, + 0.03262173756957054, + 0.03129953518509865, + 0.027399323880672455, + 0.025914214551448822, + 0.025443241000175476, + 0.02372812107205391, + 0.023283906280994415, + 0.02258637361228466, + 0.02222456783056259, + 0.02158087119460106, + 0.02078208513557911, + 0.020184556022286415 + ], + "sigma_after": [ + 0.03364939977857642, + 0.03334736415653379, + 0.03279753606636429, + 0.033613087020554426, + 0.03441296878456878, + 0.03472477506588263, + 0.03539631435732604, + 0.03540490974387066, + 0.035370717588915655, + 0.035105207797500884, + 0.035005645669294885, + 0.03483091229083695, + 0.034733744627862294, + 0.03455472775693053, + 0.03433146508517494, + 0.03417176917805422 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.0.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.492143068498807, + "effective_rank_before": 13.492143068498807, + "effective_rank_after": 15.99561343105195, + "nuclear_norm_before_full": 0.5750494729727507, + "nuclear_norm_before": 0.5750494729727507, + "nuclear_norm_after": 0.5750494729727507, + "fro_norm_before_full": 0.17385469612711052, + "fro_norm_after": 0.14380157917325542, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1182376891374588, + 0.06428247690200806, + 0.05434524267911911, + 0.03660374507308006, + 0.03438723459839821, + 0.032576560974121094, + 0.028873823583126068, + 0.026107631623744965, + 0.025254368782043457, + 0.024970397353172302, + 0.02327197603881359, + 0.02284301444888115, + 0.021542277187108994, + 0.021196134388446808, + 0.020689871162176132, + 0.019867029041051865 + ], + "sigma_after": [ + 0.03485123721093503, + 0.03395101196538073, + 0.034935608564796584, + 0.0359466552222954, + 0.03643684792009517, + 0.036824028616397185, + 0.0371932021214633, + 0.03688643469976893, + 0.036701011695861194, + 0.03663265373154897, + 0.036186414362891335, + 0.036071722038081094, + 0.035748159166215945, + 0.03567233523235237, + 0.0355720488046516, + 0.035440101620015936 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.0.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 14.496710815239075, + "effective_rank_before": 14.496710815239075, + "effective_rank_after": 15.994305852041578, + "nuclear_norm_before_full": 0.16267558885738254, + "nuclear_norm_before": 0.16267558885738254, + "nuclear_norm_after": 0.16267558885738254, + "fro_norm_before_full": 0.045096535402077934, + "fro_norm_after": 0.04068329910491081, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.024505598470568657, + 0.016722287982702255, + 0.014197345823049545, + 0.012823279947042465, + 0.012209068052470684, + 0.011181170120835304, + 0.008508102037012577, + 0.008232534863054752, + 0.007945343852043152, + 0.0075992015190422535, + 0.007232775446027517, + 0.0070309825241565704, + 0.006730030290782452, + 0.0062833684496581554, + 0.005916906986385584, + 0.0055575924925506115 + ], + "sigma_after": [ + 0.009951995898851474, + 0.009563462693095015, + 0.009827447550751715, + 0.009953378185290393, + 0.009958648076212861, + 0.009942383490482869, + 0.010360847854034815, + 0.010414839849812077, + 0.01045561858218257, + 0.010477350802706415, + 0.010462844869523173, + 0.010438169513301124, + 0.010381564369231948, + 0.010265463965572402, + 0.010158844067037669, + 0.010062729089296034 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.0.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.583339799426945, + "effective_rank_before": 11.583339799426945, + "effective_rank_after": 15.995875128053886, + "nuclear_norm_before_full": 0.3159178188070655, + "nuclear_norm_before": 0.3159178188070655, + "nuclear_norm_after": 0.3159178188070655, + "fro_norm_before_full": 0.10906827375128302, + "fro_norm_after": 0.0789999001291991, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.0802629292011261, + 0.04602064937353134, + 0.03746357560157776, + 0.01959896832704544, + 0.01680564135313034, + 0.014105274342000484, + 0.013598654419183731, + 0.01220343541353941, + 0.011412348598241806, + 0.010831809602677822, + 0.010378733277320862, + 0.00973917730152607, + 0.009433058090507984, + 0.008458693511784077, + 0.00832596980035305, + 0.007278900593519211 + ], + "sigma_after": [ + 0.018988103568020165, + 0.01933015745055667, + 0.019119147577102483, + 0.020685461478412315, + 0.02066464003762836, + 0.02008955730721669, + 0.019978529603881297, + 0.019761118953359514, + 0.019712894948254275, + 0.0197094802593192, + 0.019719502935967616, + 0.019738312163824306, + 0.019742680160252192, + 0.019687350702873363, + 0.019666731674339386, + 0.01932414998605766 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.0.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.277906057185536, + "effective_rank_before": 13.277906057185536, + "effective_rank_after": 15.997028675107472, + "nuclear_norm_before_full": 0.29705796111375093, + "nuclear_norm_before": 0.29705796111375093, + "nuclear_norm_after": 0.29705796111375093, + "fro_norm_before_full": 0.09121125877756245, + "fro_norm_after": 0.07427828583586947, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.060132723301649094, + 0.04309914633631706, + 0.023747900500893593, + 0.017996199429035187, + 0.015476631931960583, + 0.014591265469789505, + 0.014365150593221188, + 0.01379616279155016, + 0.013116536661982536, + 0.01287174504250288, + 0.012401942163705826, + 0.0120138730853796, + 0.01147609855979681, + 0.011226996779441833, + 0.010734332725405693, + 0.010011255741119385 + ], + "sigma_after": [ + 0.01804319128552692, + 0.018125546373082863, + 0.01813688523093517, + 0.018719614585065263, + 0.019096271770999346, + 0.019068684936559832, + 0.019045262608822108, + 0.0189591444329463, + 0.018814077765107684, + 0.018753820968828553, + 0.01863190410031148, + 0.01852985694272663, + 0.018395844265731653, + 0.018339646929311375, + 0.018244643160296517, + 0.018153565757499213 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.0.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.58020412635125, + "effective_rank_before": 11.58020412635125, + "effective_rank_after": 15.997728910826853, + "nuclear_norm_before_full": 0.1430797278881073, + "nuclear_norm_before": 0.1430797278881073, + "nuclear_norm_after": 0.1430797278881073, + "fro_norm_before_full": 0.04941469870540165, + "fro_norm_after": 0.035775048892014036, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.033699698746204376, + 0.027336614206433296, + 0.013007419183850288, + 0.008032168261706829, + 0.00718044675886631, + 0.0066472552716732025, + 0.006337197031825781, + 0.005904322490096092, + 0.005096934270113707, + 0.00476851686835289, + 0.004702376667410135, + 0.004560282453894615, + 0.00428195483982563, + 0.004119397606700659, + 0.003823040286079049, + 0.003582102945074439 + ], + "sigma_after": [ + 0.008750507401183476, + 0.008877845985108272, + 0.008878962216010713, + 0.009340959374129692, + 0.009223651615128175, + 0.009101613415868992, + 0.009029329727929138, + 0.008943343798956496, + 0.008870372939556262, + 0.008873031859813847, + 0.008874841684509818, + 0.008879213724340427, + 0.008885239974070366, + 0.00888338775320012, + 0.00885906341202502, + 0.008808363006276467 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.1.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.774325165042232, + "effective_rank_before": 7.774325165042232, + "effective_rank_after": 15.864464987405285, + "nuclear_norm_before_full": 0.36791297933086753, + "nuclear_norm_before": 0.36791297933086753, + "nuclear_norm_after": 0.3679129793308676, + "fro_norm_before_full": 0.17853043271573726, + "fro_norm_after": 0.0927352337380699, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.16559454798698425, + 0.048460107296705246, + 0.026598429307341576, + 0.016864970326423645, + 0.015428670682013035, + 0.012523924931883812, + 0.01194668561220169, + 0.01050332561135292, + 0.009761182591319084, + 0.008977732621133327, + 0.008068924769759178, + 0.007706542499363422, + 0.007002212107181549, + 0.006440327037125826, + 0.006133498623967171, + 0.0059018973261117935 + ], + "sigma_after": [ + 0.02562444588101126, + 0.025666341608846833, + 0.02674888532044778, + 0.02564122120004691, + 0.025668446091578104, + 0.02533739612364522, + 0.02513724409630859, + 0.0243266156315896, + 0.023703406175873992, + 0.022869840367802607, + 0.02165754094589723, + 0.02109705730615147, + 0.01987862548278787, + 0.018783583703142405, + 0.018139757988519883, + 0.017632571407217818 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.1.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.599108269081027, + "effective_rank_before": 8.599108269081027, + "effective_rank_after": 15.762864100995268, + "nuclear_norm_before_full": 0.30535146268084645, + "nuclear_norm_before": 0.30535146268084645, + "nuclear_norm_after": 0.30535146268084645, + "fro_norm_before_full": 0.1330178149529974, + "fro_norm_after": 0.07740107536046664, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11540109664201736, + 0.04803438112139702, + 0.026913391426205635, + 0.021083801984786987, + 0.016494981944561005, + 0.01206692960113287, + 0.01044480875134468, + 0.009586901403963566, + 0.008528138510882854, + 0.008181099779903889, + 0.006322136148810387, + 0.005357068497687578, + 0.005046812351793051, + 0.004563213791698217, + 0.003727816743776202, + 0.003598883980885148 + ], + "sigma_after": [ + 0.02138320450698794, + 0.02056682050254125, + 0.021936531442231164, + 0.0224526338064674, + 0.021630690215311638, + 0.02138240095270446, + 0.021346244820791723, + 0.021191761895859217, + 0.020759653747727022, + 0.02054196239220774, + 0.01853991985754582, + 0.016851097048830043, + 0.016206053643053636, + 0.015101174718791067, + 0.012914438353269676, + 0.012546874776526667 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.1.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.726958050172108, + "effective_rank_before": 7.726958050172108, + "effective_rank_after": 15.56014683520953, + "nuclear_norm_before_full": 0.3316703678574413, + "nuclear_norm_before": 0.3316703678574413, + "nuclear_norm_after": 0.33167036785744125, + "fro_norm_before_full": 0.1596077380605881, + "fro_norm_after": 0.08500744786918712, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14784200489521027, + 0.03840481862425804, + 0.026567751541733742, + 0.019833948463201523, + 0.018897701054811478, + 0.012530562467873096, + 0.01118557434529066, + 0.01044232677668333, + 0.00900163222104311, + 0.008310746401548386, + 0.006207828875631094, + 0.0059091695584356785, + 0.005504888482391834, + 0.004211223218590021, + 0.0035893081221729517, + 0.0032308828085660934 + ], + "sigma_after": [ + 0.02430860082683589, + 0.024322543893716936, + 0.02561638262798044, + 0.024630271157640528, + 0.02448605548622944, + 0.024299123591871936, + 0.024033345327180128, + 0.023738603543792643, + 0.02274955794210627, + 0.02203878700416604, + 0.018755066470213633, + 0.018143499413876562, + 0.01725755148841118, + 0.013987941553763166, + 0.012196667889857443, + 0.011106369639799045 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.1.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.964493249680691, + "effective_rank_before": 9.964493249680691, + "effective_rank_after": 15.994366138748545, + "nuclear_norm_before_full": 0.1977805895730853, + "nuclear_norm_before": 0.1977805895730853, + "nuclear_norm_after": 0.1977805895730853, + "fro_norm_before_full": 0.08058215317031685, + "fro_norm_after": 0.04946243990425245, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07110193371772766, + 0.023374589160084724, + 0.015934431925415993, + 0.0111001618206501, + 0.009466659277677536, + 0.008557293564081192, + 0.008232679218053818, + 0.007032071240246296, + 0.0066213891841471195, + 0.006103552412241697, + 0.0055540334433317184, + 0.005250005517154932, + 0.005172446370124817, + 0.00503748282790184, + 0.004836500622332096, + 0.004405359271913767 + ], + "sigma_after": [ + 0.012497661478786885, + 0.012475692856364455, + 0.012880671518135402, + 0.012855809931813874, + 0.012557633849325879, + 0.012482858317674112, + 0.012476402465184744, + 0.012497691471389455, + 0.012492142057450816, + 0.01244266163596189, + 0.012303455015128792, + 0.012172877482911518, + 0.01213227355113922, + 0.012053927064411355, + 0.011918019548994664, + 0.011540811328412252 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.1.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.53244561212599, + "effective_rank_before": 9.53244561212599, + "effective_rank_after": 15.980086250629643, + "nuclear_norm_before_full": 0.376931581646204, + "nuclear_norm_before": 0.376931581646204, + "nuclear_norm_after": 0.376931581646204, + "fro_norm_before_full": 0.15876633582741032, + "fro_norm_after": 0.09434785195205693, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14157705008983612, + 0.046331025660037994, + 0.02896423079073429, + 0.022271575406193733, + 0.017330734059214592, + 0.01492332760244608, + 0.013677501119673252, + 0.013560127466917038, + 0.012132623232901096, + 0.01163810957223177, + 0.010682167485356331, + 0.010500128380954266, + 0.009386776946485043, + 0.008609909564256668, + 0.007955480366945267, + 0.007390813902020454 + ], + "sigma_after": [ + 0.024096770400841794, + 0.024051386672193083, + 0.025209593715556436, + 0.024875262015248242, + 0.024098321418418014, + 0.02408235570735861, + 0.024106067038521563, + 0.024105797398568177, + 0.024015055483084698, + 0.02392683682885117, + 0.02363133406257622, + 0.02355270698096112, + 0.022881506071981778, + 0.0221891388391046, + 0.02144205060594772, + 0.02066739840699075 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.1.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.728642731670089, + "effective_rank_before": 8.728642731670089, + "effective_rank_after": 15.95486880587951, + "nuclear_norm_before_full": 0.32697975682094693, + "nuclear_norm_before": 0.32697975682094693, + "nuclear_norm_after": 0.32697975682094693, + "fro_norm_before_full": 0.14805827956394577, + "fro_norm_after": 0.08196973613532717, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.13593743741512299, + 0.03882428631186485, + 0.022046998143196106, + 0.01815866306424141, + 0.013616492040455341, + 0.012782212346792221, + 0.012166121043264866, + 0.011418618261814117, + 0.009528933092951775, + 0.00905585940927267, + 0.008425358682870865, + 0.008158483542501926, + 0.007424929179251194, + 0.0070794071070849895, + 0.006214240100234747, + 0.006141717080026865 + ], + "sigma_after": [ + 0.02146356546366937, + 0.021482485950217477, + 0.022403200911703527, + 0.021708382049756336, + 0.021486404630764223, + 0.021499963004182325, + 0.021489706112125393, + 0.021432189501389354, + 0.0208834246420557, + 0.02061443968309777, + 0.02015014382763716, + 0.019913804293020622, + 0.019131237214498626, + 0.018691211558356187, + 0.0173769486052284, + 0.017252649373244472 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.1.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 6.7272061782419845, + "effective_rank_before": 6.7272061782419845, + "effective_rank_after": 15.224726318482686, + "nuclear_norm_before_full": 0.16773979680147022, + "nuclear_norm_before": 0.16773979680147022, + "nuclear_norm_after": 0.16773979680147022, + "fro_norm_before_full": 0.08796170719871424, + "fro_norm_after": 0.043829965532708764, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08315261453390121, + 0.019090477377176285, + 0.01318622101098299, + 0.008794817142188549, + 0.00792821403592825, + 0.006322578527033329, + 0.005984226241707802, + 0.004248057492077351, + 0.004050846677273512, + 0.003056746209040284, + 0.0026504492852836847, + 0.0024184915237128735, + 0.0019508951809257269, + 0.0018606416415423155, + 0.0016087692929431796, + 0.0014357506297528744 + ], + "sigma_after": [ + 0.013459050622083066, + 0.013609842122968084, + 0.014046010759212224, + 0.01344354580123934, + 0.013462202660044303, + 0.013336623658548295, + 0.013227950884333315, + 0.011766496322555144, + 0.011479452246181736, + 0.009596075669423143, + 0.008616003427804686, + 0.008004170466832298, + 0.006663380559405047, + 0.006389222812980141, + 0.0056002755359788265, + 0.005039493251880581 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.10.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 6.493101338597557, + "effective_rank_before": 6.493101338597557, + "effective_rank_after": 15.815614004391959, + "nuclear_norm_before_full": 0.33712818194180727, + "nuclear_norm_before": 0.33712818194180727, + "nuclear_norm_after": 0.33712818194180727, + "fro_norm_before_full": 0.18697772172035867, + "fro_norm_after": 0.08525124641385856, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.18046468496322632, + 0.034069254994392395, + 0.018980130553245544, + 0.011565858498215675, + 0.011411589570343494, + 0.009739836677908897, + 0.00893536489456892, + 0.008460172452032566, + 0.007892923429608345, + 0.007517342921346426, + 0.006976074539124966, + 0.006624429952353239, + 0.006576974410563707, + 0.006340770050883293, + 0.0060596889816224575, + 0.005513085052371025 + ], + "sigma_after": [ + 0.025261318220253515, + 0.02635559800177376, + 0.025153568711234774, + 0.02423466073371254, + 0.024145193134732963, + 0.022809242879313556, + 0.021899692829792552, + 0.021273495115967593, + 0.02043649038358276, + 0.019827848261489558, + 0.018873855881918304, + 0.01820555275374895, + 0.018112453109111856, + 0.01763883378958394, + 0.017053210511598266, + 0.015847167623992384 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.10.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.925269820814774, + "effective_rank_before": 10.925269820814774, + "effective_rank_after": 15.99816741674836, + "nuclear_norm_before_full": 0.5709119690582156, + "nuclear_norm_before": 0.5709119690582156, + "nuclear_norm_after": 0.5709119690582156, + "fro_norm_before_full": 0.21977448777466285, + "fro_norm_after": 0.1427445059433685, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.19374753534793854, + 0.05277149751782417, + 0.03757109493017197, + 0.03116423636674881, + 0.02721499465405941, + 0.026784496381878853, + 0.024572579190135002, + 0.02321346290409565, + 0.021958760917186737, + 0.021339653059840202, + 0.020557036623358727, + 0.02005298249423504, + 0.019168898463249207, + 0.018554827198386192, + 0.016682304441928864, + 0.01555760856717825 + ], + "sigma_after": [ + 0.03549665992782869, + 0.03545913731765211, + 0.03733010258217418, + 0.03669871935288878, + 0.03588744676107498, + 0.03581271824104748, + 0.035528998465505265, + 0.035449857540634115, + 0.035436395928680246, + 0.035445564865994976, + 0.03546541010200023, + 0.03547935138338888, + 0.03549634985035124, + 0.03549483404691742, + 0.03534587484146923, + 0.03508454785060786 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.10.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.69010983109159, + "effective_rank_before": 9.69010983109159, + "effective_rank_after": 15.991334172257542, + "nuclear_norm_before_full": 0.5891149174422026, + "nuclear_norm_before": 0.5891149174422026, + "nuclear_norm_after": 0.5891149174422026, + "fro_norm_before_full": 0.2483561709783953, + "fro_norm_after": 0.14735773431427698, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2243819385766983, + 0.06424851715564728, + 0.03954159468412399, + 0.032339178025722504, + 0.028272762894630432, + 0.025764968246221542, + 0.0226593054831028, + 0.0206295158714056, + 0.020170751959085464, + 0.018218528479337692, + 0.017961056903004646, + 0.016829296946525574, + 0.016212239861488342, + 0.014882652088999748, + 0.014048155397176743, + 0.012954454869031906 + ], + "sigma_after": [ + 0.037248436091844306, + 0.037244515464424637, + 0.039151064074621156, + 0.037985119460979454, + 0.03736528681941995, + 0.037227230676067494, + 0.037268849132522944, + 0.037277740063423846, + 0.03725893601258146, + 0.03701347777876528, + 0.0369547546737821, + 0.036600937002508285, + 0.03633281398308379, + 0.035540291393141354, + 0.03487250541464027, + 0.03377295940039622 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.10.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.05894590089853, + "effective_rank_before": 11.05894590089853, + "effective_rank_after": 15.992364478124577, + "nuclear_norm_before_full": 0.14620700059458613, + "nuclear_norm_before": 0.14620700059458613, + "nuclear_norm_after": 0.14620700059458613, + "fro_norm_before_full": 0.05275479883515329, + "fro_norm_after": 0.03656909685323122, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.040980298072099686, + 0.022255808115005493, + 0.013815661892294884, + 0.00973816029727459, + 0.008505753241479397, + 0.006614794488996267, + 0.00616081478074193, + 0.00552034517750144, + 0.005159362219274044, + 0.004950157832354307, + 0.0047441087663173676, + 0.004412285052239895, + 0.003709448967128992, + 0.0035803758073598146, + 0.0032407131511718035, + 0.002818912733346224 + ], + "sigma_after": [ + 0.008877134634531415, + 0.009005787519473998, + 0.009190588852029096, + 0.009603242724572646, + 0.009666053779645373, + 0.009315743848800078, + 0.009236965546074548, + 0.009184482684040289, + 0.009184153613757887, + 0.009189601257492514, + 0.009195537878450797, + 0.009196801675880382, + 0.009082811566456387, + 0.009031429845171466, + 0.008830142758732683, + 0.008416522409476584 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.10.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.310674574428132, + "effective_rank_before": 9.310674574428132, + "effective_rank_after": 15.9838181209933, + "nuclear_norm_before_full": 0.3610210078768432, + "nuclear_norm_before": 0.3610210078768432, + "nuclear_norm_after": 0.36102100787684316, + "fro_norm_before_full": 0.15630649548542333, + "fro_norm_after": 0.0903453884265663, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14192555844783783, + 0.041129421442747116, + 0.025051571428775787, + 0.0201248861849308, + 0.017883772030472755, + 0.014368073083460331, + 0.012666733004152775, + 0.012127568945288658, + 0.011426207609474659, + 0.011073618195950985, + 0.010134458541870117, + 0.00989347044378519, + 0.009200590662658215, + 0.008309527300298214, + 0.00798953790217638, + 0.0077160126529634 + ], + "sigma_after": [ + 0.0230730188575419, + 0.023086464305777266, + 0.024268904163771917, + 0.023492327118744032, + 0.023159887006189903, + 0.023088773231345464, + 0.023083758036314586, + 0.023038968190101157, + 0.022924650169515566, + 0.022836930952495966, + 0.022478122487997104, + 0.022351972236242065, + 0.021899003007769178, + 0.0210948134928319, + 0.020738817353394196, + 0.02040459726681096 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.10.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.02013842768928, + "effective_rank_before": 10.02013842768928, + "effective_rank_after": 15.98902638605612, + "nuclear_norm_before_full": 0.31333779683336616, + "nuclear_norm_before": 0.31333779683336616, + "nuclear_norm_after": 0.31333779683336616, + "fro_norm_before_full": 0.1257014651186264, + "fro_norm_after": 0.07838752068879942, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10792093724012375, + 0.045702915638685226, + 0.02134082466363907, + 0.017018888145685196, + 0.015539159066975117, + 0.013822603970766068, + 0.01330858375877142, + 0.011604229919612408, + 0.010709621012210846, + 0.009299512021243572, + 0.008952151983976364, + 0.008710877038538456, + 0.00845748744904995, + 0.007434588856995106, + 0.007237624377012253, + 0.006277791690081358 + ], + "sigma_after": [ + 0.0198382446479156, + 0.019090417231929495, + 0.020896034480317686, + 0.020372921210582012, + 0.020079907958961217, + 0.019866442251091933, + 0.019840686652886998, + 0.019852595409929057, + 0.019865946941715857, + 0.019737147210387967, + 0.01965117256032107, + 0.019573820240201666, + 0.019475106360695084, + 0.01885931386606531, + 0.01869471705255289, + 0.01764332275781232 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.10.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.7961210128429865, + "effective_rank_before": 7.7961210128429865, + "effective_rank_after": 15.799287309040606, + "nuclear_norm_before_full": 0.1400698283687234, + "nuclear_norm_before": 0.1400698283687234, + "nuclear_norm_after": 0.1400698283687234, + "fro_norm_before_full": 0.06694743733012772, + "fro_norm_after": 0.03543967502579555, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06129594147205353, + 0.01997440680861473, + 0.010575109161436558, + 0.007604360114783049, + 0.006087956950068474, + 0.005206708796322346, + 0.0044295769184827805, + 0.0038073365576565266, + 0.003423127578571439, + 0.003286065300926566, + 0.003078412963077426, + 0.0026075139176100492, + 0.002459708135575056, + 0.0022379043512046337, + 0.002060304395854473, + 0.0019353949464857578 + ], + "sigma_after": [ + 0.00994565114348156, + 0.009916586401577915, + 0.010453025049215843, + 0.009980938074807648, + 0.009957251158957743, + 0.009935550655733828, + 0.009734048207686278, + 0.009334382660527343, + 0.008943697142921134, + 0.008774024565897152, + 0.00848481205808508, + 0.0076796680543921945, + 0.0073834018404281745, + 0.006899923881068009, + 0.00647975188712369, + 0.0061671155868198 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.11.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.694552028667734, + "effective_rank_before": 7.694552028667734, + "effective_rank_after": 15.89056287784928, + "nuclear_norm_before_full": 0.38188759004697204, + "nuclear_norm_before": 0.38188759004697204, + "nuclear_norm_after": 0.38188759004697204, + "fro_norm_before_full": 0.186397130676676, + "fro_norm_after": 0.09611578867146238, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.17289987206459045, + 0.05179000645875931, + 0.024097830057144165, + 0.021560300141572952, + 0.01522831991314888, + 0.014009911566972733, + 0.010511505417525768, + 0.009777042083442211, + 0.00934982392936945, + 0.008768649771809578, + 0.008056900463998318, + 0.007917053066194057, + 0.00761146005243063, + 0.007116634864360094, + 0.006648329086601734, + 0.006543951109051704 + ], + "sigma_after": [ + 0.0267655806964483, + 0.026810225045490643, + 0.027281095001438417, + 0.026901102346542315, + 0.02679508782355434, + 0.02667580499859315, + 0.02504757080401281, + 0.02434262051294398, + 0.02386032024274813, + 0.02311426643474946, + 0.022053882597299734, + 0.021826111339797453, + 0.021305927198403077, + 0.020397996716227, + 0.01946400893129628, + 0.019245989357426958 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.11.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.233025445540152, + "effective_rank_before": 11.233025445540152, + "effective_rank_after": 15.998024536425746, + "nuclear_norm_before_full": 0.6308371592313051, + "nuclear_norm_before": 0.6308371592313051, + "nuclear_norm_after": 0.6308371592313051, + "fro_norm_before_full": 0.2360005837352264, + "fro_norm_after": 0.15772891075900278, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.20474301278591156, + 0.05971786752343178, + 0.04309077188372612, + 0.03455314785242081, + 0.03261904418468475, + 0.0289568230509758, + 0.027282170951366425, + 0.02693747729063034, + 0.024505706503987312, + 0.02374855987727642, + 0.023303505033254623, + 0.02261885069310665, + 0.022024229168891907, + 0.02114083245396614, + 0.018909567967057228, + 0.016685592010617256 + ], + "sigma_after": [ + 0.03914992472766843, + 0.03910192202729308, + 0.04095341216088253, + 0.040705663839871994, + 0.04031005989290114, + 0.03954702552532501, + 0.03929404316384805, + 0.039254197117313365, + 0.03910191914619951, + 0.03909677314908692, + 0.039100917153104923, + 0.039114853627727354, + 0.03913126495587421, + 0.039154937487818293, + 0.03911725044630869, + 0.038702994810081694 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.11.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.851629198931729, + "effective_rank_before": 10.851629198931729, + "effective_rank_after": 15.998389017312201, + "nuclear_norm_before_full": 0.575507789850235, + "nuclear_norm_before": 0.575507789850235, + "nuclear_norm_after": 0.575507789850235, + "fro_norm_before_full": 0.22092154001090852, + "fro_norm_after": 0.14389157421246904, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.19206203520298004, + 0.06551267206668854, + 0.03491455316543579, + 0.030197149142622948, + 0.027386318892240524, + 0.026625152677297592, + 0.02463477849960327, + 0.023921534419059753, + 0.022318968549370766, + 0.02086825668811798, + 0.01993793249130249, + 0.018914474174380302, + 0.01851728744804859, + 0.01783592998981476, + 0.016256870701909065, + 0.015603875741362572 + ], + "sigma_after": [ + 0.035807575126193564, + 0.03567982547033115, + 0.03758469007888138, + 0.03679195012270375, + 0.036215865336190076, + 0.036086342979465336, + 0.03584331972557047, + 0.03579431416880119, + 0.03575429775956172, + 0.0357790595837041, + 0.0358043540609502, + 0.03581694670603626, + 0.03581187362275305, + 0.03578274958739335, + 0.03556512455429927, + 0.0353895009674001 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.11.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.232354737127883, + "effective_rank_before": 10.232354737127883, + "effective_rank_after": 15.98512873504934, + "nuclear_norm_before_full": 0.1310899117961526, + "nuclear_norm_before": 0.1310899117961526, + "nuclear_norm_after": 0.1310899117961526, + "fro_norm_before_full": 0.05075146958871388, + "fro_norm_after": 0.03280277380824029, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.041898272931575775, + 0.019973408430814743, + 0.010058269836008549, + 0.008971982635557652, + 0.007530887145549059, + 0.006757390685379505, + 0.0055009969510138035, + 0.004698057658970356, + 0.0044208914041519165, + 0.003466047579422593, + 0.003362504532560706, + 0.0032673736568540335, + 0.003091728314757347, + 0.0027697808109223843, + 0.0026815799064934254, + 0.0026407393161207438 + ], + "sigma_after": [ + 0.008240444556990384, + 0.008042493443710346, + 0.008588549687406934, + 0.008750002822625308, + 0.008683274908272528, + 0.008525313755010505, + 0.008332504340468547, + 0.008331367334675377, + 0.008337279858856678, + 0.008195095535663902, + 0.008149054655451512, + 0.008098993443044563, + 0.007985043551292448, + 0.007693718447761614, + 0.007593253225461684, + 0.007543522229460272 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.11.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.75216317617274, + "effective_rank_before": 9.75216317617274, + "effective_rank_after": 15.988826939067433, + "nuclear_norm_before_full": 0.36316559091210365, + "nuclear_norm_before": 0.36316559091210365, + "nuclear_norm_after": 0.36316559091210365, + "fro_norm_before_full": 0.15230125735469563, + "fro_norm_after": 0.09085391551840791, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1376117616891861, + 0.03593817353248596, + 0.0280490443110466, + 0.020571745932102203, + 0.017842737957835197, + 0.014793241396546364, + 0.014096417464315891, + 0.013702440075576305, + 0.012645670212805271, + 0.012180960737168789, + 0.010909193195402622, + 0.010200604796409607, + 0.009851755574345589, + 0.00870494358241558, + 0.008388115093111992, + 0.007678785361349583 + ], + "sigma_after": [ + 0.023008488561999885, + 0.02302968000874937, + 0.02405502440831223, + 0.023610552848026923, + 0.02313646875890329, + 0.022999160528536497, + 0.023016068379726667, + 0.023025682765844234, + 0.02302654598435108, + 0.023003159147684818, + 0.022800446180572656, + 0.02256006292830495, + 0.022398120823789724, + 0.021619958430332216, + 0.02132962110728845, + 0.020546550049680676 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.11.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.126778107187068, + "effective_rank_before": 10.126778107187068, + "effective_rank_after": 15.982441809582124, + "nuclear_norm_before_full": 0.3105122931301594, + "nuclear_norm_before": 0.3105122931301594, + "nuclear_norm_after": 0.3105122931301594, + "fro_norm_before_full": 0.12332215182703549, + "fro_norm_after": 0.07771101692614386, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10519319027662277, + 0.0457170307636261, + 0.020202690735459328, + 0.01735529862344265, + 0.014756160788238049, + 0.01454673707485199, + 0.0135849853977561, + 0.011817678809165955, + 0.011209817603230476, + 0.009999845176935196, + 0.009575382806360722, + 0.00833260640501976, + 0.008074580691754818, + 0.007839781232178211, + 0.006709745619446039, + 0.005596761126071215 + ], + "sigma_after": [ + 0.01967936051783195, + 0.01894199599084151, + 0.020745054534833864, + 0.02036789474429487, + 0.01985898416321328, + 0.0198289127753861, + 0.019728777820085124, + 0.019701691253509648, + 0.019718019409887343, + 0.01971077273028632, + 0.019673026807698182, + 0.019353550159002517, + 0.019234141926657, + 0.01910582615517693, + 0.01818188774971568, + 0.01668239639173902 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.11.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.371557508519658, + "effective_rank_before": 7.371557508519658, + "effective_rank_after": 15.621917791238726, + "nuclear_norm_before_full": 0.14697897888254374, + "nuclear_norm_before": 0.14697897888254374, + "nuclear_norm_after": 0.14697897888254374, + "fro_norm_before_full": 0.07310194679696717, + "fro_norm_after": 0.037559663646731085, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06815477460622787, + 0.018106995150446892, + 0.0121668865904212, + 0.007375415880233049, + 0.006730671506375074, + 0.005633060820400715, + 0.0043377336114645, + 0.004077439662069082, + 0.0037974121514707804, + 0.003227938897907734, + 0.0030757745262235403, + 0.002580879954621196, + 0.002228478668257594, + 0.002034488832578063, + 0.0018521700985729694, + 0.001598857925273478 + ], + "sigma_after": [ + 0.010885783566877513, + 0.010883834297076405, + 0.011469498723794217, + 0.010883795655393212, + 0.01089581467319097, + 0.010867924990902141, + 0.010360766139672524, + 0.010144829286085334, + 0.009857512791087372, + 0.009079641309046354, + 0.008825239385028237, + 0.007857815194753807, + 0.007039606300138431, + 0.006545069597909624, + 0.0060531503385299855, + 0.005328696633057598 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.12.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.182725589352565, + "effective_rank_before": 9.182725589352565, + "effective_rank_after": 15.98354036358758, + "nuclear_norm_before_full": 0.3405103664845228, + "nuclear_norm_before": 0.3405103664845228, + "nuclear_norm_after": 0.3405103664845228, + "fro_norm_before_full": 0.14517489584197732, + "fro_norm_after": 0.08521415522269986, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12559407949447632, + 0.05700036138296127, + 0.024803401902318, + 0.015231352299451828, + 0.013942095451056957, + 0.0122842313721776, + 0.011224236339330673, + 0.010991593822836876, + 0.010372406803071499, + 0.010019966401159763, + 0.009485816583037376, + 0.009068479761481285, + 0.008150092326104641, + 0.008012102916836739, + 0.007325662299990654, + 0.007004487328231335 + ], + "sigma_after": [ + 0.021894142679985046, + 0.02112441063544232, + 0.023038679591731843, + 0.02187505575438256, + 0.021878428047442038, + 0.021905967767065708, + 0.021841612946912052, + 0.021809323353321287, + 0.021679073402340588, + 0.0215708640964855, + 0.021350505001696645, + 0.021124450514167493, + 0.02043257461307079, + 0.020303086769022632, + 0.019549981569137512, + 0.019132209742318804 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.12.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.17614479600117, + "effective_rank_before": 12.17614479600117, + "effective_rank_after": 15.996633831970144, + "nuclear_norm_before_full": 0.5803238302469254, + "nuclear_norm_before": 0.5803238302469254, + "nuclear_norm_after": 0.5803238302469255, + "fro_norm_before_full": 0.199210595350257, + "fro_norm_after": 0.14511162171945527, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.16222843527793884, + 0.06251964718103409, + 0.03727332130074501, + 0.03356233239173889, + 0.03281496465206146, + 0.027622798457741737, + 0.026425039395689964, + 0.025879042223095894, + 0.02425367757678032, + 0.023496264591813087, + 0.022690510377287865, + 0.022329827770590782, + 0.02119342051446438, + 0.020917652174830437, + 0.02016681618988514, + 0.016950080171227455 + ], + "sigma_after": [ + 0.0352153249140397, + 0.035371803684728455, + 0.037321969476931435, + 0.0376875642444087, + 0.03768094909734082, + 0.036894062207348935, + 0.03660946163780245, + 0.036481848521005146, + 0.03613950168637692, + 0.03601068934040662, + 0.03590176204821961, + 0.03586318805807577, + 0.03578377634318751, + 0.035773819853133694, + 0.03576307591668933, + 0.035825033217230386 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.12.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.832179460574919, + "effective_rank_before": 11.832179460574919, + "effective_rank_after": 15.995986544501042, + "nuclear_norm_before_full": 0.5712075047194958, + "nuclear_norm_before": 0.5712075047194958, + "nuclear_norm_after": 0.5712075047194958, + "fro_norm_before_full": 0.19919037661436223, + "fro_norm_after": 0.14283768509479255, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15960940718650818, + 0.07383870333433151, + 0.039543550461530685, + 0.03249582275748253, + 0.0299272071570158, + 0.02739042043685913, + 0.026382416486740112, + 0.025777366012334824, + 0.0239760410040617, + 0.022693373262882233, + 0.021583139896392822, + 0.02083979733288288, + 0.01859927922487259, + 0.01698336936533451, + 0.016434598714113235, + 0.01513301208615303 + ], + "sigma_after": [ + 0.03456909523605256, + 0.03400271313046795, + 0.03646490269432674, + 0.037233623516095, + 0.036946582165065994, + 0.03641409015391279, + 0.03617699663724937, + 0.036037749675535, + 0.03567284127346666, + 0.03548740548237539, + 0.035390209335302014, + 0.03535823589130502, + 0.035380861631485835, + 0.03541127889438878, + 0.03539795669876576, + 0.03526296230370089 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.12.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.258639439815786, + "effective_rank_before": 11.258639439815786, + "effective_rank_after": 15.992133480519161, + "nuclear_norm_before_full": 0.13829461415298283, + "nuclear_norm_before": 0.13829461415298283, + "nuclear_norm_after": 0.13829461415298283, + "fro_norm_before_full": 0.04927905319700287, + "fro_norm_after": 0.03459051829868741, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03890558332204819, + 0.016768068075180054, + 0.015147506259381771, + 0.009975634515285492, + 0.007899652235209942, + 0.007180714514106512, + 0.006135790608823299, + 0.005469586234539747, + 0.004930694121867418, + 0.004620688501745462, + 0.0044472417794167995, + 0.003936602268368006, + 0.0037867159117013216, + 0.0033690184354782104, + 0.0031140039209276438, + 0.0026071134489029646 + ], + "sigma_after": [ + 0.008433550831807479, + 0.008432434277470046, + 0.0086140168656571, + 0.008911685980970849, + 0.00912989945876617, + 0.00902625018084697, + 0.008793366068752672, + 0.008695368196210423, + 0.008676231360297442, + 0.008683047628319764, + 0.008688156278117849, + 0.008679600110218448, + 0.008661764382711202, + 0.008544329117910442, + 0.008405230419269815, + 0.007919682995656152 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.12.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.826361601015744, + "effective_rank_before": 10.826361601015744, + "effective_rank_after": 15.995200128326882, + "nuclear_norm_before_full": 0.39661922864615917, + "nuclear_norm_before": 0.39661922864615917, + "nuclear_norm_after": 0.39661922864615917, + "fro_norm_before_full": 0.14675337534769956, + "fro_norm_after": 0.09918444507160286, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11676764488220215, + 0.06275057047605515, + 0.03137411177158356, + 0.02662687934935093, + 0.019648920744657516, + 0.01886306144297123, + 0.017758965492248535, + 0.01578306034207344, + 0.012815508991479874, + 0.012433735653758049, + 0.011998176574707031, + 0.011402430012822151, + 0.01067979447543621, + 0.010115482844412327, + 0.009272819384932518, + 0.00832806620746851 + ], + "sigma_after": [ + 0.024220150395452306, + 0.024440982153226036, + 0.02522013734213395, + 0.02604132005954764, + 0.02546851024500701, + 0.025295846371600293, + 0.025082685028446974, + 0.024861899615722233, + 0.024882545135630752, + 0.024882277651149302, + 0.024868638517296607, + 0.024816400490440777, + 0.02467896390541174, + 0.024496177021637887, + 0.024064160596811275, + 0.023298534116644388 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.12.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.928735568683168, + "effective_rank_before": 11.928735568683168, + "effective_rank_after": 15.996551276519067, + "nuclear_norm_before_full": 0.3165628840215504, + "nuclear_norm_before": 0.3165628840215504, + "nuclear_norm_after": 0.3165628840215504, + "fro_norm_before_full": 0.10781892419831307, + "fro_norm_after": 0.07915778783549825, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08059936761856079, + 0.048356473445892334, + 0.02434403821825981, + 0.020399179309606552, + 0.016257140785455704, + 0.015403671190142632, + 0.014760168269276619, + 0.013485378585755825, + 0.012391316704452038, + 0.011929359287023544, + 0.011680961586534977, + 0.010628046467900276, + 0.010293115861713886, + 0.00991440936923027, + 0.008708097971975803, + 0.007412159349769354 + ], + "sigma_after": [ + 0.01888399568593821, + 0.01950979453935741, + 0.019749647069314723, + 0.020446997701677877, + 0.020527915377599622, + 0.020357016078371575, + 0.020206269592479016, + 0.019909306372426973, + 0.019720720244016453, + 0.01967161532815247, + 0.01965342270761237, + 0.019633571713595005, + 0.01964141881797985, + 0.01965317133731086, + 0.019646939282321224, + 0.01935108217339677 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.12.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.129807713274194, + "effective_rank_before": 7.129807713274194, + "effective_rank_after": 15.573967620515662, + "nuclear_norm_before_full": 0.19528537627775222, + "nuclear_norm_before": 0.19528537627775222, + "nuclear_norm_after": 0.19528537627775222, + "fro_norm_before_full": 0.09968762271583637, + "fro_norm_after": 0.05000997772524135, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09381955116987228, + 0.023270025849342346, + 0.0138289425522089, + 0.01091510895639658, + 0.008879332803189754, + 0.007245038636028767, + 0.0055093225091695786, + 0.0053870598785579205, + 0.004649627488106489, + 0.003948630299419165, + 0.003702952293679118, + 0.003483816748484969, + 0.0033112498931586742, + 0.0028922625351697206, + 0.002803286537528038, + 0.0016391681274399161 + ], + "sigma_after": [ + 0.014713262601547386, + 0.014737671533382887, + 0.015168138448539833, + 0.014725063641657514, + 0.014724942087587818, + 0.014604614446038444, + 0.013656603371950207, + 0.013537673946123142, + 0.012639953376316342, + 0.011479024397612768, + 0.010998401104340095, + 0.01053737165640948, + 0.010153072564768816, + 0.009143947283590003, + 0.008916171868093278, + 0.005549463949794208 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.13.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.381190129188369, + "effective_rank_before": 9.381190129188369, + "effective_rank_after": 15.97584000196016, + "nuclear_norm_before_full": 0.3444785666652024, + "nuclear_norm_before": 0.3444785666652024, + "nuclear_norm_after": 0.3444785666652024, + "fro_norm_before_full": 0.1440476166445648, + "fro_norm_after": 0.08624715240814639, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12380845099687576, + 0.055982597172260284, + 0.024603094905614853, + 0.018535761162638664, + 0.017009856179356575, + 0.014095351099967957, + 0.012769140303134918, + 0.010834680870175362, + 0.010085180401802063, + 0.009678545407950878, + 0.008948449045419693, + 0.008657303638756275, + 0.008487951941788197, + 0.007386530749499798, + 0.007218524347990751, + 0.006377148441970348 + ], + "sigma_after": [ + 0.022220545814125303, + 0.021423232344663812, + 0.023398181238984694, + 0.02262150750938497, + 0.022366227616743273, + 0.02221429242067995, + 0.02224552694540012, + 0.02213752927958398, + 0.021955543594413457, + 0.02180512430417074, + 0.021422838555358578, + 0.021224670615199222, + 0.02109625456285457, + 0.02000066801735375, + 0.019790824283424414, + 0.01855559956286156 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.13.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.6740006258754, + "effective_rank_before": 12.6740006258754, + "effective_rank_after": 15.995443185805948, + "nuclear_norm_before_full": 0.5856379922479391, + "nuclear_norm_before": 0.5856379922479391, + "nuclear_norm_after": 0.5856379922479391, + "fro_norm_before_full": 0.19021824444384614, + "fro_norm_after": 0.14645107352642756, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14043164253234863, + 0.08127399533987045, + 0.038163915276527405, + 0.03335629403591156, + 0.03177543357014656, + 0.029010361060500145, + 0.027284884825348854, + 0.025850707665085793, + 0.025739410892128944, + 0.024546168744564056, + 0.024307038635015488, + 0.022881226614117622, + 0.021323347464203835, + 0.02075311169028282, + 0.019901122897863388, + 0.019039331004023552 + ], + "sigma_after": [ + 0.034659180466541595, + 0.03540654843858863, + 0.03704097206173016, + 0.037862862324794626, + 0.037928612060911776, + 0.03769119036369763, + 0.03734701402626099, + 0.03699777915948899, + 0.03696979801896861, + 0.036674479045841576, + 0.036617886823100645, + 0.03631705014159825, + 0.03609171125143174, + 0.0360417171324839, + 0.03599946859402329, + 0.03599172233847664 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.13.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.153839590880313, + "effective_rank_before": 11.153839590880313, + "effective_rank_after": 15.997806945268554, + "nuclear_norm_before_full": 0.576515443623066, + "nuclear_norm_before": 0.576515443623066, + "nuclear_norm_after": 0.5765154436230657, + "fro_norm_before_full": 0.21545844752815752, + "fro_norm_after": 0.1441487969803959, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.18447262048721313, + 0.0661817416548729, + 0.03509339317679405, + 0.03165486082434654, + 0.029404569417238235, + 0.026022091507911682, + 0.024835048243403435, + 0.024325096979737282, + 0.022931182757019997, + 0.021564222872257233, + 0.021198291331529617, + 0.019892826676368713, + 0.018609512597322464, + 0.017740311101078987, + 0.016685105860233307, + 0.015904568135738373 + ], + "sigma_after": [ + 0.03573339372119769, + 0.035484084120512994, + 0.037637096980452825, + 0.03722615893921003, + 0.03676221084858519, + 0.036076100164638185, + 0.03590699238842344, + 0.03585158987297512, + 0.035756204461403374, + 0.03573607612409264, + 0.03574037892668842, + 0.03577314039819313, + 0.03579790705279879, + 0.035782409055564145, + 0.03569430908341377, + 0.035557391484916046 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.13.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.697745905722696, + "effective_rank_before": 11.697745905722696, + "effective_rank_after": 15.994322592418941, + "nuclear_norm_before_full": 0.12767319474369287, + "nuclear_norm_before": 0.12767319474369287, + "nuclear_norm_after": 0.12767319474369285, + "fro_norm_before_full": 0.04408489718240794, + "fro_norm_after": 0.03192963795979451, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.0334535650908947, + 0.01849287748336792, + 0.012019800953567028, + 0.007084507495164871, + 0.006992193404585123, + 0.006641035433858633, + 0.006157624069601297, + 0.00539973983541131, + 0.0051041338592767715, + 0.004722764249891043, + 0.004286689683794975, + 0.003893706714734435, + 0.0037327196914702654, + 0.0036354477051645517, + 0.003370373509824276, + 0.0026860155630856752 + ], + "sigma_after": [ + 0.007628967776114878, + 0.0077642283895451615, + 0.007940827659123698, + 0.008343242526512383, + 0.008335482896319578, + 0.00828831691023179, + 0.008188371674582204, + 0.008016752422442042, + 0.007968302980967438, + 0.007933292867649372, + 0.00792937851899994, + 0.007940210118306468, + 0.007940712821781808, + 0.00793793548806977, + 0.007911407819732194, + 0.007605763873314119 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.13.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.112273313915999, + "effective_rank_before": 11.112273313915999, + "effective_rank_after": 15.996139748406943, + "nuclear_norm_before_full": 0.35160183999687433, + "nuclear_norm_before": 0.35160183999687433, + "nuclear_norm_after": 0.3516018399968743, + "fro_norm_before_full": 0.1296902719424787, + "fro_norm_after": 0.08792180651362233, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10872635990381241, + 0.03809370473027229, + 0.03186824545264244, + 0.02118682488799095, + 0.02064887061715126, + 0.01841549761593342, + 0.015050924383103848, + 0.01374566089361906, + 0.013387532904744148, + 0.012397781014442444, + 0.011331409215927124, + 0.010350927710533142, + 0.009914196096360683, + 0.009480071254074574, + 0.008930083364248276, + 0.008073749952018261 + ], + "sigma_after": [ + 0.021752608731689202, + 0.021832313975042306, + 0.021857787130427863, + 0.023022334674950726, + 0.022988795014588753, + 0.022640975208076775, + 0.021974124949476648, + 0.021866987135590148, + 0.021858052926335615, + 0.021867410565460977, + 0.021893825005608738, + 0.02186460603073926, + 0.02181262287464071, + 0.021725404396120147, + 0.021550287253749915, + 0.021093704124376514 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.13.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.836250775971239, + "effective_rank_before": 12.836250775971239, + "effective_rank_after": 15.997011062037561, + "nuclear_norm_before_full": 0.29388159420341253, + "nuclear_norm_before": 0.29388159420341253, + "nuclear_norm_after": 0.29388159420341253, + "fro_norm_before_full": 0.09329402148413206, + "fro_norm_after": 0.07348414055531509, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06393957883119583, + 0.044857922941446304, + 0.01924290508031845, + 0.018460514023900032, + 0.017504405230283737, + 0.016309943050146103, + 0.014164579100906849, + 0.013552901335060596, + 0.012999167665839195, + 0.011908994056284428, + 0.011356591247022152, + 0.011016354896128178, + 0.010320971719920635, + 0.010183143429458141, + 0.009723106399178505, + 0.008340515196323395 + ], + "sigma_after": [ + 0.01776285651306571, + 0.018054096366680174, + 0.01844838179103157, + 0.018616388285789653, + 0.018811619736722237, + 0.018976669233611264, + 0.01887562851170043, + 0.018753211552588495, + 0.018620584237434092, + 0.01834561677705568, + 0.018223851747231023, + 0.018161364775158676, + 0.018071550446155753, + 0.018060251081449083, + 0.018037735081422934, + 0.018061788066315792 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.13.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.369693448736708, + "effective_rank_before": 8.369693448736708, + "effective_rank_after": 15.630748162615514, + "nuclear_norm_before_full": 0.14842285274062306, + "nuclear_norm_before": 0.14842285274062306, + "nuclear_norm_after": 0.14842285274062306, + "fro_norm_before_full": 0.06711463380071946, + "fro_norm_after": 0.03785454792192539, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06062600761651993, + 0.0183558352291584, + 0.013790604658424854, + 0.008710806258022785, + 0.007314989808946848, + 0.006653802935034037, + 0.006206661928445101, + 0.004490201827138662, + 0.0040641892701387405, + 0.003951047081500292, + 0.003293790156021714, + 0.0031690613832324743, + 0.002571394434198737, + 0.0023159710690379143, + 0.0017484130803495646, + 0.0011600760044530034 + ], + "sigma_after": [ + 0.010516392260828164, + 0.010527258612739164, + 0.010759113430822274, + 0.010718585960533229, + 0.010523394351238682, + 0.010509818933644263, + 0.010519858664225721, + 0.010309152914087504, + 0.010065124386550045, + 0.00997913169514488, + 0.009271510962536658, + 0.00909332604651937, + 0.00802786140657927, + 0.007463582139161245, + 0.005983259115113154, + 0.0041554818608994315 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.14.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.473564732631836, + "effective_rank_before": 9.473564732631836, + "effective_rank_after": 15.979746438017328, + "nuclear_norm_before_full": 0.33955494640395045, + "nuclear_norm_before": 0.33955494640395045, + "nuclear_norm_after": 0.3395549464039504, + "fro_norm_before_full": 0.1401308748640356, + "fro_norm_after": 0.08499522795467726, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11838900297880173, + 0.0576944425702095, + 0.024870077148079872, + 0.02123953402042389, + 0.01411760225892067, + 0.01381507609039545, + 0.012839955277740955, + 0.010479049757122993, + 0.010049008764326572, + 0.00978341605514288, + 0.009247906506061554, + 0.008168204687535763, + 0.007573361974209547, + 0.007334114983677864, + 0.007236809935420752, + 0.006717383395880461 + ], + "sigma_after": [ + 0.02174005830199255, + 0.02125570857215108, + 0.022908934494837926, + 0.022810167133650028, + 0.02179852896072646, + 0.02180119673754709, + 0.02182418851069955, + 0.021716552306984892, + 0.02161944103942079, + 0.021539848604137653, + 0.021326225301754466, + 0.020633336992565462, + 0.020072894513036177, + 0.01980720930328679, + 0.019692233555102506, + 0.019008422076056998 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.14.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.319273060496208, + "effective_rank_before": 12.319273060496208, + "effective_rank_after": 15.996015696861344, + "nuclear_norm_before_full": 0.6171642858535051, + "nuclear_norm_before": 0.6171642858535051, + "nuclear_norm_after": 0.6171642858535051, + "fro_norm_before_full": 0.20496303264907081, + "fro_norm_after": 0.15432947884409862, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15483835339546204, + 0.08030270040035248, + 0.048469532281160355, + 0.04464491456747055, + 0.034630462527275085, + 0.030075618997216225, + 0.02812330238521099, + 0.02657807618379593, + 0.02516711875796318, + 0.0236800704151392, + 0.02331218309700489, + 0.021364212036132812, + 0.02042071893811226, + 0.01971406117081642, + 0.01854974590241909, + 0.017293214797973633 + ], + "sigma_after": [ + 0.03685592102860458, + 0.03699394322727765, + 0.03836631832580959, + 0.03875944564507567, + 0.04036993567363176, + 0.039900554513413115, + 0.03944828558472421, + 0.03906825583807756, + 0.0387550336285042, + 0.03850148696328634, + 0.038454380680476914, + 0.03831699216134554, + 0.03831170748254734, + 0.03832588017744368, + 0.03836204893396467, + 0.03837409598932229 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.14.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.442332960626498, + "effective_rank_before": 11.442332960626498, + "effective_rank_after": 15.997018232241357, + "nuclear_norm_before_full": 0.5423952303826809, + "nuclear_norm_before": 0.5423952303826809, + "nuclear_norm_after": 0.5423952303826809, + "fro_norm_before_full": 0.19686275907297207, + "fro_norm_after": 0.13562425985519372, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1652052104473114, + 0.06111881136894226, + 0.03615511581301689, + 0.03348725289106369, + 0.029076602309942245, + 0.025915728881955147, + 0.024276690557599068, + 0.02329661138355732, + 0.022212451323866844, + 0.02101866714656353, + 0.02008703537285328, + 0.018137792125344276, + 0.017473779618740082, + 0.01611270196735859, + 0.015119783580303192, + 0.013700995594263077 + ], + "sigma_after": [ + 0.033404023935593385, + 0.03325497596672113, + 0.03511836414594518, + 0.035353046058603135, + 0.03498205006330673, + 0.03430334823449747, + 0.033962730511164586, + 0.03379727588193596, + 0.033661540331051065, + 0.03357610992167207, + 0.03355394982768579, + 0.03359032377032302, + 0.03360707178094452, + 0.0335933614272814, + 0.0334973602533349, + 0.03313969827262054 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.14.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.404506363151455, + "effective_rank_before": 13.404506363151455, + "effective_rank_after": 15.995983389349673, + "nuclear_norm_before_full": 0.14889200986362994, + "nuclear_norm_before": 0.14889200986362994, + "nuclear_norm_after": 0.14889200986362994, + "fro_norm_before_full": 0.044185674935147475, + "fro_norm_after": 0.03723236681827519, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.02446368895471096, + 0.020241517573595047, + 0.0152842802926898, + 0.01361450832337141, + 0.01158995646983385, + 0.008491926826536655, + 0.007595418021082878, + 0.006880695465952158, + 0.006472250912338495, + 0.006093364208936691, + 0.005616677459329367, + 0.005286380648612976, + 0.004885775037109852, + 0.004442973993718624, + 0.0042478847317397594, + 0.003684710944071412 + ], + "sigma_after": [ + 0.009197801540413431, + 0.009162129180123902, + 0.00890308501794335, + 0.00911997884458379, + 0.009198118754519152, + 0.009543113874080833, + 0.009678747844281006, + 0.009646474332572095, + 0.009568309142695073, + 0.009470858966213363, + 0.009341944885666172, + 0.009266996166328959, + 0.009208085697231562, + 0.009189475039665083, + 0.009193299931725761, + 0.009203590645586405 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.14.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.034083846592315, + "effective_rank_before": 11.034083846592315, + "effective_rank_after": 15.993212661494384, + "nuclear_norm_before_full": 0.35546793742105365, + "nuclear_norm_before": 0.35546793742105365, + "nuclear_norm_after": 0.35546793742105365, + "fro_norm_before_full": 0.12975699663864232, + "fro_norm_after": 0.08890441861687823, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10493095964193344, + 0.04611722007393837, + 0.03456280380487442, + 0.023747608065605164, + 0.01885291002690792, + 0.017317021265625954, + 0.016855308786034584, + 0.013635396026074886, + 0.012991750612854958, + 0.011928075924515724, + 0.011027121916413307, + 0.009874418377876282, + 0.00921652652323246, + 0.008857437409460545, + 0.008545522578060627, + 0.007007856387645006 + ], + "sigma_after": [ + 0.021874035537994566, + 0.021495615070296067, + 0.022310175847825697, + 0.023331163060670228, + 0.023169135121857514, + 0.022831589056892483, + 0.022728028704238307, + 0.022287485491553766, + 0.022281022458158703, + 0.022305702508966473, + 0.02231895246497088, + 0.02222859347114202, + 0.022070144192523244, + 0.02193662695693224, + 0.02178855188439101, + 0.020511115592640464 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.14.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.56346302310422, + "effective_rank_before": 11.56346302310422, + "effective_rank_after": 15.99486185360218, + "nuclear_norm_before_full": 0.28302910178899765, + "nuclear_norm_before": 0.28302910178899765, + "nuclear_norm_after": 0.2830291017889976, + "fro_norm_before_full": 0.09925735747992527, + "fro_norm_after": 0.0707800178952108, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07626365125179291, + 0.04416695609688759, + 0.01886541210114956, + 0.017358962446451187, + 0.015084114857017994, + 0.014060739427804947, + 0.013738522306084633, + 0.01218361221253872, + 0.011590561829507351, + 0.011273186653852463, + 0.010146263986825943, + 0.009891667403280735, + 0.007773548364639282, + 0.007633876986801624, + 0.006725668907165527, + 0.006272356957197189 + ], + "sigma_after": [ + 0.0169136143739423, + 0.017408973174033598, + 0.01825587647887982, + 0.018458097940167215, + 0.01836252543648237, + 0.01816441649697056, + 0.01809109561584016, + 0.017745126117669267, + 0.017646446695916307, + 0.017606418538177276, + 0.017542369676918575, + 0.017542871317280256, + 0.017533280272291417, + 0.01751429338444306, + 0.017249011923592603, + 0.01699468434639282 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.14.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.839970763621855, + "effective_rank_before": 9.839970763621855, + "effective_rank_after": 15.946555848904348, + "nuclear_norm_before_full": 0.15972557687200606, + "nuclear_norm_before": 0.15972557687200606, + "nuclear_norm_after": 0.15972557687200606, + "fro_norm_before_full": 0.06370530349684565, + "fro_norm_after": 0.04005682406062851, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.053268324583768845, + 0.02496841736137867, + 0.013857394456863403, + 0.009304627776145935, + 0.00808486808091402, + 0.007003755308687687, + 0.0066174669191241264, + 0.005920377094298601, + 0.005272549577057362, + 0.005217254161834717, + 0.004294876009225845, + 0.0038424090016633272, + 0.00341968797147274, + 0.0033150045201182365, + 0.0031183313112705946, + 0.0022202327381819487 + ], + "sigma_after": [ + 0.010317822089156922, + 0.010019359342735377, + 0.010499519917690851, + 0.010798079893992257, + 0.010544843176386372, + 0.010393374028595134, + 0.010376674106646236, + 0.010385121322083463, + 0.010389955818624803, + 0.010387650485567557, + 0.010192529965922986, + 0.009925459317267227, + 0.009527815484605757, + 0.009403959024134064, + 0.009141860206114736, + 0.007421552692482318 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.15.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.324485831564003, + "effective_rank_before": 9.324485831564003, + "effective_rank_after": 15.972633545803802, + "nuclear_norm_before_full": 0.3274357430636883, + "nuclear_norm_before": 0.3274357430636883, + "nuclear_norm_after": 0.3274357430636883, + "fro_norm_before_full": 0.13950466525726807, + "fro_norm_after": 0.08199587453002458, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12373434752225876, + 0.04576920345425606, + 0.021413231268525124, + 0.018410397693514824, + 0.01616104319691658, + 0.013084125705063343, + 0.012084738351404667, + 0.011283489875495434, + 0.010715430602431297, + 0.00972151942551136, + 0.008894109167158604, + 0.008562972769141197, + 0.007541842758655548, + 0.007207526825368404, + 0.0067192502319812775, + 0.006132514216005802 + ], + "sigma_after": [ + 0.02114480944133751, + 0.020682031762295383, + 0.022126056173748906, + 0.021592093919297877, + 0.021221816369375264, + 0.021128378774417363, + 0.021148263892599026, + 0.021129375736158886, + 0.02107609140811814, + 0.020855006303450614, + 0.02049664688539608, + 0.02029735012463531, + 0.019442573163293415, + 0.019075087047331664, + 0.018453467271457476, + 0.017566694790775328 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.15.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.148565915711156, + "effective_rank_before": 12.148565915711156, + "effective_rank_after": 15.996046951440727, + "nuclear_norm_before_full": 0.615880848839879, + "nuclear_norm_before": 0.615880848839879, + "nuclear_norm_after": 0.615880848839879, + "fro_norm_before_full": 0.20798303254569156, + "fro_norm_after": 0.1540081871011351, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15897658467292786, + 0.08619967848062515, + 0.04348868876695633, + 0.03798866271972656, + 0.03078947775065899, + 0.030052246525883675, + 0.028320953249931335, + 0.027923963963985443, + 0.02740594372153282, + 0.023836595937609673, + 0.022502267733216286, + 0.022122640162706375, + 0.021320100873708725, + 0.020198987796902657, + 0.018816519528627396, + 0.01593753695487976 + ], + "sigma_after": [ + 0.036685153869136164, + 0.03719346962621399, + 0.03884372383089798, + 0.03989077881252948, + 0.039719029459821815, + 0.03956712645593414, + 0.03916303943900314, + 0.03906646027625205, + 0.038941235369623156, + 0.038249450030234514, + 0.03812543338756275, + 0.03810575815015135, + 0.038085099151746564, + 0.038094921024635776, + 0.038135975990623454, + 0.03801419396551269 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.15.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.062397060138345, + "effective_rank_before": 12.062397060138345, + "effective_rank_after": 15.995983474418392, + "nuclear_norm_before_full": 0.5821797978132963, + "nuclear_norm_before": 0.5821797978132963, + "nuclear_norm_after": 0.5821797978132963, + "fro_norm_before_full": 0.19682629279038574, + "fro_norm_after": 0.14558153184609376, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14689865708351135, + 0.08863403648138046, + 0.04117702320218086, + 0.03551655635237694, + 0.03118806891143322, + 0.02968127839267254, + 0.026883767917752266, + 0.02429838292300701, + 0.02369098737835884, + 0.022882699966430664, + 0.021080931648612022, + 0.020259728655219078, + 0.01982993073761463, + 0.018861284479498863, + 0.016900165006518364, + 0.014396298676729202 + ], + "sigma_after": [ + 0.03462397149029188, + 0.035793381531042015, + 0.03670529182784084, + 0.037757700476181204, + 0.037841972580659945, + 0.037636112920836816, + 0.03703096758457952, + 0.03643539101730998, + 0.03632010519791292, + 0.036190899552108365, + 0.03601862977415124, + 0.03599328486917784, + 0.03599139468216279, + 0.03600821531407396, + 0.036054470499011705, + 0.03577800849595538 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.15.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.042273934559107, + "effective_rank_before": 12.042273934559107, + "effective_rank_after": 15.995379430961307, + "nuclear_norm_before_full": 0.1512815379537642, + "nuclear_norm_before": 0.1512815379537642, + "nuclear_norm_after": 0.1512815379537642, + "fro_norm_before_full": 0.049837419898712786, + "fro_norm_after": 0.03783132970157375, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.034707359969615936, + 0.02031845971941948, + 0.016503121703863144, + 0.013531829230487347, + 0.009812312200665474, + 0.008309951983392239, + 0.006891115568578243, + 0.006159396842122078, + 0.005780266132205725, + 0.005278131924569607, + 0.0051023387350142, + 0.00437235739082098, + 0.004314975347369909, + 0.003845229744911194, + 0.0033829640597105026, + 0.0029717274010181427 + ], + "sigma_after": [ + 0.009256297619897866, + 0.009194903480592051, + 0.009239384320917617, + 0.009466589605585914, + 0.009766150729010601, + 0.009959497848964612, + 0.009744796515945842, + 0.009568825516009865, + 0.009501185982539991, + 0.009455199561790854, + 0.009450985931897886, + 0.009466735896476342, + 0.009467298660495465, + 0.009437761896816286, + 0.009295802615554877, + 0.009010121771268136 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.15.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.71257367084231, + "effective_rank_before": 10.71257367084231, + "effective_rank_after": 15.994790699931603, + "nuclear_norm_before_full": 0.3529755654744804, + "nuclear_norm_before": 0.3529755654744804, + "nuclear_norm_after": 0.3529755654744804, + "fro_norm_before_full": 0.1333243951283065, + "fro_norm_after": 0.08827265511229028, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11172506213188171, + 0.04381671920418739, + 0.029450712725520134, + 0.02532065659761429, + 0.02166738174855709, + 0.01527268998324871, + 0.014446130022406578, + 0.012554723769426346, + 0.012217344716191292, + 0.012144909240305424, + 0.010743900202214718, + 0.009916086681187153, + 0.00903328973799944, + 0.008650654926896095, + 0.008379089646041393, + 0.007636214140802622 + ], + "sigma_after": [ + 0.022023211209017707, + 0.0216726451070717, + 0.022337061913899517, + 0.02303489538825649, + 0.02330660067099271, + 0.022226374433308736, + 0.022156337534490336, + 0.022147457206605898, + 0.022157134464373698, + 0.02215911448994249, + 0.022147271920061867, + 0.02203696601301693, + 0.02176099233089601, + 0.02157355768409441, + 0.021411074969276443, + 0.020824870139175444 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.15.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.250956451638649, + "effective_rank_before": 11.250956451638649, + "effective_rank_after": 15.992862860608252, + "nuclear_norm_before_full": 0.2884580818936229, + "nuclear_norm_before": 0.2884580818936229, + "nuclear_norm_after": 0.2884580818936229, + "fro_norm_before_full": 0.10261855817016975, + "fro_norm_after": 0.07214639403976611, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07530974596738815, + 0.05281969532370567, + 0.020114796236157417, + 0.018120726570487022, + 0.01494544930756092, + 0.01398936565965414, + 0.012964224442839622, + 0.011888423934578896, + 0.011141273193061352, + 0.010538180358707905, + 0.009386534802615643, + 0.008811376057565212, + 0.008540372364223003, + 0.007516835816204548, + 0.006801541429013014, + 0.0055695404298603535 + ], + "sigma_after": [ + 0.017332147598347676, + 0.018008322519095746, + 0.01859834562693834, + 0.018905515015545247, + 0.018705226520222728, + 0.018494063409505615, + 0.01826241032699751, + 0.018075052825926006, + 0.018002801382423685, + 0.017981625049640126, + 0.01800259962825597, + 0.018012471177794157, + 0.018008129133134976, + 0.017877183921618068, + 0.01760671523681284, + 0.016585472521364157 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.15.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.461736013506245, + "effective_rank_before": 9.461736013506245, + "effective_rank_after": 15.96632271071156, + "nuclear_norm_before_full": 0.17620169604197145, + "nuclear_norm_before": 0.17620169604197145, + "nuclear_norm_after": 0.17620169604197145, + "fro_norm_before_full": 0.07355221635435157, + "fro_norm_after": 0.04413933102552694, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06446202099323273, + 0.02140907011926174, + 0.019146781414747238, + 0.008608229458332062, + 0.008442336693406105, + 0.007562787272036076, + 0.006123557221144438, + 0.005798880942165852, + 0.005525222513824701, + 0.005429853685200214, + 0.004880418069660664, + 0.004272495862096548, + 0.004208266269415617, + 0.003986355382949114, + 0.0034142276272177696, + 0.0029311925172805786 + ], + "sigma_after": [ + 0.011435865900484968, + 0.011412464785551742, + 0.011424525629097779, + 0.011486303267764945, + 0.011466922193102535, + 0.01141651477809429, + 0.011433312738571646, + 0.011414097965933219, + 0.011377555396997476, + 0.011359243636336782, + 0.011180860892765355, + 0.010793443872286826, + 0.010738258333951948, + 0.01052411485394172, + 0.009789871076989802, + 0.008948340720100414 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.16.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.43999496300734, + "effective_rank_before": 7.43999496300734, + "effective_rank_after": 15.873111580392122, + "nuclear_norm_before_full": 0.38307190081104636, + "nuclear_norm_before": 0.38307190081104636, + "nuclear_norm_after": 0.3830719008110463, + "fro_norm_before_full": 0.1956657892319074, + "fro_norm_after": 0.09651138757653908, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.18642596900463104, + 0.03764352947473526, + 0.02330966852605343, + 0.018210457637906075, + 0.015404523350298405, + 0.013359357602894306, + 0.012651205994188786, + 0.011506002396345139, + 0.009493728168308735, + 0.009285259060561657, + 0.009112686850130558, + 0.008443264290690422, + 0.007485012989491224, + 0.007189067546278238, + 0.006906713824719191, + 0.006645454093813896 + ], + "sigma_after": [ + 0.026820226808553116, + 0.027787339249537628, + 0.026976183861827777, + 0.02678781404593214, + 0.026756638858132155, + 0.026350753808017036, + 0.026068327612608713, + 0.025404473334239574, + 0.02347940347046372, + 0.023217400015618374, + 0.022991060177529332, + 0.02203060851831002, + 0.02042313121168985, + 0.01987085912365949, + 0.019319491389935307, + 0.018788189324992103 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.16.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.716109627002755, + "effective_rank_before": 10.716109627002755, + "effective_rank_after": 15.997059683788326, + "nuclear_norm_before_full": 0.6911191921681166, + "nuclear_norm_before": 0.6911191921681166, + "nuclear_norm_after": 0.6911191921681165, + "fro_norm_before_full": 0.26983317045650324, + "fro_norm_after": 0.17281184957298815, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.23898527026176453, + 0.06015449017286301, + 0.046899065375328064, + 0.04542620852589607, + 0.03749241679906845, + 0.030932486057281494, + 0.028895407915115356, + 0.027045445516705513, + 0.025592992082238197, + 0.02411985583603382, + 0.023650379851460457, + 0.022425271570682526, + 0.02192496694624424, + 0.019856665283441544, + 0.019111189991235733, + 0.01860707998275757 + ], + "sigma_after": [ + 0.04291527169190523, + 0.04318208404027006, + 0.045101147744452244, + 0.04515126518049262, + 0.04421627477781798, + 0.04303239846074101, + 0.042877760489932644, + 0.04284618853278053, + 0.04287227593496229, + 0.04291047466995422, + 0.04291810180923771, + 0.042909356520209295, + 0.0428879086743398, + 0.042618812712770265, + 0.042424387645244664, + 0.04225548328300589 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.16.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.231363700707039, + "effective_rank_before": 10.231363700707039, + "effective_rank_after": 15.997222656802313, + "nuclear_norm_before_full": 0.6265872046351433, + "nuclear_norm_before": 0.6265872046351433, + "nuclear_norm_after": 0.6265872046351432, + "fro_norm_before_full": 0.2531210639122472, + "fro_norm_after": 0.1566740917369759, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.22524099051952362, + 0.06937194615602493, + 0.03865021839737892, + 0.035245381295681, + 0.03160841017961502, + 0.026544859632849693, + 0.024817338213324547, + 0.02408100850880146, + 0.02177891694009304, + 0.02163250744342804, + 0.020623767748475075, + 0.018917066976428032, + 0.01875058375298977, + 0.01690628193318844, + 0.016633227467536926, + 0.015784699469804764 + ], + "sigma_after": [ + 0.03917542546711128, + 0.039186660014292427, + 0.04097244578734172, + 0.040384956606804176, + 0.039663633423362835, + 0.03912907355802389, + 0.03912523750453837, + 0.039140907215937215, + 0.03918695498473277, + 0.039186580752374946, + 0.03916003812724091, + 0.03896914846225475, + 0.03893705571032487, + 0.038362052150588244, + 0.03823693255189765, + 0.037770102318317136 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.16.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.353995321461042, + "effective_rank_before": 12.353995321461042, + "effective_rank_after": 15.993954251833364, + "nuclear_norm_before_full": 0.15228080190718174, + "nuclear_norm_before": 0.15228080190718174, + "nuclear_norm_after": 0.15228080190718174, + "fro_norm_before_full": 0.049394590714036055, + "fro_norm_after": 0.03808461895893181, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03446356579661369, + 0.01954672671854496, + 0.017610765993595123, + 0.011199742555618286, + 0.008461732417345047, + 0.007805566769093275, + 0.007443466689437628, + 0.007287493906915188, + 0.006474971305578947, + 0.005832822062075138, + 0.005391691345721483, + 0.005089692305773497, + 0.00470303138718009, + 0.00396546209231019, + 0.0037317839451134205, + 0.0032722866162657738 + ], + "sigma_after": [ + 0.009248403107377394, + 0.00913900728069198, + 0.009112683851645773, + 0.009500944187169097, + 0.009953615786200494, + 0.009932720443347802, + 0.009879969886373895, + 0.009849682700676727, + 0.009653353485741055, + 0.009515619074468254, + 0.009462623100556797, + 0.009449679898285091, + 0.009454437199691846, + 0.009454760491433102, + 0.009421754331299312, + 0.00925154708222311 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.16.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.543575842809773, + "effective_rank_before": 10.543575842809773, + "effective_rank_after": 15.992917292909281, + "nuclear_norm_before_full": 0.34606579318642616, + "nuclear_norm_before": 0.34606579318642616, + "nuclear_norm_after": 0.34606579318642616, + "fro_norm_before_full": 0.13407805684843593, + "fro_norm_after": 0.0865546710796876, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11606984585523605, + 0.035935644060373306, + 0.027458274737000465, + 0.023357678204774857, + 0.021011408418416977, + 0.018847424536943436, + 0.014741153456270695, + 0.01361707504838705, + 0.011150804348289967, + 0.010524768382310867, + 0.010050206445157528, + 0.009738032706081867, + 0.009326234459877014, + 0.008719251491129398, + 0.007862785831093788, + 0.007655205205082893 + ], + "sigma_after": [ + 0.021644362811687037, + 0.021649851780774575, + 0.022148937735350087, + 0.0227530666228514, + 0.022707657478802757, + 0.02235527848100963, + 0.021656103253239706, + 0.02161820325426241, + 0.02165070395938483, + 0.021609963865483903, + 0.021541464376520863, + 0.02147333690587184, + 0.02134972007216063, + 0.021084125195515138, + 0.020502919247988714, + 0.020320098145522672 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.16.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.456379674599239, + "effective_rank_before": 10.456379674599239, + "effective_rank_after": 15.978353906775542, + "nuclear_norm_before_full": 0.289235498290509, + "nuclear_norm_before": 0.289235498290509, + "nuclear_norm_after": 0.289235498290509, + "fro_norm_before_full": 0.11159735358575423, + "fro_norm_after": 0.07240394232221094, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09409427642822266, + 0.040312495082616806, + 0.0195587370544672, + 0.018449699506163597, + 0.016275733709335327, + 0.013415076769888401, + 0.012055675499141216, + 0.011223237961530685, + 0.010556278750300407, + 0.01009584404528141, + 0.009021100588142872, + 0.008394905366003513, + 0.007823345251381397, + 0.007015069480985403, + 0.00598284462466836, + 0.004961178172379732 + ], + "sigma_after": [ + 0.018216266008791976, + 0.017600153380813565, + 0.01922562729012726, + 0.019255884868001994, + 0.019014718032526495, + 0.018438544721474295, + 0.01829325533059918, + 0.018274467516331275, + 0.01828656975584253, + 0.018299310454411143, + 0.018289607984517772, + 0.01821501175136064, + 0.01806851841412747, + 0.017680770397256844, + 0.016773028614675223, + 0.015303763769651329 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.16.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.869612151739089, + "effective_rank_before": 9.869612151739089, + "effective_rank_after": 15.96051074583863, + "nuclear_norm_before_full": 0.1697991630062461, + "nuclear_norm_before": 0.1697991630062461, + "nuclear_norm_after": 0.1697991630062461, + "fro_norm_before_full": 0.06649429346787963, + "fro_norm_after": 0.042550178193886366, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.053596075624227524, + 0.02728082984685898, + 0.01858602464199066, + 0.011759285815060139, + 0.009163961745798588, + 0.0066382139921188354, + 0.006063585169613361, + 0.005353859160095453, + 0.005045606289058924, + 0.004671517759561539, + 0.00432645995169878, + 0.004033184610307217, + 0.0038793166168034077, + 0.0037873785477131605, + 0.0030247364193201065, + 0.002589126816019416 + ], + "sigma_after": [ + 0.0107630468921786, + 0.010696459998571044, + 0.010996252569340734, + 0.01154151164472296, + 0.011313460019398315, + 0.010978875425241692, + 0.01099223386863004, + 0.010986212342511353, + 0.010950493318349361, + 0.010858228916965685, + 0.010708480356723105, + 0.010519743473614859, + 0.010394643077553048, + 0.01031059761207917, + 0.009312251651647724, + 0.008476671838718397 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.17.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.375343421915097, + "effective_rank_before": 7.375343421915097, + "effective_rank_after": 15.831664784971439, + "nuclear_norm_before_full": 0.37280256766825914, + "nuclear_norm_before": 0.37280256766825914, + "nuclear_norm_after": 0.3728025676682592, + "fro_norm_before_full": 0.18952485042540207, + "fro_norm_after": 0.09416472585039987, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.17976480722427368, + 0.03481356054544449, + 0.029588380828499794, + 0.021771056577563286, + 0.01636124588549137, + 0.011111724190413952, + 0.010392359457910061, + 0.00968508142977953, + 0.009172827005386353, + 0.008818242698907852, + 0.008289073593914509, + 0.007269467692822218, + 0.006823253817856312, + 0.006731749512255192, + 0.006196967791765928, + 0.006012769415974617 + ], + "sigma_after": [ + 0.026644145225529936, + 0.027842590211010775, + 0.027933052082553856, + 0.026723129194726615, + 0.02664937005693298, + 0.025228321420594405, + 0.024636865561549058, + 0.0239194325148592, + 0.023309907470236944, + 0.022841863027694528, + 0.022071217652120525, + 0.020337048145653513, + 0.019473958655249906, + 0.019289161919069323, + 0.01815657756337723, + 0.017745926967100374 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.17.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.372124908835138, + "effective_rank_before": 9.372124908835138, + "effective_rank_after": 15.993276361439962, + "nuclear_norm_before_full": 0.7674034014344215, + "nuclear_norm_before": 0.7674034014344215, + "nuclear_norm_after": 0.7674034014344214, + "fro_norm_before_full": 0.3358619107989584, + "fro_norm_after": 0.19193111251084902, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.3098064363002777, + 0.07186860591173172, + 0.04947558045387268, + 0.03876345232129097, + 0.036018650978803635, + 0.031586453318595886, + 0.030325544998049736, + 0.02689713053405285, + 0.02518310584127903, + 0.02415655180811882, + 0.023376593366265297, + 0.0222689937800169, + 0.020188964903354645, + 0.019545909017324448, + 0.0190203245729208, + 0.018921103328466415 + ], + "sigma_after": [ + 0.04858300085022754, + 0.04931784951306806, + 0.05063073527164389, + 0.048821720115258996, + 0.04862174380153461, + 0.04862000858298357, + 0.04865063800035558, + 0.04860774859747648, + 0.04841178587018862, + 0.048201673730445906, + 0.04798420859324424, + 0.047575413301962466, + 0.04643036905215773, + 0.045961965706441135, + 0.04553489035998872, + 0.045449650087443914 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.17.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.357378405281697, + "effective_rank_before": 9.357378405281697, + "effective_rank_after": 15.978782105612579, + "nuclear_norm_before_full": 0.7145466776564717, + "nuclear_norm_before": 0.7145466776564717, + "nuclear_norm_after": 0.7145466776564717, + "fro_norm_before_full": 0.3067991404232177, + "fro_norm_after": 0.17886855487072814, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2761094272136688, + 0.08983725309371948, + 0.0481671504676342, + 0.036707524210214615, + 0.03259754925966263, + 0.028377799317240715, + 0.02682773768901825, + 0.024716444313526154, + 0.02426883392035961, + 0.022951724007725716, + 0.020976992323994637, + 0.019130174070596695, + 0.01847466453909874, + 0.016333524137735367, + 0.014795984141528606, + 0.01427389495074749 + ], + "sigma_after": [ + 0.045803254860992515, + 0.0457247079779105, + 0.04807643704542671, + 0.046148616927351206, + 0.045786892306632306, + 0.04580916449200048, + 0.04584021141041254, + 0.04579506797570711, + 0.04576165498443272, + 0.04559160423628672, + 0.045068216522886353, + 0.04418369512101456, + 0.043757294183764534, + 0.0418822396846361, + 0.04002637052907756, + 0.03929124939793984 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.17.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.290617371063192, + "effective_rank_before": 12.290617371063192, + "effective_rank_after": 15.995376473483295, + "nuclear_norm_before_full": 0.15233698417432606, + "nuclear_norm_before": 0.15233698417432606, + "nuclear_norm_after": 0.15233698417432606, + "fro_norm_before_full": 0.05021983338808665, + "fro_norm_after": 0.038095260867382214, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03697265684604645, + 0.019720682874321938, + 0.01436703372746706, + 0.011145549826323986, + 0.008369780145585537, + 0.007706766482442617, + 0.007393170613795519, + 0.006669564638286829, + 0.006258074659854174, + 0.006038718856871128, + 0.005694621242582798, + 0.00550079345703125, + 0.004634333774447441, + 0.004250310827046633, + 0.003996803890913725, + 0.00361812231130898 + ], + "sigma_after": [ + 0.009098791262844498, + 0.009123187293310473, + 0.009452264285290163, + 0.009525257898272167, + 0.0099528525293083, + 0.0098981695684199, + 0.009841611435372068, + 0.00966996235335971, + 0.009573337271501278, + 0.009529680923889722, + 0.009478062987908202, + 0.009459433416282515, + 0.009454584424057898, + 0.009460537737402578, + 0.00944695598249352, + 0.009372294804613076 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.17.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.474357021118534, + "effective_rank_before": 9.474357021118534, + "effective_rank_after": 15.969774989703815, + "nuclear_norm_before_full": 0.3485752264969051, + "nuclear_norm_before": 0.3485752264969051, + "nuclear_norm_after": 0.3485752264969051, + "fro_norm_before_full": 0.14778315025169245, + "fro_norm_after": 0.08730327684881461, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1322716921567917, + 0.04375886172056198, + 0.022146020084619522, + 0.019916122779250145, + 0.016503669321537018, + 0.014837809838354588, + 0.014703522436320782, + 0.013001981191337109, + 0.011979461647570133, + 0.010821104981005192, + 0.01000251155346632, + 0.009296773001551628, + 0.008054189383983612, + 0.007820039056241512, + 0.007293522823601961, + 0.006167944520711899 + ], + "sigma_after": [ + 0.022444432419667537, + 0.02237468474836424, + 0.023421841977343767, + 0.023011104245082813, + 0.02247544762712633, + 0.022417992660171738, + 0.022418966242872516, + 0.022455169286059326, + 0.022438390005283443, + 0.022287522423741185, + 0.022038766928272856, + 0.02169372617689161, + 0.020711853635283408, + 0.02046517251621758, + 0.01983168421203638, + 0.01808847139249039 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.17.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.01467278493594, + "effective_rank_before": 11.01467278493594, + "effective_rank_after": 15.992399762208885, + "nuclear_norm_before_full": 0.28879500553011894, + "nuclear_norm_before": 0.28879500553011894, + "nuclear_norm_after": 0.28879500553011894, + "fro_norm_before_full": 0.10650894563032665, + "fro_norm_after": 0.07223280162630959, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08762459456920624, + 0.038661032915115356, + 0.0208780225366354, + 0.01964387483894825, + 0.01656987890601158, + 0.013814940117299557, + 0.01306496188044548, + 0.011288824491202831, + 0.011051730252802372, + 0.009952593594789505, + 0.009313877671957016, + 0.008694135583937168, + 0.008114834316074848, + 0.00762789836153388, + 0.006736005190759897, + 0.005757800303399563 + ], + "sigma_after": [ + 0.01785109512735386, + 0.017381078899670867, + 0.01867356180988592, + 0.018883901632991466, + 0.018949829301738507, + 0.018411908152991718, + 0.018261124893963844, + 0.018062535149504236, + 0.018056072651028363, + 0.018069388899775073, + 0.01808461557548403, + 0.018073913955216788, + 0.018013240704733684, + 0.017902510062893538, + 0.017493851338555806, + 0.01662637737433125 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.17.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.355094102041456, + "effective_rank_before": 8.355094102041456, + "effective_rank_after": 15.698662506160355, + "nuclear_norm_before_full": 0.17654250201303512, + "nuclear_norm_before": 0.17654250201303512, + "nuclear_norm_after": 0.17654250201303512, + "fro_norm_before_full": 0.07752532698475972, + "fro_norm_after": 0.04490285433651054, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06627031415700912, + 0.02924605831503868, + 0.02013782225549221, + 0.009769455529749393, + 0.008304222486913204, + 0.0066987439058721066, + 0.00597693957388401, + 0.005045740399509668, + 0.004775599576532841, + 0.004332590848207474, + 0.00374820944853127, + 0.0033775614574551582, + 0.0027488297782838345, + 0.0022261417470872402, + 0.002008766634389758, + 0.001875505899079144 + ], + "sigma_after": [ + 0.012610411241152891, + 0.01213674528763787, + 0.012622972607803633, + 0.012806917778895342, + 0.012623835232103378, + 0.01263632898398671, + 0.012596238071752714, + 0.012304803132563923, + 0.012139720063925304, + 0.011766221829100233, + 0.011047074066940424, + 0.010442085484649062, + 0.00913502585220373, + 0.0077805598331698745, + 0.007149446736683212, + 0.006744115810466813 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.18.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.496339866287068, + "effective_rank_before": 8.496339866287068, + "effective_rank_after": 15.967070698787364, + "nuclear_norm_before_full": 0.35563273541629314, + "nuclear_norm_before": 0.35563273541629314, + "nuclear_norm_after": 0.35563273541629314, + "fro_norm_before_full": 0.1662396227146286, + "fro_norm_after": 0.08908907405975455, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1554080694913864, + 0.03416959568858147, + 0.02637411653995514, + 0.018322592601180077, + 0.014412205666303635, + 0.013397471979260445, + 0.012510711327195168, + 0.010761935263872147, + 0.010300403460860252, + 0.009945681318640709, + 0.009674275293946266, + 0.008827968500554562, + 0.008390742354094982, + 0.008024245500564575, + 0.0075733535923063755, + 0.007539366837590933 + ], + "sigma_after": [ + 0.023259570471911434, + 0.02380419553090016, + 0.02445049577020236, + 0.023295944463572946, + 0.02329302498974713, + 0.02326953167270048, + 0.02317776097370883, + 0.02265527408516358, + 0.022413062604557114, + 0.022190701864657884, + 0.02199778366646466, + 0.02125870690893788, + 0.02078930169028647, + 0.020346988891736566, + 0.01973953226057227, + 0.01969085957117336 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.18.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.341116499494877, + "effective_rank_before": 10.341116499494877, + "effective_rank_after": 15.997727318192785, + "nuclear_norm_before_full": 0.7403691094368696, + "nuclear_norm_before": 0.7403691094368696, + "nuclear_norm_after": 0.7403691094368696, + "fro_norm_before_full": 0.29813579689643505, + "fro_norm_after": 0.1851187362470492, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.26705417037010193, + 0.06684213131666183, + 0.05531373992562294, + 0.04358496516942978, + 0.03558819368481636, + 0.03125334158539772, + 0.029586896300315857, + 0.02827673777937889, + 0.02610957808792591, + 0.025713663548231125, + 0.024217456579208374, + 0.023114992305636406, + 0.022518586367368698, + 0.021560918539762497, + 0.0203542523086071, + 0.019279485568404198 + ], + "sigma_after": [ + 0.046167059924183323, + 0.04643763408046053, + 0.048190318785483084, + 0.048000513128653825, + 0.04649168630742593, + 0.046126452255952445, + 0.04611732281931054, + 0.04614290555123021, + 0.04619244531857845, + 0.04619494373916385, + 0.04615931808881018, + 0.046064506765259666, + 0.045979684965388894, + 0.04578222193185298, + 0.04540335408608402, + 0.04491874168903177 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.18.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.239305734262285, + "effective_rank_before": 10.239305734262285, + "effective_rank_after": 15.994675647514851, + "nuclear_norm_before_full": 0.6444349261000752, + "nuclear_norm_before": 0.6444349261000752, + "nuclear_norm_after": 0.6444349261000752, + "fro_norm_before_full": 0.2602892347056218, + "fro_norm_after": 0.1611623878335928, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.23241673409938812, + 0.06359034031629562, + 0.0435929112136364, + 0.041141122579574585, + 0.031737588346004486, + 0.030370082706212997, + 0.02503456361591816, + 0.024199193343520164, + 0.022920556366443634, + 0.021489130333065987, + 0.021104225888848305, + 0.019522465765476227, + 0.018666420131921768, + 0.017571326345205307, + 0.016072850674390793, + 0.015005414374172688 + ], + "sigma_after": [ + 0.040316146083982124, + 0.04026974094501824, + 0.042431408555077384, + 0.0423250954593894, + 0.04069483202804448, + 0.04050000789755742, + 0.040280284075536736, + 0.04030169909408929, + 0.040330455438181916, + 0.040318827115355095, + 0.04030087074743355, + 0.04012264433222989, + 0.039932988181370845, + 0.03956411104312488, + 0.03877446233396607, + 0.037971352769717956 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.18.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.213815829291157, + "effective_rank_before": 12.213815829291157, + "effective_rank_after": 15.995507113412561, + "nuclear_norm_before_full": 0.15588924614712596, + "nuclear_norm_before": 0.15588924614712596, + "nuclear_norm_after": 0.15588924614712596, + "fro_norm_before_full": 0.0506311039111275, + "fro_norm_after": 0.03898331207218027, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03473668918013573, + 0.020297329872846603, + 0.016535019502043724, + 0.01430104672908783, + 0.010974234901368618, + 0.008283914998173714, + 0.008000030182301998, + 0.006641503423452377, + 0.006118786986917257, + 0.005376152228564024, + 0.0049947695806622505, + 0.0045232512056827545, + 0.004417043179273605, + 0.00408322224393487, + 0.0034466919023543596, + 0.003159560030326247 + ], + "sigma_after": [ + 0.009569539229236788, + 0.00942277966224518, + 0.009531464967651733, + 0.009732475833711542, + 0.009842164194649838, + 0.01023837581335746, + 0.01021666855861012, + 0.00993052290382393, + 0.00981273261127468, + 0.009724447817737718, + 0.009720462774572438, + 0.009734118938880572, + 0.009736040841204642, + 0.009726675776510738, + 0.009564741090822768, + 0.009386035132835818 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.18.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.851846689731895, + "effective_rank_before": 10.851846689731895, + "effective_rank_after": 15.991232383583961, + "nuclear_norm_before_full": 0.3711155788041651, + "nuclear_norm_before": 0.3711155788041651, + "nuclear_norm_after": 0.3711155788041651, + "fro_norm_before_full": 0.1390249415792942, + "fro_norm_after": 0.09282925732897483, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11643313616514206, + 0.04623064026236534, + 0.02820277214050293, + 0.02574084885418415, + 0.021642720326781273, + 0.016615327447652817, + 0.01639932580292225, + 0.015010309405624866, + 0.013735968619585037, + 0.012639462947845459, + 0.01222753245383501, + 0.01145576685667038, + 0.010583934374153614, + 0.00856085866689682, + 0.008333389647305012, + 0.007303584832698107 + ], + "sigma_after": [ + 0.023165809135554064, + 0.022741316190143843, + 0.02390466869810965, + 0.024339511408733652, + 0.024440670098876623, + 0.023477177362183976, + 0.023444768234457812, + 0.02330700532883287, + 0.02328583140336114, + 0.023313545091840835, + 0.02332229465562119, + 0.023314465450577422, + 0.023229771922622027, + 0.022405285221132866, + 0.02223299186033782, + 0.021190466741779344 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.18.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.323480847595597, + "effective_rank_before": 11.323480847595597, + "effective_rank_after": 15.996981199101487, + "nuclear_norm_before_full": 0.2982498640194535, + "nuclear_norm_before": 0.2982498640194535, + "nuclear_norm_after": 0.2982498640194535, + "fro_norm_before_full": 0.1075777274045827, + "fro_norm_after": 0.07457657052602434, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08692080527544022, + 0.041310615837574005, + 0.023545166477560997, + 0.01690606027841568, + 0.015233558602631092, + 0.014069801196455956, + 0.01296355202794075, + 0.012441130355000496, + 0.012218804098665714, + 0.010777481831610203, + 0.010077483020722866, + 0.009183782152831554, + 0.008947513997554779, + 0.008146573789417744, + 0.007989357225596905, + 0.007518177852034569 + ], + "sigma_after": [ + 0.01821844488590829, + 0.017882663826337665, + 0.018765738767624954, + 0.01950445588733205, + 0.019221128630178354, + 0.018957910153723164, + 0.018735924754190336, + 0.018658203296739662, + 0.018631966751020065, + 0.018564172753859486, + 0.01857914416217549, + 0.01859617497401958, + 0.018591735155125442, + 0.018513569353887618, + 0.018482321894862205, + 0.01834630877246918 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.18.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.36065835883158, + "effective_rank_before": 8.36065835883158, + "effective_rank_after": 15.789278180100037, + "nuclear_norm_before_full": 0.14091680967248976, + "nuclear_norm_before": 0.14091680967248976, + "nuclear_norm_after": 0.14091680967248976, + "fro_norm_before_full": 0.06335049986243428, + "fro_norm_after": 0.03566582618410206, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.056456468999385834, + 0.01995536871254444, + 0.012664813548326492, + 0.009453206323087215, + 0.007348347455263138, + 0.004988418892025948, + 0.004617133643478155, + 0.004287282004952431, + 0.003692388767376542, + 0.003440065309405327, + 0.002902371808886528, + 0.002847898518666625, + 0.0023865338880568743, + 0.0022921240888535976, + 0.0018896062392741442, + 0.0016947814729064703 + ], + "sigma_after": [ + 0.009856453020684275, + 0.009727814028771462, + 0.010137774275522355, + 0.01027732936379871, + 0.00989530077433111, + 0.009840489869271442, + 0.009780557491893702, + 0.009676098244600431, + 0.009316352854364076, + 0.009080685006765633, + 0.008381416226310424, + 0.008294512186154722, + 0.007434067728793478, + 0.007230345172432607, + 0.006258122575385437, + 0.005729490853409898 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.19.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.613955677814081, + "effective_rank_before": 9.613955677814081, + "effective_rank_after": 15.98824559485955, + "nuclear_norm_before_full": 0.39343083556741476, + "nuclear_norm_before": 0.39343083556741476, + "nuclear_norm_after": 0.39343083556741476, + "fro_norm_before_full": 0.16504602184050698, + "fro_norm_after": 0.09842937625917665, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1475902646780014, + 0.044896434992551804, + 0.03149373456835747, + 0.022480538114905357, + 0.020267752930521965, + 0.0168681051582098, + 0.014728732407093048, + 0.013977115042507648, + 0.01293939258903265, + 0.011049685068428516, + 0.010646822862327099, + 0.010166392661631107, + 0.009571913629770279, + 0.009274481795728207, + 0.009000471793115139, + 0.008478997275233269 + ], + "sigma_after": [ + 0.02507792997112393, + 0.025089426601139213, + 0.026046870983906577, + 0.025757919952094508, + 0.025338311890975712, + 0.0250473694675614, + 0.025085464011914697, + 0.02508963800207519, + 0.02503578480126006, + 0.024573890328274915, + 0.02438424749950208, + 0.024104470180705665, + 0.023668615672171545, + 0.023410084876138944, + 0.023146522637882708, + 0.022574288690687585 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.19.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.206517461414476, + "effective_rank_before": 9.206517461414476, + "effective_rank_after": 15.971259636796178, + "nuclear_norm_before_full": 0.7709508975967765, + "nuclear_norm_before": 0.7709508975967765, + "nuclear_norm_after": 0.7709508975967765, + "fro_norm_before_full": 0.3389520967931887, + "fro_norm_after": 0.1930769163580876, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.31175294518470764, + 0.07254045456647873, + 0.054306358098983765, + 0.04458778724074364, + 0.03906877711415291, + 0.03291213512420654, + 0.029577981680631638, + 0.027992861345410347, + 0.026332585141062737, + 0.023246394470334053, + 0.021394651383161545, + 0.02062373422086239, + 0.01811019890010357, + 0.01720634661614895, + 0.016169080510735512, + 0.015128605999052525 + ], + "sigma_after": [ + 0.049637720190978396, + 0.05038569053878918, + 0.0522312080243973, + 0.050739100128396865, + 0.04987481904044206, + 0.04965073304348663, + 0.04972176456264097, + 0.0497048893781518, + 0.049589042720926714, + 0.04890906293732223, + 0.04807369727489103, + 0.04760515511942989, + 0.04549548017636125, + 0.04449524701803859, + 0.043176027909889614, + 0.04166125953263394 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.19.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.459669645078767, + "effective_rank_before": 10.459669645078767, + "effective_rank_after": 15.995505098631615, + "nuclear_norm_before_full": 0.7676513344049454, + "nuclear_norm_before": 0.7676513344049454, + "nuclear_norm_after": 0.7676513344049455, + "fro_norm_before_full": 0.30259291344992156, + "fro_norm_after": 0.19196694589512947, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2660662829875946, + 0.07851400226354599, + 0.05590429902076721, + 0.05156262591481209, + 0.042920295149087906, + 0.03289829567074776, + 0.03016868233680725, + 0.02839646115899086, + 0.02810746058821678, + 0.02582489885389805, + 0.024107199162244797, + 0.022678960114717484, + 0.022435542196035385, + 0.02019650861620903, + 0.019656559452414513, + 0.018213260918855667 + ], + "sigma_after": [ + 0.04792432719534087, + 0.04787224332872774, + 0.05002718970730311, + 0.05040359327692361, + 0.049548556407371484, + 0.0479012024524004, + 0.047842417900498306, + 0.04787850072748148, + 0.04788646011458655, + 0.04792373586221333, + 0.04785493106004364, + 0.0476700343454754, + 0.04762292012959359, + 0.04690926503316931, + 0.04664731789649118, + 0.045738638967325475 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.19.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.674433180125227, + "effective_rank_before": 12.674433180125227, + "effective_rank_after": 15.99649154294128, + "nuclear_norm_before_full": 0.1601153239607811, + "nuclear_norm_before": 0.1601153239607811, + "nuclear_norm_after": 0.1601153239607811, + "fro_norm_before_full": 0.050294888433687956, + "fro_norm_after": 0.04003762099927089, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03044772893190384, + 0.024506527930498123, + 0.01899746246635914, + 0.012494722381234169, + 0.00993227120488882, + 0.007896470837295055, + 0.007837447337806225, + 0.006804646458476782, + 0.006513397675007582, + 0.005788804031908512, + 0.005523208528757095, + 0.00515705207362771, + 0.004866204224526882, + 0.004664537496864796, + 0.004353654570877552, + 0.0043311878107488155 + ], + "sigma_after": [ + 0.009927292888113316, + 0.009925224496571659, + 0.00953565920560644, + 0.009910236259803801, + 0.010234409443608641, + 0.010411509855206058, + 0.010403910258463999, + 0.010179538726373456, + 0.010104299258874258, + 0.009956693555986015, + 0.009927089821804288, + 0.009910563868982588, + 0.009913584482097091, + 0.009920053234985158, + 0.009927597910024531, + 0.00992766069427981 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.19.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.549596835668757, + "effective_rank_before": 10.549596835668757, + "effective_rank_after": 15.991980825362528, + "nuclear_norm_before_full": 0.35986697813495994, + "nuclear_norm_before": 0.35986697813495994, + "nuclear_norm_after": 0.35986697813495994, + "fro_norm_before_full": 0.13984059193893036, + "fro_norm_after": 0.09001116301341615, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12115003913640976, + 0.040969785302877426, + 0.02669847197830677, + 0.023784466087818146, + 0.018392324447631836, + 0.015944426879286766, + 0.015273620374500751, + 0.014091476798057556, + 0.012808423489332199, + 0.012191289104521275, + 0.011524483561515808, + 0.01090774405747652, + 0.010162006132304668, + 0.010095221921801567, + 0.00865637231618166, + 0.007216826546937227 + ], + "sigma_after": [ + 0.022551637633358865, + 0.02250368896713415, + 0.023419840894179058, + 0.023731048782482198, + 0.023022817933503543, + 0.02261208471266562, + 0.02255593873648811, + 0.02252314299676403, + 0.02254979803293255, + 0.022561953919901068, + 0.022553242677477457, + 0.02250683062312409, + 0.022373516783289794, + 0.022356337210766845, + 0.021704479245472037, + 0.0203406189854205 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.19.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.523708266528065, + "effective_rank_before": 11.523708266528065, + "effective_rank_after": 15.996035414713674, + "nuclear_norm_before_full": 0.3202051930129528, + "nuclear_norm_before": 0.3202051930129528, + "nuclear_norm_after": 0.32020519301295275, + "fro_norm_before_full": 0.11299742021555165, + "fro_norm_after": 0.08007115897214077, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08925574272871017, + 0.04448366537690163, + 0.023654595017433167, + 0.022601505741477013, + 0.017420662567019463, + 0.016359303146600723, + 0.014112676493823528, + 0.01325895357877016, + 0.01251967716962099, + 0.012062964029610157, + 0.010922955349087715, + 0.010032241232693195, + 0.009821559302508831, + 0.008278732188045979, + 0.00820442009717226, + 0.007215538993477821 + ], + "sigma_after": [ + 0.019382565558732315, + 0.019265270296882853, + 0.020328763724538478, + 0.020531407798194148, + 0.020904859549324632, + 0.020719037273316934, + 0.020214387177303644, + 0.02006377804908197, + 0.019978597482804422, + 0.019949299510207644, + 0.01994293914438234, + 0.019967717789115873, + 0.019970651211000973, + 0.01982141856799582, + 0.019801705246697502, + 0.019362794633373225 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.19.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.969202955475755, + "effective_rank_before": 9.969202955475755, + "effective_rank_after": 15.936614757041847, + "nuclear_norm_before_full": 0.1591345041524619, + "nuclear_norm_before": 0.1591345041524619, + "nuclear_norm_after": 0.1591345041524619, + "fro_norm_before_full": 0.06343442875836836, + "fro_norm_after": 0.039934612275634405, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.054639291018247604, + 0.019507022574543953, + 0.013315862976014614, + 0.011121345683932304, + 0.009309766814112663, + 0.007739085704088211, + 0.0066121225245296955, + 0.006282214540988207, + 0.005684468895196915, + 0.004951781127601862, + 0.004660214763134718, + 0.004074172116816044, + 0.003345902543514967, + 0.00280237989500165, + 0.0027118567377328873, + 0.0023770162370055914 + ], + "sigma_after": [ + 0.010339317025400016, + 0.010257138388293183, + 0.010529139527792362, + 0.010869277319104179, + 0.010760987880756787, + 0.010443153610441128, + 0.01033363222157164, + 0.010332626138329614, + 0.010347104301050769, + 0.010324708572402025, + 0.010275291494204544, + 0.010048272850677123, + 0.009419080796515806, + 0.008621415734470238, + 0.008458043374793382, + 0.0077753149166591064 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.2.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.58188091826724, + "effective_rank_before": 9.58188091826724, + "effective_rank_after": 15.984294254374838, + "nuclear_norm_before_full": 0.38181981490924954, + "nuclear_norm_before": 0.38181981490924954, + "nuclear_norm_after": 0.38181981490924954, + "fro_norm_before_full": 0.15634077195835416, + "fro_norm_after": 0.09554719883945974, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.132634699344635, + 0.06074574217200279, + 0.03218535706400871, + 0.024009428918361664, + 0.016797104850411415, + 0.01594104990363121, + 0.012276578694581985, + 0.011853018775582314, + 0.010988140478730202, + 0.010532351210713387, + 0.010008192621171474, + 0.00979678612202406, + 0.009725027717649937, + 0.008673999458551407, + 0.008319792337715626, + 0.00733254523947835 + ], + "sigma_after": [ + 0.024410475961972614, + 0.02358595722677463, + 0.02502442145562571, + 0.025629540538432614, + 0.024473615058289582, + 0.02445292410537014, + 0.024443838969115526, + 0.024388767524681772, + 0.024186314101120175, + 0.024018912446485963, + 0.023763627721914895, + 0.0236395546169043, + 0.02359449948225579, + 0.022747307559070225, + 0.022375658105907432, + 0.021084400035328178 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.2.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.307791629079281, + "effective_rank_before": 8.307791629079281, + "effective_rank_after": 15.370330677300629, + "nuclear_norm_before_full": 0.38081165275070816, + "nuclear_norm_before": 0.38081165275070816, + "nuclear_norm_after": 0.38081165275070816, + "fro_norm_before_full": 0.16726702236026253, + "fro_norm_after": 0.09827108045005983, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1425437033176422, + 0.06782657653093338, + 0.03207226097583771, + 0.02710099145770073, + 0.018102990463376045, + 0.015246494673192501, + 0.0134843485429883, + 0.012300859205424786, + 0.010483074001967907, + 0.009621238335967064, + 0.008714769966900349, + 0.006625909358263016, + 0.00601773289963603, + 0.005206409376114607, + 0.003765731118619442, + 0.0016985625261440873 + ], + "sigma_after": [ + 0.02759876276057585, + 0.026842280169276478, + 0.028697894147775502, + 0.029084742539853558, + 0.027652182833849005, + 0.027656555215196017, + 0.027642390993862606, + 0.027473828310948013, + 0.02670427520610563, + 0.026029554795050834, + 0.025049497616168183, + 0.021567426447493005, + 0.020210058485362656, + 0.018157509540538623, + 0.013872775799940505, + 0.006571917888711709 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.2.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.482423396420569, + "effective_rank_before": 7.482423396420569, + "effective_rank_after": 15.29734375375406, + "nuclear_norm_before_full": 0.3711416501319036, + "nuclear_norm_before": 0.3711416501319036, + "nuclear_norm_after": 0.3711416501319036, + "fro_norm_before_full": 0.17850796625777546, + "fro_norm_after": 0.09627052494065427, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.16245555877685547, + 0.054282356053590775, + 0.03022179752588272, + 0.02112380973994732, + 0.017901364713907242, + 0.01632341928780079, + 0.01413725782185793, + 0.009951918385922909, + 0.008438038639724255, + 0.007346182130277157, + 0.006508643738925457, + 0.006233902648091316, + 0.005934697575867176, + 0.0048045930452644825, + 0.003639831207692623, + 0.0018382788402959704 + ], + "sigma_after": [ + 0.028369507399241406, + 0.02820625751201314, + 0.02988371327044188, + 0.028565580204044712, + 0.028360178398820074, + 0.028394068547618275, + 0.02836107556674438, + 0.026429507716745265, + 0.024553190369040877, + 0.022685408372616164, + 0.020945016491614437, + 0.020315744128149963, + 0.01959801491290388, + 0.0165916247334157, + 0.013042560651939355, + 0.006840201856554108 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.2.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.97194696724507, + "effective_rank_before": 13.97194696724507, + "effective_rank_after": 15.995898581989948, + "nuclear_norm_before_full": 0.17868351796641946, + "nuclear_norm_before": 0.17868351796641946, + "nuclear_norm_after": 0.17868351796641946, + "fro_norm_before_full": 0.05117606800498241, + "fro_norm_after": 0.04468233148400682, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.02678716368973255, + 0.021979253739118576, + 0.020513182505965233, + 0.012891915626823902, + 0.012642288580536842, + 0.011310792528092861, + 0.009551351889967918, + 0.009254027158021927, + 0.008278357796370983, + 0.007377222180366516, + 0.007261499762535095, + 0.006775675341486931, + 0.006426495034247637, + 0.006214641500264406, + 0.0057861157692968845, + 0.005633534863591194 + ], + "sigma_after": [ + 0.011018160727481782, + 0.01083816708430236, + 0.010667697173986341, + 0.01100453166640438, + 0.011003856600548246, + 0.011105104059879007, + 0.011488087382940791, + 0.011540766945049274, + 0.011585510715130098, + 0.011427750339567519, + 0.011397266277758254, + 0.011260448937512282, + 0.011166053598597504, + 0.011115939396866628, + 0.011040570089623353, + 0.01102360697077163 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.2.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.672077279054516, + "effective_rank_before": 8.672077279054516, + "effective_rank_after": 15.960037382256093, + "nuclear_norm_before_full": 0.4193084663711488, + "nuclear_norm_before": 0.4193084663711488, + "nuclear_norm_after": 0.4193084663711488, + "fro_norm_before_full": 0.19267779830650192, + "fro_norm_after": 0.10508150041070764, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.17924101650714874, + 0.040448885411024094, + 0.029368849471211433, + 0.023568324744701385, + 0.02145657315850258, + 0.016062794253230095, + 0.014139893464744091, + 0.012978681363165379, + 0.011980590410530567, + 0.011623742058873177, + 0.011130307801067829, + 0.010884862393140793, + 0.010241333395242691, + 0.009669755585491657, + 0.00887591578066349, + 0.007636940572410822 + ], + "sigma_after": [ + 0.027406450102431756, + 0.027926767150658102, + 0.02869613958357103, + 0.02770827407818977, + 0.027469673543571712, + 0.02744573270425296, + 0.02725082304248204, + 0.02691677012950673, + 0.026439001678514373, + 0.026217188190628866, + 0.02586127059077271, + 0.025661772283920775, + 0.02506348483138234, + 0.024435721305800656, + 0.023404327374421784, + 0.021405069781043225 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.2.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.310010007146104, + "effective_rank_before": 11.310010007146104, + "effective_rank_after": 15.996155636003362, + "nuclear_norm_before_full": 0.30417296569794416, + "nuclear_norm_before": 0.30417296569794416, + "nuclear_norm_after": 0.30417296569794416, + "fro_norm_before_full": 0.11004478284143455, + "fro_norm_after": 0.07606155650110956, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09042471647262573, + 0.037452735006809235, + 0.023099588230252266, + 0.020257260650396347, + 0.01634899340569973, + 0.015573023818433285, + 0.014061912894248962, + 0.013166103512048721, + 0.012125201523303986, + 0.011623415164649487, + 0.010120153427124023, + 0.008939089253544807, + 0.008454431779682636, + 0.008060131222009659, + 0.0075067393481731415, + 0.006959469988942146 + ], + "sigma_after": [ + 0.018694352715116005, + 0.018383654216044334, + 0.01927704142670678, + 0.019790934845388464, + 0.019738108095729887, + 0.01958314932744522, + 0.01924501458457775, + 0.01907527900000182, + 0.018950641294744743, + 0.018923300420323502, + 0.01893676479639523, + 0.018934121441352626, + 0.01888419715427447, + 0.018806233745396635, + 0.0186213747550841, + 0.018328797879362594 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.2.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 6.667891471131118, + "effective_rank_before": 6.667891471131118, + "effective_rank_after": 15.533571670110135, + "nuclear_norm_before_full": 0.19090072298422456, + "nuclear_norm_before": 0.19090072298422456, + "nuclear_norm_after": 0.19090072298422456, + "fro_norm_before_full": 0.10317204108640968, + "fro_norm_after": 0.04904526529051482, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09902332723140717, + 0.01868470013141632, + 0.01169517170637846, + 0.009276509284973145, + 0.008077053353190422, + 0.006709233392030001, + 0.006353893782943487, + 0.005134884733706713, + 0.004562638700008392, + 0.004266798030585051, + 0.004032005090266466, + 0.0030745475087314844, + 0.0027040913701057434, + 0.002649095840752125, + 0.0023768628016114235, + 0.0022799100261181593 + ], + "sigma_after": [ + 0.014653872243000896, + 0.015316010267386403, + 0.014657545988141587, + 0.014627105241368928, + 0.014598343681778342, + 0.014245044502621793, + 0.014055610834860633, + 0.012973173810783806, + 0.012195338855154924, + 0.011719188178253303, + 0.011304616068790529, + 0.00927987820007549, + 0.008358187702918855, + 0.008215202504108614, + 0.007485124424844773, + 0.0072164804801356855 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.20.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.774706493192216, + "effective_rank_before": 10.774706493192216, + "effective_rank_after": 15.996856158441012, + "nuclear_norm_before_full": 0.3864388456568122, + "nuclear_norm_before": 0.3864388456568122, + "nuclear_norm_after": 0.38643884565681214, + "fro_norm_before_full": 0.1469035206631902, + "fro_norm_after": 0.09662880138934021, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1254223883152008, + 0.04225492477416992, + 0.03491036966443062, + 0.02469228021800518, + 0.020757107064127922, + 0.016708413138985634, + 0.01567818596959114, + 0.014553094282746315, + 0.01387978345155716, + 0.01330486498773098, + 0.012670084834098816, + 0.011904067359864712, + 0.011154343374073505, + 0.009966921992599964, + 0.009364208206534386, + 0.009217808023095131 + ], + "sigma_after": [ + 0.02411278998363517, + 0.024137561064589457, + 0.02415342107446732, + 0.025423634112755753, + 0.024969845296655798, + 0.024215937899194726, + 0.024140889345185792, + 0.02412703430984086, + 0.024141572528955397, + 0.024157568478521282, + 0.024167331943941512, + 0.024146963551498302, + 0.024064678109508703, + 0.023727347849931694, + 0.023421559765037715, + 0.023330710343092677 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.20.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.05380708492239, + "effective_rank_before": 10.05380708492239, + "effective_rank_after": 15.991828158708987, + "nuclear_norm_before_full": 0.7690479159355164, + "nuclear_norm_before": 0.7690479159355164, + "nuclear_norm_after": 0.7690479159355164, + "fro_norm_before_full": 0.3147971978140537, + "fro_norm_after": 0.1923590652600084, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.28211408853530884, + 0.07734807580709457, + 0.053602226078510284, + 0.046326201409101486, + 0.03818575292825699, + 0.03611060604453087, + 0.03294983133673668, + 0.027854036539793015, + 0.02646264247596264, + 0.02555054798722267, + 0.02271558716893196, + 0.022023679688572884, + 0.021531863138079643, + 0.020583301782608032, + 0.01941078156232834, + 0.016278693452477455 + ], + "sigma_after": [ + 0.04834741046967789, + 0.04829625025872359, + 0.050881569495149714, + 0.050320805189611746, + 0.04877919949433908, + 0.04850626740383573, + 0.04830489580749159, + 0.048371412844114725, + 0.04837228044746742, + 0.04833938223479311, + 0.04793299160276203, + 0.04773487380046591, + 0.047563931825292005, + 0.04715538510131726, + 0.04648893009044229, + 0.04365232987003227 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.20.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.949225919611296, + "effective_rank_before": 9.949225919611296, + "effective_rank_after": 15.992059433471821, + "nuclear_norm_before_full": 0.6955309314653277, + "nuclear_norm_before": 0.6955309314653277, + "nuclear_norm_after": 0.6955309314653277, + "fro_norm_before_full": 0.287064659437082, + "fro_norm_after": 0.17396817808070253, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.25773686170578003, + 0.07165711373090744, + 0.050839800387620926, + 0.041318077594041824, + 0.032149169594049454, + 0.02966570481657982, + 0.02712203375995159, + 0.0252065546810627, + 0.024736709892749786, + 0.0230246689170599, + 0.02159152925014496, + 0.021062493324279785, + 0.01941188983619213, + 0.01784992404282093, + 0.01713613234460354, + 0.01502226758748293 + ], + "sigma_after": [ + 0.04379138224091145, + 0.04374664607919136, + 0.045983920453625954, + 0.045420779687901314, + 0.04386638703009756, + 0.04375020081654124, + 0.043776462560636664, + 0.04382003895836273, + 0.04382318568777915, + 0.04377117935301457, + 0.04360384450552125, + 0.04350165495164608, + 0.042999782545693796, + 0.042206830204913434, + 0.041720722410536974, + 0.03974791397895425 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.20.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.303302732219343, + "effective_rank_before": 12.303302732219343, + "effective_rank_after": 15.995263747619253, + "nuclear_norm_before_full": 0.15593422180972993, + "nuclear_norm_before": 0.15593422180972993, + "nuclear_norm_after": 0.15593422180972993, + "fro_norm_before_full": 0.05119363943778966, + "fro_norm_after": 0.038995123489202664, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03770158067345619, + 0.01727953553199768, + 0.016816897317767143, + 0.01194726675748825, + 0.009903364814817905, + 0.008277886547148228, + 0.007843278348445892, + 0.007121672388166189, + 0.006461224518716335, + 0.005745138507336378, + 0.005205982364714146, + 0.00501166982576251, + 0.004554337821900845, + 0.004274320788681507, + 0.004106215201318264, + 0.0036838504020124674 + ], + "sigma_after": [ + 0.009328482230282358, + 0.00942734700341398, + 0.009483800323909873, + 0.009707050354255566, + 0.010043069799851317, + 0.010195206592648042, + 0.010146548768290796, + 0.009995307939774043, + 0.00983357379909031, + 0.00971118289458913, + 0.009683409764412709, + 0.009685278772750602, + 0.009698599241269718, + 0.009698502184800907, + 0.009688895817047271, + 0.009607966323343311 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.20.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.255197645567417, + "effective_rank_before": 12.255197645567417, + "effective_rank_after": 15.996315262921488, + "nuclear_norm_before_full": 0.401882597245276, + "nuclear_norm_before": 0.401882597245276, + "nuclear_norm_after": 0.401882597245276, + "fro_norm_before_full": 0.13259520051872556, + "fro_norm_after": 0.10049383924934413, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09645234793424606, + 0.05594443902373314, + 0.034988727420568466, + 0.027707599103450775, + 0.025741003453731537, + 0.021479787304997444, + 0.01937585510313511, + 0.017923451960086823, + 0.01539939921349287, + 0.014663610607385635, + 0.01411597989499569, + 0.013307507149875164, + 0.01296162512153387, + 0.011705862358212471, + 0.010386164300143719, + 0.009729237295687199 + ], + "sigma_after": [ + 0.024063069141312426, + 0.024438825394235586, + 0.02494454471086989, + 0.025417605457903707, + 0.02582416749473772, + 0.026236557556073056, + 0.025934632275593, + 0.025590124347921375, + 0.025052966864392023, + 0.024967897222711356, + 0.0249318720418701, + 0.02491739733834468, + 0.0249223225794873, + 0.024958005373652083, + 0.0249047930548236, + 0.024777816391348087 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.20.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.101437810083358, + "effective_rank_before": 11.101437810083358, + "effective_rank_after": 15.995874108248413, + "nuclear_norm_before_full": 0.31903522089123726, + "nuclear_norm_before": 0.31903522089123726, + "nuclear_norm_after": 0.3190352208912372, + "fro_norm_before_full": 0.11697988735263881, + "fro_norm_after": 0.07977945790384598, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09634682536125183, + 0.039639756083488464, + 0.028471140190958977, + 0.019629916176199913, + 0.017382102087140083, + 0.015375135466456413, + 0.014555088244378567, + 0.012957478873431683, + 0.01232850644737482, + 0.010629194788634777, + 0.00969245657324791, + 0.009562747552990913, + 0.009063256904482841, + 0.008582106791436672, + 0.007620297838002443, + 0.00719921151176095 + ], + "sigma_after": [ + 0.019684475506195467, + 0.01935713388505754, + 0.01990396154551704, + 0.02097063013844712, + 0.020765135861307914, + 0.020338461960251967, + 0.020166453023460314, + 0.019941512773734404, + 0.01990656659002505, + 0.01992523757932087, + 0.019927949973460684, + 0.019922241982299024, + 0.019877066058032144, + 0.019787732483322423, + 0.019414862601406048, + 0.019145798929399188 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.20.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.750290951971673, + "effective_rank_before": 9.750290951971673, + "effective_rank_after": 15.895908498416146, + "nuclear_norm_before_full": 0.1677609784528613, + "nuclear_norm_before": 0.1677609784528613, + "nuclear_norm_after": 0.1677609784528613, + "fro_norm_before_full": 0.06755840468265047, + "fro_norm_after": 0.04219682473423358, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.05811522528529167, + 0.02035335823893547, + 0.015835361555218697, + 0.011684738099575043, + 0.010464007034897804, + 0.008551071397960186, + 0.00642118277028203, + 0.005901221185922623, + 0.005673128645867109, + 0.00496236328035593, + 0.0045564197935163975, + 0.004154752008616924, + 0.0035056108608841896, + 0.0029178564436733723, + 0.0024328671861439943, + 0.0022318146657198668 + ], + "sigma_after": [ + 0.011037290055527588, + 0.010986091905621389, + 0.011053934220736925, + 0.011614829647387099, + 0.011572928946693132, + 0.011206703105996117, + 0.011037396337938966, + 0.011049773968765247, + 0.011048824726554106, + 0.010970407322855659, + 0.010837398470117702, + 0.010611105671775742, + 0.00998593105694878, + 0.009085129245691279, + 0.008076389264197525, + 0.007586844506054059 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.21.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.528177398005031, + "effective_rank_before": 10.528177398005031, + "effective_rank_after": 15.997618719051315, + "nuclear_norm_before_full": 0.36704180389642715, + "nuclear_norm_before": 0.36704180389642715, + "nuclear_norm_after": 0.3670418038964271, + "fro_norm_before_full": 0.14530964436551633, + "fro_norm_after": 0.09177423581204701, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12897124886512756, + 0.036749523133039474, + 0.02389267273247242, + 0.0213052649050951, + 0.018958838656544685, + 0.015688486397266388, + 0.015148495323956013, + 0.013724720105528831, + 0.013272468000650406, + 0.012953550554811954, + 0.012458637356758118, + 0.011473643593490124, + 0.011420384049415588, + 0.010891998186707497, + 0.010259796865284443, + 0.00987207517027855 + ], + "sigma_after": [ + 0.022842523069430826, + 0.02280986851360239, + 0.024028770131243812, + 0.02375090205016956, + 0.023281797510069733, + 0.02283006795513, + 0.022811178340484014, + 0.022822723561430915, + 0.022834927477814487, + 0.02284224026711617, + 0.022846915486626793, + 0.022804839676967836, + 0.022799646262822718, + 0.0227259378440426, + 0.022573111576398814, + 0.022436354173076455 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.21.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.070701782708388, + "effective_rank_before": 10.070701782708388, + "effective_rank_after": 15.998293226979545, + "nuclear_norm_before_full": 0.7643486186861992, + "nuclear_norm_before": 0.7643486186861992, + "nuclear_norm_after": 0.7643486186861992, + "fro_norm_before_full": 0.3133693388298186, + "fro_norm_after": 0.19110757446922777, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2804199159145355, + 0.08695743978023529, + 0.044897764921188354, + 0.03900734707713127, + 0.036698468029499054, + 0.03332339599728584, + 0.02928273193538189, + 0.027819180861115456, + 0.027617279440164566, + 0.02688533067703247, + 0.024198736995458603, + 0.022965572774410248, + 0.02207385189831257, + 0.02163836359977722, + 0.020885610952973366, + 0.019677627831697464 + ], + "sigma_after": [ + 0.047880394158338946, + 0.04790507642491333, + 0.04961525567121736, + 0.04846793486901619, + 0.048120444365907275, + 0.047849975930565544, + 0.04786436359587153, + 0.04789905937618604, + 0.04790217205956807, + 0.04790683916265914, + 0.047759728626674554, + 0.047549528284998144, + 0.047316985654503034, + 0.047174693648662516, + 0.04687903495790912, + 0.04625713189920841 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.21.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.832374879688262, + "effective_rank_before": 10.832374879688262, + "effective_rank_after": 15.996851581641298, + "nuclear_norm_before_full": 0.7313113883137703, + "nuclear_norm_before": 0.7313113883137703, + "nuclear_norm_after": 0.7313113883137703, + "fro_norm_before_full": 0.28137648954536904, + "fro_norm_after": 0.18286390614486647, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.24638524651527405, + 0.06987739354372025, + 0.05319447070360184, + 0.04401283338665962, + 0.037616048008203506, + 0.033468522131443024, + 0.0317862369120121, + 0.02986261434853077, + 0.02879074588418007, + 0.025837667286396027, + 0.02505076490342617, + 0.024229682981967926, + 0.023448636755347252, + 0.020950868725776672, + 0.01965738832950592, + 0.017142267897725105 + ], + "sigma_after": [ + 0.045558801585219466, + 0.0454799763095343, + 0.04737696702478032, + 0.04776349349585365, + 0.046618661216491074, + 0.045834911995438085, + 0.045629963981119535, + 0.04550360376747221, + 0.04548099828088235, + 0.045532205245223265, + 0.04555111555951779, + 0.04555992957425086, + 0.045549806967368406, + 0.04529140178071096, + 0.04494483793022483, + 0.04363471359968311 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.21.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.378912934113107, + "effective_rank_before": 12.378912934113107, + "effective_rank_after": 15.996964879886347, + "nuclear_norm_before_full": 0.15849530580453575, + "nuclear_norm_before": 0.15849530580453575, + "nuclear_norm_after": 0.15849530580453575, + "fro_norm_before_full": 0.05139592315612885, + "fro_norm_after": 0.03963138276781382, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03520208224654198, + 0.023626621812582016, + 0.015671024098992348, + 0.010560298338532448, + 0.009902388788759708, + 0.008144590072333813, + 0.007955696433782578, + 0.0067716240882873535, + 0.006330560427159071, + 0.006062279921025038, + 0.005929537117481232, + 0.005201856140047312, + 0.005015742499381304, + 0.004358280450105667, + 0.004171489272266626, + 0.0035912340972572565 + ], + "sigma_after": [ + 0.009652351387901284, + 0.009773080357402207, + 0.00974072438050795, + 0.010031124354675042, + 0.010169946821830205, + 0.010301702101952904, + 0.010278622909957376, + 0.01001817150559033, + 0.009915737099912949, + 0.009865692883328392, + 0.009845654669090551, + 0.009797808150481046, + 0.009800077005774822, + 0.009814602354399569, + 0.009807227050898608, + 0.00968278277083253 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.21.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.792874703968126, + "effective_rank_before": 11.792874703968126, + "effective_rank_after": 15.996014003113446, + "nuclear_norm_before_full": 0.38255809899419546, + "nuclear_norm_before": 0.38255809899419546, + "nuclear_norm_after": 0.38255809899419546, + "fro_norm_before_full": 0.13109514290245194, + "fro_norm_after": 0.09566340443110434, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09849727153778076, + 0.05725477263331413, + 0.029282281175255775, + 0.026992563158273697, + 0.022329041734337807, + 0.01983647048473358, + 0.01754966378211975, + 0.016532089561223984, + 0.014840898104012012, + 0.01406865194439888, + 0.013316989876329899, + 0.012635876424610615, + 0.01071892585605383, + 0.010541722178459167, + 0.009615842252969742, + 0.008545038290321827 + ], + "sigma_after": [ + 0.022857602897627616, + 0.0234945147002944, + 0.023943305925296143, + 0.02431825707892235, + 0.025037082839662446, + 0.024860915383902824, + 0.024364271272618424, + 0.02412915994791811, + 0.023845308563127856, + 0.023783007428382234, + 0.023763552066699948, + 0.023772751095112354, + 0.023791108459210116, + 0.023779219554141424, + 0.023629880737114115, + 0.02318816104416509 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.21.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.280341853573175, + "effective_rank_before": 11.280341853573175, + "effective_rank_after": 15.98420737216225, + "nuclear_norm_before_full": 0.317487517837435, + "nuclear_norm_before": 0.317487517837435, + "nuclear_norm_after": 0.317487517837435, + "fro_norm_before_full": 0.11508704879369062, + "fro_norm_after": 0.07944791983268386, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09536711871623993, + 0.035508330911397934, + 0.02468627691268921, + 0.02041262574493885, + 0.019738147035241127, + 0.017427703365683556, + 0.015137984417378902, + 0.01367975678294897, + 0.012464924715459347, + 0.011852847412228584, + 0.010634439997375011, + 0.010088400915265083, + 0.009588987566530704, + 0.008601573295891285, + 0.006957340054214001, + 0.005341059993952513 + ], + "sigma_after": [ + 0.0196793039811423, + 0.01969422806888142, + 0.020139510824346323, + 0.02086117944855441, + 0.020907721190232872, + 0.020767525683802485, + 0.020289507060881787, + 0.020002224391574924, + 0.019871192687842952, + 0.01984999283665016, + 0.019869608749900222, + 0.01988246347896645, + 0.019879199083608017, + 0.019773263152721033, + 0.019001057860153405, + 0.01701953933817624 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.21.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.904998191646802, + "effective_rank_before": 9.904998191646802, + "effective_rank_after": 15.917325380517456, + "nuclear_norm_before_full": 0.15472017996944487, + "nuclear_norm_before": 0.15472017996944487, + "nuclear_norm_after": 0.15472017996944487, + "fro_norm_before_full": 0.060347878146191906, + "fro_norm_after": 0.03886124841504682, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.048390839248895645, + 0.025678087025880814, + 0.015379699878394604, + 0.011646478436887264, + 0.00753340870141983, + 0.006723893340677023, + 0.005480214953422546, + 0.005243093241006136, + 0.004870230332016945, + 0.004357683006674051, + 0.004243684466928244, + 0.0036896190140396357, + 0.0035970339085906744, + 0.0032998621463775635, + 0.0027890456840395927, + 0.0017973065841943026 + ], + "sigma_after": [ + 0.009872592080439946, + 0.009918870384081735, + 0.01009508857276394, + 0.010472346411247984, + 0.010232713771443601, + 0.010117140085492384, + 0.010105741514454713, + 0.010110044588218877, + 0.01010037880990009, + 0.010018073632577942, + 0.00998328569834659, + 0.009692674755022978, + 0.00962076853453009, + 0.009336563984085456, + 0.008634675025517492, + 0.00640922212132104 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.22.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.970023237533256, + "effective_rank_before": 9.970023237533256, + "effective_rank_after": 15.992424476648434, + "nuclear_norm_before_full": 0.3847266808152199, + "nuclear_norm_before": 0.3847266808152199, + "nuclear_norm_after": 0.3847266808152199, + "fro_norm_before_full": 0.15664091510422073, + "fro_norm_after": 0.09622688288619134, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.13766349852085114, + 0.04913754761219025, + 0.026970822364091873, + 0.023901507258415222, + 0.017150139436125755, + 0.015379339456558228, + 0.01432892307639122, + 0.013068364933133125, + 0.01280156709253788, + 0.012563602067530155, + 0.011895062401890755, + 0.011091019958257675, + 0.010502324439585209, + 0.010262923315167427, + 0.00976178515702486, + 0.008248253725469112 + ], + "sigma_after": [ + 0.024208900777981532, + 0.023904138491279498, + 0.02545147628241659, + 0.02530974604743688, + 0.024204239384392683, + 0.024170433981476502, + 0.024195590333516266, + 0.024203911242793173, + 0.02419419661520569, + 0.024180159329242466, + 0.024106265015422092, + 0.023928026285858564, + 0.0237165905146488, + 0.02360735191502501, + 0.023329621239919253, + 0.022016033358604907 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.22.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.217602578150386, + "effective_rank_before": 7.217602578150386, + "effective_rank_after": 15.891295645299058, + "nuclear_norm_before_full": 0.9153339304029942, + "nuclear_norm_before": 0.9153339304029942, + "nuclear_norm_after": 0.915333930402994, + "fro_norm_before_full": 0.4841631104085574, + "fro_norm_after": 0.23034834278755906, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.4674060046672821, + 0.06635832041501999, + 0.046537380665540695, + 0.041714441031217575, + 0.03641961142420769, + 0.03480582684278488, + 0.030402416363358498, + 0.02611577697098255, + 0.0248260460793972, + 0.023607611656188965, + 0.022856757044792175, + 0.02142033912241459, + 0.01996327005326748, + 0.019088279455900192, + 0.01806599274277687, + 0.0157458558678627 + ], + "sigma_after": [ + 0.06383101892513493, + 0.06543166457912777, + 0.06358312830454066, + 0.06366765298608801, + 0.06334947332806191, + 0.06305339572875607, + 0.06147511771664342, + 0.05847934613220839, + 0.057233092162685725, + 0.05589432470031498, + 0.054988503420236846, + 0.053079073753855725, + 0.05090024077968422, + 0.04947314841290896, + 0.0476925866495849, + 0.04320216282316151 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.22.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.585811334217574, + "effective_rank_before": 8.585811334217574, + "effective_rank_after": 15.963196215287374, + "nuclear_norm_before_full": 0.8196650557219982, + "nuclear_norm_before": 0.8196650557219982, + "nuclear_norm_after": 0.8196650557219983, + "fro_norm_before_full": 0.3823785241760685, + "fro_norm_after": 0.20537982770773355, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.35880258679389954, + 0.06782068312168121, + 0.05766492709517479, + 0.0453156977891922, + 0.04010706767439842, + 0.02993122488260269, + 0.029091423377394676, + 0.02649855613708496, + 0.025072097778320312, + 0.02326120436191559, + 0.02257367968559265, + 0.021261505782604218, + 0.019881924614310265, + 0.01811651512980461, + 0.017767474055290222, + 0.016498487442731857 + ], + "sigma_after": [ + 0.05339016053478679, + 0.056083295729916875, + 0.05580057823761333, + 0.05375713333628289, + 0.05337652066678945, + 0.05333336385396686, + 0.05323620252274431, + 0.05266834077732682, + 0.05213543721924507, + 0.051174939373773765, + 0.05071600276000566, + 0.04968078071718616, + 0.048349630054693435, + 0.04625408139168994, + 0.04578516204472536, + 0.043923426501251576 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.22.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.201225672883327, + "effective_rank_before": 13.201225672883327, + "effective_rank_after": 15.99594677284425, + "nuclear_norm_before_full": 0.17716475808992982, + "nuclear_norm_before": 0.17716475808992982, + "nuclear_norm_after": 0.17716475808992982, + "fro_norm_before_full": 0.053548354210560374, + "fro_norm_after": 0.04430242333448397, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.031069476157426834, + 0.02553803287446499, + 0.019126705825328827, + 0.013226361945271492, + 0.012060851790010929, + 0.01049770973622799, + 0.008988602086901665, + 0.008544137701392174, + 0.007886195555329323, + 0.0073454855009913445, + 0.00678245397284627, + 0.005993444938212633, + 0.005654849577695131, + 0.004961301106959581, + 0.004802957642823458, + 0.00468619167804718 + ], + "sigma_after": [ + 0.01094390968887024, + 0.010938562920945662, + 0.01053947797126941, + 0.010931077314157994, + 0.011000532458870994, + 0.011305910828313295, + 0.01151937452308658, + 0.011499052616806827, + 0.01139117436962488, + 0.011255158091776164, + 0.011106295401301999, + 0.010960476955698519, + 0.010935331862573675, + 0.010941864736868418, + 0.010947033940955121, + 0.01094952440881003 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.22.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.141229828785544, + "effective_rank_before": 11.141229828785544, + "effective_rank_after": 15.998205336839867, + "nuclear_norm_before_full": 0.3661096468567848, + "nuclear_norm_before": 0.3661096468567848, + "nuclear_norm_after": 0.3661096468567849, + "fro_norm_before_full": 0.1356015226647797, + "fro_norm_after": 0.09153774601235885, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11480189859867096, + 0.03883444517850876, + 0.030271975323557854, + 0.024893352761864662, + 0.018785737454891205, + 0.01627645082771778, + 0.015881823375821114, + 0.015136618167161942, + 0.014466899447143078, + 0.013103872537612915, + 0.012238893657922745, + 0.011905788443982601, + 0.010868850164115429, + 0.010237295180559158, + 0.009395564906299114, + 0.009010180830955505 + ], + "sigma_after": [ + 0.022731901513547648, + 0.02279922394140588, + 0.022955959446230866, + 0.02384562925184844, + 0.02349841487239389, + 0.022982534593772485, + 0.022923681122291647, + 0.022839801588757116, + 0.02279653750934562, + 0.02278975472274546, + 0.022813576591810644, + 0.02282011307089317, + 0.022794081504711815, + 0.02271450507763052, + 0.022485184870802033, + 0.02231874717859766 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.22.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.236232126417871, + "effective_rank_before": 11.236232126417871, + "effective_rank_after": 15.994292747807744, + "nuclear_norm_before_full": 0.33000878244638443, + "nuclear_norm_before": 0.33000878244638443, + "nuclear_norm_after": 0.33000878244638443, + "fro_norm_before_full": 0.11991063680600036, + "fro_norm_after": 0.08253156307959812, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09855976700782776, + 0.040951959788799286, + 0.02678893506526947, + 0.02038767747581005, + 0.01783490926027298, + 0.016880199313163757, + 0.016013333573937416, + 0.014171812683343887, + 0.012591380625963211, + 0.01131557859480381, + 0.011074908077716827, + 0.01029654685407877, + 0.0096652302891016, + 0.008471899665892124, + 0.008156681433320045, + 0.006847962737083435 + ], + "sigma_after": [ + 0.02032421214858071, + 0.01997023115002278, + 0.02071731518851118, + 0.02166532628907982, + 0.021460125304708744, + 0.02126873053045288, + 0.021073254177912785, + 0.02070891394287484, + 0.020568483066544796, + 0.020576879225377008, + 0.020584035729218075, + 0.020598431123303434, + 0.02057757191210123, + 0.020353358050775733, + 0.020232668876913957, + 0.0193292457300065 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.22.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.203818172679728, + "effective_rank_before": 10.203818172679728, + "effective_rank_after": 15.984417482725718, + "nuclear_norm_before_full": 0.1780179983470589, + "nuclear_norm_before": 0.1780179983470589, + "nuclear_norm_after": 0.1780179983470589, + "fro_norm_before_full": 0.06864409817941222, + "fro_norm_after": 0.04454717090003655, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.05576883256435394, + 0.027455121278762817, + 0.018485071137547493, + 0.01035630889236927, + 0.008788430131971836, + 0.007862385362386703, + 0.007544654421508312, + 0.006526983343064785, + 0.005591203924268484, + 0.0055510178208351135, + 0.004783651325851679, + 0.004664191044867039, + 0.003955512307584286, + 0.003729635151103139, + 0.0036803570110350847, + 0.0032746426295489073 + ], + "sigma_after": [ + 0.011188528368654395, + 0.011021978115597991, + 0.011391578485839361, + 0.011894464033037067, + 0.011575503892093221, + 0.01142279833298307, + 0.01139368797822871, + 0.011380603232216575, + 0.011385971738995283, + 0.011383557020556398, + 0.01123585155416846, + 0.011189384137838381, + 0.01072481360993828, + 0.010495720173750547, + 0.01043982697456649, + 0.009893730698594658 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.23.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.57547315494069, + "effective_rank_before": 9.57547315494069, + "effective_rank_after": 15.993660600674554, + "nuclear_norm_before_full": 0.39159304928034544, + "nuclear_norm_before": 0.39159304928034544, + "nuclear_norm_after": 0.39159304928034544, + "fro_norm_before_full": 0.1659908686580082, + "fro_norm_after": 0.0979369595175283, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14933432638645172, + 0.04694914072751999, + 0.028144696727395058, + 0.018675686791539192, + 0.016658663749694824, + 0.016188235953450203, + 0.014704742468893528, + 0.013755328953266144, + 0.013160242699086666, + 0.011844207532703876, + 0.011503206565976143, + 0.011223936453461647, + 0.010675780475139618, + 0.009973626583814621, + 0.009651081636548042, + 0.009150145575404167 + ], + "sigma_after": [ + 0.024766987560637888, + 0.024776610223320904, + 0.026073134472702444, + 0.024819183930013172, + 0.024744517554480875, + 0.02475006368720582, + 0.0247840824995535, + 0.024779916344513462, + 0.024746573071207697, + 0.024525340368446367, + 0.024422776457106925, + 0.024321630458500172, + 0.024073021078460124, + 0.023644841926770337, + 0.023402096713927827, + 0.022962272933497906 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.23.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.759776440509913, + "effective_rank_before": 9.759776440509913, + "effective_rank_after": 15.991620800141812, + "nuclear_norm_before_full": 0.83238735049963, + "nuclear_norm_before": 0.83238735049963, + "nuclear_norm_after": 0.83238735049963, + "fro_norm_before_full": 0.34993207074215893, + "fro_norm_after": 0.20820577259042777, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.31731507182121277, + 0.07773272693157196, + 0.05796955153346062, + 0.05727098509669304, + 0.03885507956147194, + 0.03617975860834122, + 0.03181123360991478, + 0.029676664620637894, + 0.027410926297307014, + 0.027091752737760544, + 0.024682994931936264, + 0.023593908175826073, + 0.02221045270562172, + 0.021069783717393875, + 0.02005547657608986, + 0.01946098357439041 + ], + "sigma_after": [ + 0.0524108026257013, + 0.0528245666821883, + 0.05519973011048822, + 0.05518231353673373, + 0.052502025637106445, + 0.05238888238378446, + 0.05245204984800352, + 0.05247377505806308, + 0.05235832293201464, + 0.05232324027562833, + 0.051839840397016404, + 0.05146261916168695, + 0.05080407982309349, + 0.05008773890437712, + 0.04930331852516148, + 0.04877404459858249 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.23.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.393400962763149, + "effective_rank_before": 9.393400962763149, + "effective_rank_after": 15.995043799527112, + "nuclear_norm_before_full": 0.7865049112588167, + "nuclear_norm_before": 0.7865049112588167, + "nuclear_norm_after": 0.7865049112588167, + "fro_norm_before_full": 0.34275446921122216, + "fro_norm_after": 0.19668695051695975, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.3147733211517334, + 0.08061660081148148, + 0.0518198236823082, + 0.03886951506137848, + 0.03194202482700348, + 0.031026260927319527, + 0.0282601211220026, + 0.027317287400364876, + 0.025861110538244247, + 0.0250957440584898, + 0.02468164823949337, + 0.02346652001142502, + 0.021758567541837692, + 0.021488532423973083, + 0.02028856985270977, + 0.019239263609051704 + ], + "sigma_after": [ + 0.04963623334949768, + 0.04972479956583672, + 0.051919766928488514, + 0.04981030466972276, + 0.049678759076506346, + 0.04970098231064686, + 0.049700495823701604, + 0.0496494598691022, + 0.04948472771299951, + 0.049344925979021355, + 0.04925118170931269, + 0.048892130404873224, + 0.048138531662883614, + 0.04798900215241444, + 0.04721234062015428, + 0.04637126942365494 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.23.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.588956190162895, + "effective_rank_before": 13.588956190162895, + "effective_rank_after": 15.994792786750583, + "nuclear_norm_before_full": 0.1748089944012463, + "nuclear_norm_before": 0.1748089944012463, + "nuclear_norm_after": 0.1748089944012463, + "fro_norm_before_full": 0.051342267363126644, + "fro_norm_after": 0.0437164687237602, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.029828567057847977, + 0.019987501204013824, + 0.01821690797805786, + 0.01567283272743225, + 0.013840260915458202, + 0.010925901122391224, + 0.009036419913172722, + 0.007963505573570728, + 0.007771339733153582, + 0.007548114284873009, + 0.00684579461812973, + 0.006202446762472391, + 0.005864573642611504, + 0.005326681304723024, + 0.004969982896000147, + 0.004808164667338133 + ], + "sigma_after": [ + 0.01082284214685355, + 0.010429204114979761, + 0.010431231236839325, + 0.010744806664407915, + 0.010826764199597605, + 0.010985351928017046, + 0.011368771271791289, + 0.01134273048222772, + 0.01130946122688283, + 0.011261942794995433, + 0.011075412841102888, + 0.01091322519847093, + 0.01085396851489284, + 0.010811164274711696, + 0.010813349401850436, + 0.010818768103625054 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.23.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.693048014172353, + "effective_rank_before": 11.693048014172353, + "effective_rank_after": 15.997613207822926, + "nuclear_norm_before_full": 0.4150440450757742, + "nuclear_norm_before": 0.4150440450757742, + "nuclear_norm_after": 0.41504404507577425, + "fro_norm_before_full": 0.1459583053208705, + "fro_norm_after": 0.10377661438376258, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11893682926893234, + 0.042581044137477875, + 0.034601617604494095, + 0.031382594257593155, + 0.02534453012049198, + 0.021837642416357994, + 0.01904672011733055, + 0.01820472814142704, + 0.015217754058539867, + 0.014761957339942455, + 0.014092108234763145, + 0.013043588027358055, + 0.012366869486868382, + 0.011937442235648632, + 0.011229874566197395, + 0.010458745062351227 + ], + "sigma_after": [ + 0.025348572077215137, + 0.025723461093233268, + 0.025770085300779674, + 0.02610104975389598, + 0.02709261822705785, + 0.026882773955959546, + 0.02627272725536497, + 0.026090524139383186, + 0.02574096208004643, + 0.025736072797442576, + 0.025745492967985766, + 0.02577526631794736, + 0.025780047053612042, + 0.02576613862235546, + 0.02569600107001544, + 0.025522252363479568 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.23.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.691223498009444, + "effective_rank_before": 11.691223498009444, + "effective_rank_after": 15.995951046154513, + "nuclear_norm_before_full": 0.34928489103913307, + "nuclear_norm_before": 0.34928489103913307, + "nuclear_norm_after": 0.34928489103913307, + "fro_norm_before_full": 0.12171567172641798, + "fro_norm_after": 0.08734344697930525, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09675450623035431, + 0.04117985814809799, + 0.029473353177309036, + 0.026302916929125786, + 0.020015206187963486, + 0.01941312663257122, + 0.016713693737983704, + 0.01543378084897995, + 0.012673096731305122, + 0.012261022813618183, + 0.011875483207404613, + 0.010678217746317387, + 0.010276075452566147, + 0.009521964006125927, + 0.008474847301840782, + 0.008237741887569427 + ], + "sigma_after": [ + 0.02116246573899669, + 0.021127693086161355, + 0.021720375805664167, + 0.02199900899009651, + 0.02287556768704286, + 0.02284021485616457, + 0.022360512113909756, + 0.02206443989251143, + 0.021715993466503377, + 0.02171236385070028, + 0.021717338718825415, + 0.02174899170724326, + 0.02174878759749661, + 0.021700134890814207, + 0.021445942790356862, + 0.021345059846645746 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.23.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.023860789209516, + "effective_rank_before": 10.023860789209516, + "effective_rank_after": 15.982755502910358, + "nuclear_norm_before_full": 0.17605486093088984, + "nuclear_norm_before": 0.17605486093088984, + "nuclear_norm_after": 0.17605486093088987, + "fro_norm_before_full": 0.07006693027508683, + "fro_norm_after": 0.04406037433249803, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06011031195521355, + 0.022077249363064766, + 0.016829317435622215, + 0.011156887747347355, + 0.009049915708601475, + 0.008823246695101261, + 0.006738394498825073, + 0.00587430177256465, + 0.005607125349342823, + 0.005247865337878466, + 0.0049094101414084435, + 0.004789697006344795, + 0.004019174259155989, + 0.0038964811246842146, + 0.003543783910572529, + 0.003381698625162244 + ], + "sigma_after": [ + 0.011223408041970404, + 0.011089157072506524, + 0.011228164622534903, + 0.01180221474931346, + 0.011435561219992237, + 0.011390965583460554, + 0.011225446897196595, + 0.011238838040140435, + 0.011226157578595514, + 0.01118013493235757, + 0.011093538823079258, + 0.011050239746808554, + 0.010563981206741131, + 0.01044728578117789, + 0.010041611371159737, + 0.0098181552638551 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.24.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.403169281936995, + "effective_rank_before": 10.403169281936995, + "effective_rank_after": 15.995487231807797, + "nuclear_norm_before_full": 0.42924148961901665, + "nuclear_norm_before": 0.42924148961901665, + "nuclear_norm_after": 0.4292414896190166, + "fro_norm_before_full": 0.16612606028385885, + "fro_norm_after": 0.10734064565953834, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1395377814769745, + 0.06033783778548241, + 0.03716367855668068, + 0.026509245857596397, + 0.020469609647989273, + 0.017336266115307808, + 0.016792956739664078, + 0.015119166113436222, + 0.014310956001281738, + 0.014048355631530285, + 0.013184850104153156, + 0.01209330651909113, + 0.011442476883530617, + 0.011022715829312801, + 0.010300504975020885, + 0.009571781381964684 + ], + "sigma_after": [ + 0.026881287416233635, + 0.025987768377141583, + 0.027192788572424853, + 0.028398014198456795, + 0.027320873890599163, + 0.027000113789524438, + 0.026994651642935647, + 0.02702928530490962, + 0.02704210052575296, + 0.027040403571984797, + 0.02699854608908218, + 0.02682072849935772, + 0.026618016067106384, + 0.026438623035329012, + 0.02602505777201064, + 0.02545323086616718 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.24.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.907390704665363, + "effective_rank_before": 7.907390704665363, + "effective_rank_after": 15.929309720586946, + "nuclear_norm_before_full": 0.8711070492863655, + "nuclear_norm_before": 0.8711070492863655, + "nuclear_norm_after": 0.8711070492863654, + "fro_norm_before_full": 0.43346382531695626, + "fro_norm_after": 0.218712901389313, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.41390058398246765, + 0.061230771243572235, + 0.05537018924951553, + 0.043568212538957596, + 0.037367213517427444, + 0.03401216119527817, + 0.031471364200115204, + 0.028745217248797417, + 0.025188349187374115, + 0.02357417345046997, + 0.0231490358710289, + 0.02101871185004711, + 0.020587826147675514, + 0.018871333450078964, + 0.01660110056400299, + 0.016450805589556694 + ], + "sigma_after": [ + 0.05857187715534956, + 0.06047148916350558, + 0.05944271231265087, + 0.058428644395379904, + 0.058530741360578634, + 0.05840466653265898, + 0.05803430316774887, + 0.05721832192498329, + 0.0552487821696295, + 0.053937429960986605, + 0.05354346401345841, + 0.051244008322494414, + 0.05071046770251758, + 0.048346956744516625, + 0.044622059425685756, + 0.04435112493422084 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.24.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.961768698893081, + "effective_rank_before": 8.961768698893081, + "effective_rank_after": 15.985921325891, + "nuclear_norm_before_full": 0.8403274398297071, + "nuclear_norm_before": 0.8403274398297071, + "nuclear_norm_after": 0.8403274398297073, + "fro_norm_before_full": 0.3793564410082756, + "fro_norm_after": 0.21026288318564917, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.3519757091999054, + 0.0849938914179802, + 0.05214276909828186, + 0.04259348660707474, + 0.037191227078437805, + 0.03332420438528061, + 0.029351791366934776, + 0.026435663923621178, + 0.02609175443649292, + 0.025552228093147278, + 0.025222953408956528, + 0.02349659986793995, + 0.022389816120266914, + 0.02132115326821804, + 0.02055812068283558, + 0.01768607087433338 + ], + "sigma_after": [ + 0.05384626922307256, + 0.054227233222443265, + 0.055461006525433156, + 0.05400019538223672, + 0.05385370436178903, + 0.05393456740526051, + 0.053779491295049275, + 0.053190178644452135, + 0.053080187000388496, + 0.052887179509750255, + 0.05275649294197522, + 0.051895217136099056, + 0.051170596531352835, + 0.05032850050441871, + 0.04963555546289034, + 0.04628106468309564 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.24.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.241791625282826, + "effective_rank_before": 13.241791625282826, + "effective_rank_after": 15.996627246607508, + "nuclear_norm_before_full": 0.20531469210982323, + "nuclear_norm_before": 0.20531469210982323, + "nuclear_norm_after": 0.20531469210982323, + "fro_norm_before_full": 0.061790755416170066, + "fro_norm_after": 0.051339547903696936, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03575571998953819, + 0.029033750295639038, + 0.02062246948480606, + 0.017403241246938705, + 0.014882050454616547, + 0.012005660682916641, + 0.010864549316465855, + 0.010149224661290646, + 0.00876588560640812, + 0.00790098961442709, + 0.007375914137810469, + 0.0068360064178705215, + 0.0064194235019385815, + 0.006080607417970896, + 0.006017180625349283, + 0.005202018655836582 + ], + "sigma_after": [ + 0.012699383716759369, + 0.012680918100890982, + 0.012379414384814777, + 0.012702954772907036, + 0.012691684356580616, + 0.013144187378941274, + 0.013341661169815033, + 0.013363810843059749, + 0.013137931921644629, + 0.012906925583423243, + 0.012788553791109004, + 0.012710781185614016, + 0.01268647142355631, + 0.012686462367830536, + 0.01268789091189372, + 0.012705660200982943 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.24.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.05821534331033, + "effective_rank_before": 9.05821534331033, + "effective_rank_after": 15.961636763794061, + "nuclear_norm_before_full": 0.3922554370947182, + "nuclear_norm_before": 0.3922554370947182, + "nuclear_norm_after": 0.3922554370947182, + "fro_norm_before_full": 0.17352798906541705, + "fro_norm_after": 0.0982928690515552, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15925416350364685, + 0.037402957677841187, + 0.03269199654459953, + 0.02032393030822277, + 0.019921625033020973, + 0.017362819984555244, + 0.01546457689255476, + 0.013357960619032383, + 0.012566870078444481, + 0.011173931881785393, + 0.010267343372106552, + 0.009759058244526386, + 0.009306314401328564, + 0.008485130034387112, + 0.007594692520797253, + 0.007322065997868776 + ], + "sigma_after": [ + 0.025539329346221606, + 0.025887446007304093, + 0.0266191790450756, + 0.02569673983002487, + 0.025652004633411826, + 0.025538239422408914, + 0.0255772344056103, + 0.02549866741211034, + 0.025358010239759515, + 0.024851749273757832, + 0.024288682988105357, + 0.0238757112178525, + 0.023442980539072403, + 0.022490238553085167, + 0.021196035547322477, + 0.02074318863359541 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.24.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.302798563311415, + "effective_rank_before": 11.302798563311415, + "effective_rank_after": 15.993863492527428, + "nuclear_norm_before_full": 0.374546161852777, + "nuclear_norm_before": 0.374546161852777, + "nuclear_norm_after": 0.37454616185277695, + "fro_norm_before_full": 0.13472231147884992, + "fro_norm_after": 0.09367224005538718, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10829652100801468, + 0.05148892477154732, + 0.02937224879860878, + 0.024393117055296898, + 0.020724210888147354, + 0.017956752330064774, + 0.01655968651175499, + 0.0155519749969244, + 0.013698797672986984, + 0.013421811163425446, + 0.012038659304380417, + 0.01190115511417389, + 0.010880661197006702, + 0.010855359025299549, + 0.009926125407218933, + 0.007480156607925892 + ], + "sigma_after": [ + 0.022891776834849678, + 0.022509969393407034, + 0.023647414744379782, + 0.024514128043673724, + 0.024506875297202472, + 0.023949923801930235, + 0.02365186583937951, + 0.023492270090187152, + 0.023377049390022674, + 0.023378114231180153, + 0.023412590040864694, + 0.02341529050918008, + 0.023395867641838857, + 0.02339406273718873, + 0.023256261424359433, + 0.021752701833132757 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.24.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.848644793585585, + "effective_rank_before": 9.848644793585585, + "effective_rank_after": 15.95735969780258, + "nuclear_norm_before_full": 0.17652150290086865, + "nuclear_norm_before": 0.17652150290086865, + "nuclear_norm_after": 0.17652150290086865, + "fro_norm_before_full": 0.06957044138541361, + "fro_norm_after": 0.0442432643436479, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.0569828525185585, + 0.02607811614871025, + 0.021176666021347046, + 0.010610141791403294, + 0.00837748870253563, + 0.007451954297721386, + 0.006657083053141832, + 0.006035712081938982, + 0.006000442896038294, + 0.004918981343507767, + 0.004820480477064848, + 0.004150765016674995, + 0.0038371144328266382, + 0.0036344986874610186, + 0.0031041637994349003, + 0.002685041632503271 + ], + "sigma_after": [ + 0.011329998673359445, + 0.011042100328529814, + 0.01141497044248516, + 0.012021119552345317, + 0.011584298180046044, + 0.01148759198399797, + 0.011484485509509257, + 0.011498864370028212, + 0.011498993681999295, + 0.011363370077951783, + 0.011329454505245137, + 0.010945561886905918, + 0.01065441898548001, + 0.010422058002514611, + 0.009632778407091286, + 0.008811438313379414 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.25.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.93802256699663, + "effective_rank_before": 10.93802256699663, + "effective_rank_after": 15.996656050076318, + "nuclear_norm_before_full": 0.4352142522111535, + "nuclear_norm_before": 0.4352142522111535, + "nuclear_norm_after": 0.43521425221115356, + "fro_norm_before_full": 0.16246255462235631, + "fro_norm_after": 0.10882638412803172, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1352529376745224, + 0.05834527686238289, + 0.032778020948171616, + 0.026369716972112656, + 0.022105632349848747, + 0.01991979032754898, + 0.019287463277578354, + 0.01674024388194084, + 0.016082774847745895, + 0.014965595677495003, + 0.014274331741034985, + 0.013162335380911827, + 0.012361333705484867, + 0.012010945938527584, + 0.010899944230914116, + 0.010657908394932747 + ], + "sigma_after": [ + 0.026944577674729853, + 0.02614590780921454, + 0.027890171554228464, + 0.02857228445271915, + 0.027897777450185582, + 0.02744283564069986, + 0.027338902024756445, + 0.027134442340410403, + 0.027135188251666213, + 0.02716249429338068, + 0.027178305194538212, + 0.027157291566011127, + 0.027069156968441737, + 0.027001806426239592, + 0.026631001302435037, + 0.02651210926149664 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.25.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.357688189631256, + "effective_rank_before": 11.357688189631256, + "effective_rank_after": 15.996648972218685, + "nuclear_norm_before_full": 0.7861616499722004, + "nuclear_norm_before": 0.7861616499722004, + "nuclear_norm_after": 0.7861616499722003, + "fro_norm_before_full": 0.2867987920404595, + "fro_norm_after": 0.1965818627221022, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.23947884142398834, + 0.09728014469146729, + 0.0505211241543293, + 0.047705356031656265, + 0.040464818477630615, + 0.036152515560388565, + 0.03228546306490898, + 0.03179680183529854, + 0.030172353610396385, + 0.02970363385975361, + 0.02881437912583351, + 0.026765260845422745, + 0.026598479598760605, + 0.02466898411512375, + 0.022736988961696625, + 0.021016504615545273 + ], + "sigma_after": [ + 0.048460960519706275, + 0.04741387896825681, + 0.05126915474741199, + 0.05137502960648183, + 0.050446323824346426, + 0.04949880121481205, + 0.04890123051703504, + 0.04885760117475622, + 0.04876799703227559, + 0.048757082107778474, + 0.04875224044351328, + 0.04879503034800393, + 0.04879997795305213, + 0.048837393498884094, + 0.048756339128370664, + 0.048472608887515506 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.25.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.598731758751411, + "effective_rank_before": 9.598731758751411, + "effective_rank_after": 15.988937676525039, + "nuclear_norm_before_full": 0.8257282804697752, + "nuclear_norm_before": 0.8257282804697752, + "nuclear_norm_after": 0.8257282804697752, + "fro_norm_before_full": 0.3510798771742458, + "fro_norm_after": 0.2065733047097316, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.3186679482460022, + 0.08505348861217499, + 0.06060098856687546, + 0.04438792169094086, + 0.0382048599421978, + 0.03483132645487785, + 0.031502705067396164, + 0.030362103134393692, + 0.02844914421439171, + 0.02660709246993065, + 0.025073645636439323, + 0.023235758766531944, + 0.021625854074954987, + 0.02031118981540203, + 0.018629861995577812, + 0.018184391781687737 + ], + "sigma_after": [ + 0.05239233018623857, + 0.052386809865375035, + 0.05514609936470089, + 0.05319408438404822, + 0.05243653616433344, + 0.05237193932875622, + 0.05244517494946111, + 0.05245979327117976, + 0.052415770318329793, + 0.052229522357916926, + 0.051910913800772625, + 0.05126329180825261, + 0.050397654352798284, + 0.049445738375375956, + 0.04786307927056425, + 0.04736954267167157 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.25.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.892225128604759, + "effective_rank_before": 11.892225128604759, + "effective_rank_after": 15.996324832279525, + "nuclear_norm_before_full": 0.19151820288971066, + "nuclear_norm_before": 0.19151820288971066, + "nuclear_norm_after": 0.19151820288971066, + "fro_norm_before_full": 0.06485568890385943, + "fro_norm_after": 0.04789055417281918, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04891814664006233, + 0.02302050217986107, + 0.020203322172164917, + 0.014862703159451485, + 0.012222493067383766, + 0.009674321860074997, + 0.008790383115410805, + 0.007958090864121914, + 0.007775175850838423, + 0.0067549943923950195, + 0.005762857384979725, + 0.005633316934108734, + 0.005327444989234209, + 0.005084153264760971, + 0.004797361325472593, + 0.004732935689389706 + ], + "sigma_after": [ + 0.01147949251641138, + 0.011514817811930728, + 0.01181966364105828, + 0.011983616777420861, + 0.012437990550588566, + 0.01245959706775654, + 0.0122613962395584, + 0.012070871290306796, + 0.012036802470632156, + 0.011935377213801822, + 0.01195218063357157, + 0.011954064090023135, + 0.011948437964135868, + 0.011927910865421876, + 0.011876334426265157, + 0.011859649330827523 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.25.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.107946750523558, + "effective_rank_before": 11.107946750523558, + "effective_rank_after": 15.997237574946395, + "nuclear_norm_before_full": 0.44170885998755693, + "nuclear_norm_before": 0.44170885998755693, + "nuclear_norm_after": 0.44170885998755693, + "fro_norm_before_full": 0.1639185558696995, + "fro_norm_after": 0.11044642048704066, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.13893109560012817, + 0.04685114696621895, + 0.03488457575440407, + 0.029513441026210785, + 0.02558583952486515, + 0.02256784774363041, + 0.019150301814079285, + 0.017993047833442688, + 0.016044726595282555, + 0.014594209380447865, + 0.014195374213159084, + 0.013656508177518845, + 0.012748189270496368, + 0.01215431373566389, + 0.011939061805605888, + 0.010899180546402931 + ], + "sigma_after": [ + 0.027338871679563184, + 0.027415711910083473, + 0.02783646240205296, + 0.028743702481964457, + 0.02876115477925731, + 0.02822266516913073, + 0.0275560674692271, + 0.027437859579920513, + 0.027397259231917902, + 0.027434670151158642, + 0.027439790810353415, + 0.027433785946164187, + 0.02736750056729088, + 0.027267525407264008, + 0.027217003844844684, + 0.026838828557363463 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.25.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.472346471591008, + "effective_rank_before": 11.472346471591008, + "effective_rank_after": 15.991268986906608, + "nuclear_norm_before_full": 0.3355756551027298, + "nuclear_norm_before": 0.3355756551027298, + "nuclear_norm_after": 0.3355756551027298, + "fro_norm_before_full": 0.12057207112834833, + "fro_norm_after": 0.08393898517295337, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10032590478658676, + 0.03426339849829674, + 0.02688511274755001, + 0.021404428407549858, + 0.019490113481879234, + 0.018224095925688744, + 0.01612834259867668, + 0.014743593521416187, + 0.013712571933865547, + 0.012696399353444576, + 0.011861948296427727, + 0.011816577054560184, + 0.010370909236371517, + 0.009054973721504211, + 0.008361537009477615, + 0.006235748529434204 + ], + "sigma_after": [ + 0.02071853839315478, + 0.020890528068300794, + 0.021030172326517372, + 0.021929355575811594, + 0.021969433685317154, + 0.021827282914802203, + 0.021391489835059504, + 0.021098046836368838, + 0.020945451554896504, + 0.0208741952634905, + 0.020869904290980384, + 0.020870705069090518, + 0.020903534890994613, + 0.02079620932849103, + 0.02059809977451784, + 0.018862707294936196 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.25.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.940839440625593, + "effective_rank_before": 8.940839440625593, + "effective_rank_after": 15.884752850887274, + "nuclear_norm_before_full": 0.17040598182938993, + "nuclear_norm_before": 0.17040598182938993, + "nuclear_norm_after": 0.17040598182938993, + "fro_norm_before_full": 0.07304688076984854, + "fro_norm_after": 0.04288631465259528, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06311895698308945, + 0.02735629305243492, + 0.015448332764208317, + 0.00841003842651844, + 0.007463783957064152, + 0.007131058257073164, + 0.006102998740971088, + 0.005724603775888681, + 0.005394128616899252, + 0.00452790129929781, + 0.004240148235112429, + 0.0038844887167215347, + 0.0035138395614922047, + 0.0033088005147874355, + 0.002646087668836117, + 0.002134521258994937 + ], + "sigma_after": [ + 0.011459313724373157, + 0.011011500637351831, + 0.011647910102082522, + 0.011501175534633314, + 0.011447423756639303, + 0.011449835511689387, + 0.011465215736491103, + 0.01144191152762904, + 0.011390705443804827, + 0.011035222980497087, + 0.010820491432965897, + 0.010471813500661174, + 0.009999668951809187, + 0.009687347017171047, + 0.00841600799175679, + 0.007160437979834265 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.26.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.170358845583284, + "effective_rank_before": 10.170358845583284, + "effective_rank_after": 15.995921815072101, + "nuclear_norm_before_full": 0.43665238190442324, + "nuclear_norm_before": 0.43665238190442324, + "nuclear_norm_after": 0.43665238190442324, + "fro_norm_before_full": 0.17409612117406373, + "fro_norm_after": 0.1091910635176219, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15087857842445374, + 0.0566970556974411, + 0.033097464591264725, + 0.027721315622329712, + 0.020884623751044273, + 0.017980128526687622, + 0.01673072576522827, + 0.015186003409326077, + 0.013805530965328217, + 0.013454018160700798, + 0.012920244596898556, + 0.011945982463657856, + 0.011797010898590088, + 0.01166130043566227, + 0.01150453556329012, + 0.010387863032519817 + ], + "sigma_after": [ + 0.027390253683954258, + 0.02684303664959814, + 0.0284667475183201, + 0.028772416130119043, + 0.027594150176567796, + 0.027357143736579256, + 0.027366632640223553, + 0.027402031369141745, + 0.027357765627354553, + 0.027320043909937978, + 0.02723298568567719, + 0.026956323224376275, + 0.026898007890045077, + 0.026840729568970403, + 0.026769446275637227, + 0.026084667817920654 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.26.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.59687629312848, + "effective_rank_before": 11.59687629312848, + "effective_rank_after": 15.997688889805847, + "nuclear_norm_before_full": 0.7730281166732311, + "nuclear_norm_before": 0.7730281166732311, + "nuclear_norm_after": 0.7730281166732312, + "fro_norm_before_full": 0.2769920217343807, + "fro_norm_after": 0.1932851217381665, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.23047105967998505, + 0.0867597907781601, + 0.05460856482386589, + 0.04865122586488724, + 0.04026187211275101, + 0.036113087087869644, + 0.034675728529691696, + 0.034303028136491776, + 0.03043956495821476, + 0.028760986402630806, + 0.027255311608314514, + 0.02610613964498043, + 0.02581332065165043, + 0.024153104051947594, + 0.023376887664198875, + 0.021278444677591324 + ], + "sigma_after": [ + 0.04749403748554257, + 0.047329537727029324, + 0.04941709570844785, + 0.050316546272365655, + 0.04974583191304854, + 0.04882480988120745, + 0.048518907181439176, + 0.04844565102104817, + 0.047917549963051274, + 0.04784127462241485, + 0.04784158409905014, + 0.047869537671348175, + 0.04787829721456915, + 0.0479172416004797, + 0.04791499021870993, + 0.04775522409347927 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.26.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.367933279645062, + "effective_rank_before": 10.367933279645062, + "effective_rank_after": 15.997947242016679, + "nuclear_norm_before_full": 0.8032958991825581, + "nuclear_norm_before": 0.8032958991825581, + "nuclear_norm_after": 0.8032958991825581, + "fro_norm_before_full": 0.32228233271148216, + "fro_norm_after": 0.2008499456959005, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.28717896342277527, + 0.08471272885799408, + 0.05240216106176376, + 0.041367195546627045, + 0.03620263189077377, + 0.033910561352968216, + 0.03238590434193611, + 0.03124130517244339, + 0.030614865943789482, + 0.028649795800447464, + 0.02760379947721958, + 0.02566627413034439, + 0.024763578549027443, + 0.023570900782942772, + 0.022519785910844803, + 0.020505446940660477 + ], + "sigma_after": [ + 0.05017425002438253, + 0.05015420236427724, + 0.052766569288416915, + 0.05101246186059404, + 0.05025077777300892, + 0.05012078875415887, + 0.05010442251919735, + 0.050119559490003525, + 0.050134154210138615, + 0.05018303330239854, + 0.05019174678367533, + 0.050118445489110215, + 0.050021486340288135, + 0.04980670516593581, + 0.049515867685258806, + 0.04862142813171329 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.26.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.819768118028367, + "effective_rank_before": 12.819768118028367, + "effective_rank_after": 15.996798531580074, + "nuclear_norm_before_full": 0.185910914093256, + "nuclear_norm_before": 0.185910914093256, + "nuclear_norm_after": 0.18591091409325594, + "fro_norm_before_full": 0.05891858240285778, + "fro_norm_after": 0.04648703968956386, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04262137785553932, + 0.01830097660422325, + 0.01783665455877781, + 0.015677273273468018, + 0.01230565644800663, + 0.011026495136320591, + 0.009067891165614128, + 0.008464335463941097, + 0.00764915905892849, + 0.00757193099707365, + 0.007004662416875362, + 0.006823863834142685, + 0.006028327159583569, + 0.005923126358538866, + 0.0048985774628818035, + 0.004710606299340725 + ], + "sigma_after": [ + 0.01112723035380754, + 0.011402380441679499, + 0.011451214114736747, + 0.011520832368148676, + 0.011738190193947083, + 0.012006848459836439, + 0.012062600148566149, + 0.011942432409469917, + 0.011735817208286902, + 0.011716707288912057, + 0.011594969683264497, + 0.011565770641464123, + 0.011506041569885649, + 0.011505895699543137, + 0.011523045709715187, + 0.011510937801992355 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.26.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.695072535737445, + "effective_rank_before": 11.695072535737445, + "effective_rank_after": 15.997424522297846, + "nuclear_norm_before_full": 0.4369336059316993, + "nuclear_norm_before": 0.4369336059316993, + "nuclear_norm_after": 0.43693360593169933, + "fro_norm_before_full": 0.15474169521119321, + "fro_norm_after": 0.10925110166993805, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12748226523399353, + 0.047517478466033936, + 0.0335610955953598, + 0.02845514379441738, + 0.02383972890675068, + 0.02259760908782482, + 0.02030443772673607, + 0.018494151532649994, + 0.01726592145860195, + 0.01603309065103531, + 0.01501164585351944, + 0.014732679352164268, + 0.013774310238659382, + 0.013301236554980278, + 0.01253535132855177, + 0.012027460150420666 + ], + "sigma_after": [ + 0.026736536645913522, + 0.026836900394484872, + 0.027363971541163448, + 0.028280626703708688, + 0.028336634525558235, + 0.02812629330457017, + 0.02761424860745909, + 0.02725054248506519, + 0.027093725473039728, + 0.027028552138692957, + 0.027033642239709037, + 0.02704083242935443, + 0.027067767153771533, + 0.027072952805004016, + 0.027049864152543, + 0.027000515331661396 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.26.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.83836137371182, + "effective_rank_before": 11.83836137371182, + "effective_rank_after": 15.997199322295039, + "nuclear_norm_before_full": 0.3575692158192396, + "nuclear_norm_before": 0.3575692158192396, + "nuclear_norm_after": 0.3575692158192396, + "fro_norm_before_full": 0.1247733804209421, + "fro_norm_after": 0.0894080393522444, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1015697717666626, + 0.03813326358795166, + 0.02667887508869171, + 0.02545749582350254, + 0.020420625805854797, + 0.018515275791287422, + 0.018057281151413918, + 0.015548082068562508, + 0.014798569492995739, + 0.013378503732383251, + 0.012887394987046719, + 0.011250804178416729, + 0.011099522933363914, + 0.010503871366381645, + 0.010143673047423363, + 0.009126204997301102 + ], + "sigma_after": [ + 0.021761816887990115, + 0.021967017924420056, + 0.022441768656628072, + 0.02266525076134814, + 0.023284780635866788, + 0.023061240637193278, + 0.022970785503167314, + 0.02240551500415206, + 0.022269710329134745, + 0.022121544973045772, + 0.022105865546478594, + 0.022135211792377756, + 0.022138757932707514, + 0.022140618981254145, + 0.02212628252684596, + 0.021973047726629318 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.26.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.005086252094593, + "effective_rank_before": 9.005086252094593, + "effective_rank_after": 15.82926414953238, + "nuclear_norm_before_full": 0.17326689301989973, + "nuclear_norm_before": 0.17326689301989973, + "nuclear_norm_after": 0.17326689301989975, + "fro_norm_before_full": 0.07249308222295281, + "fro_norm_after": 0.043741365450013815, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.059623029083013535, + 0.03260670229792595, + 0.015552311204373837, + 0.009590217843651772, + 0.008198683150112629, + 0.007312378846108913, + 0.0065537551417946815, + 0.005610478576272726, + 0.005201209336519241, + 0.004354118835180998, + 0.0041276197880506516, + 0.003882152261212468, + 0.003682220820337534, + 0.0027306063566356897, + 0.002279120497405529, + 0.0019622889813035727 + ], + "sigma_after": [ + 0.01162930941222752, + 0.011695675514685873, + 0.01193452802261773, + 0.012035449194352378, + 0.011802306637852786, + 0.01176263568520692, + 0.011778807545319207, + 0.011748719862932522, + 0.01166675774155242, + 0.011238755880848604, + 0.011045689391157219, + 0.010791429909633008, + 0.010547228220748654, + 0.0088834076318509, + 0.007792152641604964, + 0.0069140397273090345 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.27.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.279740745486942, + "effective_rank_before": 10.279740745486942, + "effective_rank_after": 15.994701222438497, + "nuclear_norm_before_full": 0.43714311346411705, + "nuclear_norm_before": 0.43714311346411705, + "nuclear_norm_after": 0.4371431134641171, + "fro_norm_before_full": 0.1728603304299202, + "fro_norm_after": 0.10932186315704025, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.149142324924469, + 0.05814392492175102, + 0.03112492710351944, + 0.024666406214237213, + 0.02319345809519291, + 0.018442433327436447, + 0.01753854937851429, + 0.01593014784157276, + 0.014387434348464012, + 0.013691510073840618, + 0.013149579055607319, + 0.012869296595454216, + 0.01233154721558094, + 0.011590871028602123, + 0.011265360750257969, + 0.009675342589616776 + ], + "sigma_after": [ + 0.02740567482406435, + 0.026676487966407356, + 0.028731838612885793, + 0.028387479313415665, + 0.028083968620442195, + 0.02739854849005412, + 0.027380699330015433, + 0.02741058874583156, + 0.02742222041748266, + 0.027382169052784143, + 0.027316130528468072, + 0.027266839515562604, + 0.02713775233687012, + 0.02687150103088017, + 0.02671672481200445, + 0.025554489866948388 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.27.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.584680395956353, + "effective_rank_before": 12.584680395956353, + "effective_rank_after": 15.99709851705749, + "nuclear_norm_before_full": 0.7622134331613779, + "nuclear_norm_before": 0.7622134331613779, + "nuclear_norm_after": 0.7622134331613778, + "fro_norm_before_full": 0.25058927624062655, + "fro_norm_after": 0.19058796924368662, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.19447995722293854, + 0.07584337890148163, + 0.06792885810136795, + 0.05518932268023491, + 0.040022678673267365, + 0.03634050861001015, + 0.03518588840961456, + 0.03271119296550751, + 0.03125642240047455, + 0.029808679595589638, + 0.029077859595417976, + 0.0284725334495306, + 0.027798203751444817, + 0.02686023712158203, + 0.026330789551138878, + 0.02490692213177681 + ], + "sigma_after": [ + 0.04560518422445081, + 0.04696287444101521, + 0.047263124670340204, + 0.04764765642578478, + 0.049641113233951824, + 0.04905189595935713, + 0.048784295570845716, + 0.04817151113711762, + 0.047835178318228394, + 0.047553606186124316, + 0.047438798758468716, + 0.047359535686259877, + 0.0472889391815186, + 0.047222007081806475, + 0.04719972993593173, + 0.04718798235017647 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.27.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.319051847662854, + "effective_rank_before": 11.319051847662854, + "effective_rank_after": 15.997381929005535, + "nuclear_norm_before_full": 0.7906926963478327, + "nuclear_norm_before": 0.7906926963478327, + "nuclear_norm_after": 0.7906926963478327, + "fro_norm_before_full": 0.2922971746621549, + "fro_norm_after": 0.1977058021408112, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2507834732532501, + 0.0785895511507988, + 0.0533379502594471, + 0.04716600105166435, + 0.04377308860421181, + 0.03646257892251015, + 0.03538624942302704, + 0.03353509679436684, + 0.031969062983989716, + 0.029102977365255356, + 0.028721259906888008, + 0.026267394423484802, + 0.02581098861992359, + 0.024129081517457962, + 0.023918649181723595, + 0.02173929288983345 + ], + "sigma_after": [ + 0.04888689913816496, + 0.048949244224397115, + 0.051200147599346775, + 0.05145217398307506, + 0.05105908250893572, + 0.04954885229268749, + 0.04935491846233666, + 0.04909116411146708, + 0.04894888774626552, + 0.04887720358424108, + 0.04888237645773095, + 0.048945796837053114, + 0.04895491844347421, + 0.048941806372625275, + 0.04893237478050765, + 0.048666849805524055 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.27.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.779812236453138, + "effective_rank_before": 12.779812236453138, + "effective_rank_after": 15.996122114659247, + "nuclear_norm_before_full": 0.20459374599158764, + "nuclear_norm_before": 0.20459374599158764, + "nuclear_norm_after": 0.20459374599158764, + "fro_norm_before_full": 0.06327581351270158, + "fro_norm_after": 0.051160867071019384, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03706628456711769, + 0.030607610940933228, + 0.022445229813456535, + 0.0161418579518795, + 0.014843298122286797, + 0.012708181515336037, + 0.011245425790548325, + 0.009596483781933784, + 0.009123031981289387, + 0.007440525107085705, + 0.006734780035912991, + 0.00620471965521574, + 0.005835417192429304, + 0.00540510518476367, + 0.00503170071169734, + 0.004164093639701605 + ], + "sigma_after": [ + 0.012706252582138713, + 0.012706211257404287, + 0.012245617896114681, + 0.012693098654915063, + 0.012715381729058462, + 0.013056899250862099, + 0.013334286154582283, + 0.013279584203718555, + 0.013179023264055793, + 0.012773564478712562, + 0.012697051526617098, + 0.012691836840593764, + 0.012703280428764618, + 0.012711735525113404, + 0.012692457146840133, + 0.012407465052096115 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.27.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.765275219068277, + "effective_rank_before": 10.765275219068277, + "effective_rank_after": 15.997375803963305, + "nuclear_norm_before_full": 0.4573626760393381, + "nuclear_norm_before": 0.4573626760393381, + "nuclear_norm_after": 0.45736267603933806, + "fro_norm_before_full": 0.17292660553548278, + "fro_norm_after": 0.11435953546226592, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14543886482715607, + 0.05765718221664429, + 0.03956267982721329, + 0.02697358839213848, + 0.023497987538576126, + 0.021554280072450638, + 0.020435767248272896, + 0.018131723627448082, + 0.015801314264535904, + 0.014261678792536259, + 0.01349551323801279, + 0.013235661201179028, + 0.01242812443524599, + 0.011920824646949768, + 0.011790893971920013, + 0.011176591739058495 + ], + "sigma_after": [ + 0.028453667910245936, + 0.027882055894119916, + 0.02868922567055692, + 0.029989189015167984, + 0.02936188700572015, + 0.028954071584541673, + 0.02876674845696554, + 0.02857121674965684, + 0.028598299170779613, + 0.028602065439061106, + 0.028543607415918163, + 0.028508728192350192, + 0.028336559021794, + 0.028168647285956083, + 0.028117026175411373, + 0.027819681051092567 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.27.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.508845192034249, + "effective_rank_before": 10.508845192034249, + "effective_rank_after": 15.992650384869886, + "nuclear_norm_before_full": 0.4050183743238449, + "nuclear_norm_before": 0.4050183743238449, + "nuclear_norm_after": 0.4050183743238449, + "fro_norm_before_full": 0.15778372072343738, + "fro_norm_after": 0.10130022779369292, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.13655658066272736, + 0.04786404222249985, + 0.03078893944621086, + 0.02230735681951046, + 0.021399611607193947, + 0.01959408074617386, + 0.01711934432387352, + 0.015903767198324203, + 0.014362598769366741, + 0.013654015026986599, + 0.013486681506037712, + 0.012081068009138107, + 0.010966180823743343, + 0.01064309012144804, + 0.01019761897623539, + 0.008093398064374924 + ], + "sigma_after": [ + 0.02541366968270331, + 0.025255406682546007, + 0.026279773251366163, + 0.026275038570765838, + 0.026084374647087606, + 0.025717380786454986, + 0.025413752467516464, + 0.025383526235541003, + 0.02541593486002581, + 0.025427912393797526, + 0.02542782624150278, + 0.025335546881949986, + 0.025065066101291788, + 0.024938971872331288, + 0.024722417651978114, + 0.0228617759969862 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.27.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.394023950196082, + "effective_rank_before": 8.394023950196082, + "effective_rank_after": 15.866252940469309, + "nuclear_norm_before_full": 0.18162844865582883, + "nuclear_norm_before": 0.18162844865582883, + "nuclear_norm_after": 0.18162844865582883, + "fro_norm_before_full": 0.08336517382429294, + "fro_norm_after": 0.04576565034720564, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.0763373076915741, + 0.020719364285469055, + 0.016535380855202675, + 0.010487773455679417, + 0.008369033224880695, + 0.006971183232963085, + 0.006367655470967293, + 0.0053716739639639854, + 0.005016019567847252, + 0.004743844736367464, + 0.00434010848402977, + 0.004001718480139971, + 0.003622883465141058, + 0.003479917999356985, + 0.0027203827630728483, + 0.0025442009791731834 + ], + "sigma_after": [ + 0.012394191332321822, + 0.01239263886162267, + 0.012785984106609851, + 0.012578887696487323, + 0.01239281447544653, + 0.012412253004914322, + 0.012367146656547447, + 0.012061668157378188, + 0.011846980530953056, + 0.0116349077291259, + 0.011234889515750981, + 0.010813990724193838, + 0.010243719531687305, + 0.010000391054112549, + 0.008445410753405916, + 0.008022574525271154 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.28.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.67280486578406, + "effective_rank_before": 9.67280486578406, + "effective_rank_after": 15.991189256649191, + "nuclear_norm_before_full": 0.4376162439584732, + "nuclear_norm_before": 0.4376162439584732, + "nuclear_norm_after": 0.4376162439584732, + "fro_norm_before_full": 0.18506710220667477, + "fro_norm_after": 0.10946402634869455, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.16792255640029907, + 0.0416608490049839, + 0.03205129876732826, + 0.029851820319890976, + 0.0204954631626606, + 0.01769375242292881, + 0.0156702920794487, + 0.014217326417565346, + 0.013911965303122997, + 0.013723522424697876, + 0.013287292793393135, + 0.01286234986037016, + 0.012186573818325996, + 0.01157219335436821, + 0.010686478577554226, + 0.009822509251534939 + ], + "sigma_after": [ + 0.027556226161904313, + 0.027720536396849894, + 0.028999076838052067, + 0.02899488587177716, + 0.027601195853462088, + 0.02755688427427452, + 0.02758988992242468, + 0.02749728026228852, + 0.027451961555400625, + 0.027418160974719868, + 0.027320863059325785, + 0.02719747871562016, + 0.02693410596099888, + 0.02661249200323372, + 0.025989436448734905, + 0.02517576965940598 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.28.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.11287602443397, + "effective_rank_before": 12.11287602443397, + "effective_rank_after": 15.997316543880324, + "nuclear_norm_before_full": 0.7769927438348532, + "nuclear_norm_before": 0.7769927438348532, + "nuclear_norm_after": 0.7769927438348532, + "fro_norm_before_full": 0.26904908054778764, + "fro_norm_after": 0.19428101338108983, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.22242015600204468, + 0.07080603390932083, + 0.06009207293391228, + 0.04425674304366112, + 0.04268236458301544, + 0.03972253575921059, + 0.03405112400650978, + 0.0333421565592289, + 0.03308125212788582, + 0.032297682017087936, + 0.0309161227196455, + 0.028807392343878746, + 0.027925541624426842, + 0.026628689840435982, + 0.02607206255197525, + 0.023890813812613487 + ], + "sigma_after": [ + 0.047500976104489945, + 0.0480012126967833, + 0.048314890746551944, + 0.05052115184259554, + 0.050422081319144406, + 0.04998996111998304, + 0.048715400743394456, + 0.04856856563680225, + 0.048517438351409155, + 0.0483748548466751, + 0.04816995819254569, + 0.04798761999611206, + 0.047957969757012275, + 0.0479560575754134, + 0.0479667643351502, + 0.0480278405707904 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.28.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.973474053779547, + "effective_rank_before": 9.973474053779547, + "effective_rank_after": 15.995876007278786, + "nuclear_norm_before_full": 0.8250306881964207, + "nuclear_norm_before": 0.8250306881964207, + "nuclear_norm_after": 0.8250306881964207, + "fro_norm_before_full": 0.3424657485191061, + "fro_norm_after": 0.20631074526082888, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.30994412302970886, + 0.07878975570201874, + 0.05959087610244751, + 0.04333119094371796, + 0.037309110164642334, + 0.03426467627286911, + 0.032964084297418594, + 0.030613668262958527, + 0.028913350775837898, + 0.028371546417474747, + 0.026614151895046234, + 0.02624218352138996, + 0.024466920644044876, + 0.023062385618686676, + 0.020734092220664024, + 0.01981857232749462 + ], + "sigma_after": [ + 0.05175092187759145, + 0.05191563928740336, + 0.05443863714918674, + 0.05254120839476426, + 0.05178948980384692, + 0.051720830531247014, + 0.05174054528837788, + 0.05179802765278491, + 0.051806458443084104, + 0.0517939506411485, + 0.05167123785559311, + 0.05162469276133064, + 0.05127035013334539, + 0.050798393926633476, + 0.04953215857027415, + 0.04883814587980876 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.28.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.395131808018721, + "effective_rank_before": 11.395131808018721, + "effective_rank_after": 15.990222308549278, + "nuclear_norm_before_full": 0.17844510055147111, + "nuclear_norm_before": 0.17844510055147111, + "nuclear_norm_after": 0.17844510055147111, + "fro_norm_before_full": 0.06205245617543212, + "fro_norm_after": 0.04463846046639093, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04575488716363907, + 0.024231450632214546, + 0.02319653145968914, + 0.011205664835870266, + 0.01031301449984312, + 0.008543585427105427, + 0.007939290255308151, + 0.007242605555802584, + 0.00662099989131093, + 0.006422119680792093, + 0.005971051286906004, + 0.005305360537022352, + 0.0046043312177062035, + 0.003980019595474005, + 0.0038038850761950016, + 0.0033103034365922213 + ], + "sigma_after": [ + 0.010813207754462623, + 0.01083594610333967, + 0.010799021737417503, + 0.011776624315284008, + 0.011830693793445514, + 0.011567299777127288, + 0.011424513313965186, + 0.011293009626705928, + 0.011234043065197333, + 0.01122785166187122, + 0.011231317957471341, + 0.011246332558473262, + 0.011176493436329676, + 0.010915276795423427, + 0.010789926220370734, + 0.010283542434586401 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.28.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.16284533796767, + "effective_rank_before": 12.16284533796767, + "effective_rank_after": 15.996264424942819, + "nuclear_norm_before_full": 0.4146492248401046, + "nuclear_norm_before": 0.4146492248401046, + "nuclear_norm_after": 0.4146492248401046, + "fro_norm_before_full": 0.13781204106603512, + "fro_norm_after": 0.10368656933433233, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09977427870035172, + 0.060105204582214355, + 0.040396202355623245, + 0.026319613680243492, + 0.022257013246417046, + 0.020240498706698418, + 0.019213935360312462, + 0.018134744837880135, + 0.016167741268873215, + 0.015863409265875816, + 0.014811122789978981, + 0.014179372228682041, + 0.013094807043671608, + 0.012895496562123299, + 0.011332198977470398, + 0.009863585233688354 + ], + "sigma_after": [ + 0.024825034289088013, + 0.025377029974240878, + 0.025651982531653537, + 0.0266959303089225, + 0.027030598564067074, + 0.026745138987757783, + 0.026512645810294475, + 0.026247211671292018, + 0.025839407949561827, + 0.025795822423555326, + 0.025697483518225938, + 0.02567664223286295, + 0.025690658527573112, + 0.025696966618460913, + 0.0257084719961333, + 0.02545819943641497 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.28.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.448095819143596, + "effective_rank_before": 11.448095819143596, + "effective_rank_after": 15.995929470864423, + "nuclear_norm_before_full": 0.37298876233398914, + "nuclear_norm_before": 0.37298876233398914, + "nuclear_norm_after": 0.37298876233398925, + "fro_norm_before_full": 0.1339132078615917, + "fro_norm_after": 0.09327097488494271, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1091146394610405, + 0.04937012493610382, + 0.025641201063990593, + 0.022501805797219276, + 0.01987610198557377, + 0.01874103955924511, + 0.016189059242606163, + 0.015590103343129158, + 0.014621381647884846, + 0.014096761122345924, + 0.013045978732407093, + 0.01189293060451746, + 0.011709479615092278, + 0.010830474086105824, + 0.010178392753005028, + 0.009589288383722305 + ], + "sigma_after": [ + 0.02277888393300556, + 0.022248770717759624, + 0.02403524986118383, + 0.024372786056847295, + 0.024130129674684194, + 0.023897836260901344, + 0.0233510572031009, + 0.023258922943313273, + 0.02316117727983322, + 0.023135728958321364, + 0.02313354327163692, + 0.023164488932362066, + 0.02316732300334975, + 0.023147456948784107, + 0.02307175333543347, + 0.022933653953472292 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.28.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.397910867103356, + "effective_rank_before": 8.397910867103356, + "effective_rank_after": 15.882031826454819, + "nuclear_norm_before_full": 0.16646874602884054, + "nuclear_norm_before": 0.16646874602884054, + "nuclear_norm_after": 0.16646874602884054, + "fro_norm_before_full": 0.07733014700296319, + "fro_norm_after": 0.04190985715848456, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07178530097007751, + 0.015848198905587196, + 0.013073191046714783, + 0.010715721175074577, + 0.008815801702439785, + 0.0069938297383487225, + 0.005793334450572729, + 0.0049721477553248405, + 0.004616988822817802, + 0.004160603042691946, + 0.003960857633501291, + 0.0038170809857547283, + 0.0034692902117967606, + 0.0031222093384712934, + 0.002946194726973772, + 0.0023779955226927996 + ], + "sigma_after": [ + 0.01125886040574476, + 0.011532836533608384, + 0.011863562299345326, + 0.01161296325436691, + 0.011302645272935553, + 0.011273706294899172, + 0.011217778535350494, + 0.010952208427007814, + 0.010737552239567407, + 0.01034838706514409, + 0.010133257039452382, + 0.009960331252505077, + 0.009476680187812007, + 0.008898469964640426, + 0.008568031885207833, + 0.007331475371252914 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.29.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.094566996571249, + "effective_rank_before": 9.094566996571249, + "effective_rank_after": 15.991343929805065, + "nuclear_norm_before_full": 0.4304192466661334, + "nuclear_norm_before": 0.4304192466661334, + "nuclear_norm_after": 0.4304192466661334, + "fro_norm_before_full": 0.19325019099554766, + "fro_norm_after": 0.10766259806285056, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1797465980052948, + 0.039552658796310425, + 0.027560435235500336, + 0.020665636286139488, + 0.018358448520302773, + 0.017322983592748642, + 0.01563047058880329, + 0.01494307816028595, + 0.014209822751581669, + 0.013481011614203453, + 0.013079170137643814, + 0.012052402831614017, + 0.011541328392922878, + 0.011321621015667915, + 0.010724607855081558, + 0.010228972882032394 + ], + "sigma_after": [ + 0.027347521786279916, + 0.028019748034760424, + 0.028347469443573532, + 0.027364297234859725, + 0.027361211785789024, + 0.027385363861679588, + 0.02736809287208563, + 0.027310930241121627, + 0.027197920415280276, + 0.02701702231023886, + 0.02688223846935982, + 0.026403088498919255, + 0.026082279774357287, + 0.025925762602194275, + 0.025440148202932677, + 0.024966151132701512 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.29.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.945589497707239, + "effective_rank_before": 12.945589497707239, + "effective_rank_after": 15.993991202013426, + "nuclear_norm_before_full": 0.7302453406155109, + "nuclear_norm_before": 0.7302453406155109, + "nuclear_norm_after": 0.7302453406155108, + "fro_norm_before_full": 0.23378617184580217, + "fro_norm_after": 0.1826294429973905, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.17653395235538483, + 0.08476229757070541, + 0.05092020332813263, + 0.040630586445331573, + 0.0377761647105217, + 0.03648727759718895, + 0.035070713609457016, + 0.034550976008176804, + 0.03373674675822258, + 0.03225647285580635, + 0.0314846895635128, + 0.029530569911003113, + 0.027760395780205727, + 0.027305200695991516, + 0.02594541572034359, + 0.025493677705526352 + ], + "sigma_after": [ + 0.04307709325752457, + 0.04311291244784174, + 0.04530349900933597, + 0.04713527867890989, + 0.04714254435116608, + 0.047015430975279694, + 0.04679013624738283, + 0.04668814051068296, + 0.04651146282611065, + 0.04615320855896299, + 0.04595729942036002, + 0.045477574789406804, + 0.045118792018853715, + 0.04504483424802001, + 0.044877208592809685, + 0.044839924682863255 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.29.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.525931430315872, + "effective_rank_before": 11.525931430315872, + "effective_rank_after": 15.997783194876158, + "nuclear_norm_before_full": 0.772248363122344, + "nuclear_norm_before": 0.772248363122344, + "nuclear_norm_after": 0.772248363122344, + "fro_norm_before_full": 0.27794423740800533, + "fro_norm_after": 0.19308905475602542, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.23095454275608063, + 0.0898187980055809, + 0.05847269296646118, + 0.04497678577899933, + 0.04071522504091263, + 0.034555837512016296, + 0.03250938281416893, + 0.032356228679418564, + 0.030078444629907608, + 0.028548412024974823, + 0.026962557807564735, + 0.026517262682318687, + 0.0255780890583992, + 0.023946169763803482, + 0.023662103340029716, + 0.022595830261707306 + ], + "sigma_after": [ + 0.047540779217277035, + 0.04708826293118806, + 0.048796623291401046, + 0.050437640821083415, + 0.04987934614263547, + 0.048541603966463016, + 0.048196056259805116, + 0.04817510786060557, + 0.047956254315523944, + 0.04790567174334527, + 0.04791817260077757, + 0.047929382079337494, + 0.04795700561763518, + 0.047987829288884135, + 0.04798671311698076, + 0.04795191386940088 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.29.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.422059103712163, + "effective_rank_before": 12.422059103712163, + "effective_rank_after": 15.994349684865785, + "nuclear_norm_before_full": 0.18408340634778142, + "nuclear_norm_before": 0.18408340634778142, + "nuclear_norm_after": 0.18408340634778142, + "fro_norm_before_full": 0.06038048590853884, + "fro_norm_after": 0.046037122713009235, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04544966667890549, + 0.020907824859023094, + 0.016363516449928284, + 0.013056529685854912, + 0.011193472892045975, + 0.010201046243309975, + 0.009649481624364853, + 0.009032854810357094, + 0.007975507527589798, + 0.006892555858939886, + 0.006475066300481558, + 0.006303821690380573, + 0.006053844466805458, + 0.005419560708105564, + 0.005007525905966759, + 0.004101130645722151 + ], + "sigma_after": [ + 0.01096041446830523, + 0.011037396151055012, + 0.011413289107752782, + 0.011543581984171372, + 0.011909040600867369, + 0.012007203336773974, + 0.011988085654903497, + 0.011901963103091726, + 0.011655709776313024, + 0.011440500036782972, + 0.011403555744749726, + 0.011396838606935734, + 0.011394668315469448, + 0.011411219579632433, + 0.011411084124432778, + 0.011208855756544353 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.29.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.080775747431382, + "effective_rank_before": 10.080775747431382, + "effective_rank_after": 15.99440134640591, + "nuclear_norm_before_full": 0.432346279732883, + "nuclear_norm_before": 0.432346279732883, + "nuclear_norm_after": 0.432346279732883, + "fro_norm_before_full": 0.171414956658969, + "fro_norm_after": 0.10812427724350245, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14505447447299957, + 0.0636436715722084, + 0.03706102818250656, + 0.025159761309623718, + 0.019231650978326797, + 0.018082810565829277, + 0.016492119058966637, + 0.014994412660598755, + 0.01433456502854824, + 0.013420186005532742, + 0.012136095203459263, + 0.011651413515210152, + 0.010934503749012947, + 0.01039301697164774, + 0.010201572440564632, + 0.009554998017847538 + ], + "sigma_after": [ + 0.027286301293677387, + 0.026306482736109887, + 0.02770767073530815, + 0.028485808300333162, + 0.027427082546360306, + 0.027359597607360942, + 0.027360334227066643, + 0.02739576661511637, + 0.02739229110395286, + 0.027334844282262995, + 0.027078907607239446, + 0.026907036247618348, + 0.026556164651051553, + 0.026203287305812707, + 0.026058553630289572, + 0.025486150843322668 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.29.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.118668113266088, + "effective_rank_before": 13.118668113266088, + "effective_rank_after": 15.99707075802357, + "nuclear_norm_before_full": 0.39544168021529913, + "nuclear_norm_before": 0.39544168021529913, + "nuclear_norm_after": 0.39544168021529913, + "fro_norm_before_full": 0.12178428447416254, + "fro_norm_after": 0.09887857265309054, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07432665675878525, + 0.06465285271406174, + 0.030465606600046158, + 0.026463795453310013, + 0.024536751210689545, + 0.021306326612830162, + 0.019320806488394737, + 0.018708346411585808, + 0.017644250765442848, + 0.017326664179563522, + 0.0164632648229599, + 0.014690469019114971, + 0.014478267170488834, + 0.012533938512206078, + 0.011717735789716244, + 0.010805947706103325 + ], + "sigma_after": [ + 0.024277378554131812, + 0.024262825855618897, + 0.024235246998166034, + 0.02452976388174319, + 0.024918976456916663, + 0.025499686881569582, + 0.025483481559304536, + 0.0254020852295706, + 0.025189711563882375, + 0.025113229890372884, + 0.02488976534212078, + 0.02446244972556721, + 0.024422584571457247, + 0.024236152450368184, + 0.024244812156856563, + 0.024273529097652586 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.29.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.376751145914671, + "effective_rank_before": 9.376751145914671, + "effective_rank_after": 15.927175569469764, + "nuclear_norm_before_full": 0.19875549688003957, + "nuclear_norm_before": 0.19875549688003957, + "nuclear_norm_after": 0.19875549688003957, + "fro_norm_before_full": 0.08284527066136342, + "fro_norm_after": 0.04990022237262514, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07201088964939117, + 0.026950053870677948, + 0.01897367462515831, + 0.01243339292705059, + 0.010122212581336498, + 0.008154371753334999, + 0.007114733103662729, + 0.00652990210801363, + 0.00606348505243659, + 0.0054786428809165955, + 0.0053781382739543915, + 0.0047982437536120415, + 0.004342303145676851, + 0.004042237997055054, + 0.003678701352328062, + 0.0026845138054341078 + ], + "sigma_after": [ + 0.013060045941872163, + 0.012799883396436392, + 0.013096631528594546, + 0.013627724566675454, + 0.013183862410673281, + 0.013042571660217725, + 0.013063639343646079, + 0.013038450941429315, + 0.012964387773249524, + 0.012763584990672249, + 0.012713697836062033, + 0.012317628917410707, + 0.01185697278860767, + 0.011472492520958319, + 0.010913736533867048, + 0.008840185729667067 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.3.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.758509684586645, + "effective_rank_before": 11.758509684586645, + "effective_rank_after": 15.997721937383513, + "nuclear_norm_before_full": 0.37200422398746014, + "nuclear_norm_before": 0.37200422398746014, + "nuclear_norm_after": 0.37200422398746014, + "fro_norm_before_full": 0.12752219103054244, + "fro_norm_after": 0.09301427263126862, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09624554961919785, + 0.04878643900156021, + 0.03898995742201805, + 0.0314609631896019, + 0.01851677894592285, + 0.015484626404941082, + 0.014407576061785221, + 0.01417934987694025, + 0.01382696907967329, + 0.013120407238602638, + 0.012544330209493637, + 0.011995126493275166, + 0.011634781956672668, + 0.011019342578947544, + 0.01008463092148304, + 0.009707394987344742 + ], + "sigma_after": [ + 0.022398725215772436, + 0.022421786947949336, + 0.023137596371076388, + 0.02328412824019852, + 0.024209699806680227, + 0.02352085176212984, + 0.023360458871691976, + 0.023337727926219957, + 0.023310718519706224, + 0.02328473479958518, + 0.023286910146613566, + 0.023301194725282432, + 0.023312499090715238, + 0.023324338840960992, + 0.023283067773438117, + 0.02322978494943968 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.3.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.841821588711175, + "effective_rank_before": 8.841821588711175, + "effective_rank_after": 15.86734572280321, + "nuclear_norm_before_full": 0.3612936167046428, + "nuclear_norm_before": 0.3612936167046428, + "nuclear_norm_after": 0.36129361670464283, + "fro_norm_before_full": 0.15965553448262446, + "fro_norm_after": 0.09101943705790244, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14453697204589844, + 0.04113354906439781, + 0.0317331962287426, + 0.022969895973801613, + 0.01756025105714798, + 0.013854437507689, + 0.012947994284331799, + 0.012595116160809994, + 0.011142421513795853, + 0.009651685133576393, + 0.00899273157119751, + 0.00889496598392725, + 0.00857835914939642, + 0.006688285619020462, + 0.005106963217258453, + 0.004906792193651199 + ], + "sigma_after": [ + 0.024202171213572057, + 0.024199751648193327, + 0.024944375054149977, + 0.025111873241191098, + 0.024230709080652463, + 0.02423002656075745, + 0.024210491208052957, + 0.02418461792127771, + 0.023899557954202504, + 0.02315481574236281, + 0.022626406424000996, + 0.02253601473296826, + 0.022220823347389805, + 0.019561090080078246, + 0.016236596974615436, + 0.015744295521177707 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.3.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.851346825733702, + "effective_rank_before": 7.851346825733702, + "effective_rank_after": 15.500135955388872, + "nuclear_norm_before_full": 0.3934992514550686, + "nuclear_norm_before": 0.3934992514550686, + "nuclear_norm_after": 0.3934992514550686, + "fro_norm_before_full": 0.1818271723840356, + "fro_norm_after": 0.10114590626365007, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.16118812561035156, + 0.06327198445796967, + 0.03600575029850006, + 0.019896971061825752, + 0.01887139305472374, + 0.017228351905941963, + 0.01647862419486046, + 0.010459644719958305, + 0.009129188023507595, + 0.00868690200150013, + 0.008252103812992573, + 0.006736667826771736, + 0.005420523229986429, + 0.004661480896174908, + 0.004138479009270668, + 0.0030730613507330418 + ], + "sigma_after": [ + 0.029422570466808823, + 0.02843039024281166, + 0.030318591029103205, + 0.029420743175445067, + 0.0293793796592943, + 0.02939439211776122, + 0.029414927178196, + 0.02768300805614629, + 0.02619945424461527, + 0.02557198202803036, + 0.024885717018360863, + 0.02193658264125559, + 0.018669241431654007, + 0.016500920812351716, + 0.014895541623243597, + 0.011375809729990579 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.3.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.037897609454102, + "effective_rank_before": 13.037897609454102, + "effective_rank_after": 15.99719403844936, + "nuclear_norm_before_full": 0.19133825274184346, + "nuclear_norm_before": 0.19133825274184346, + "nuclear_norm_after": 0.19133825274184346, + "fro_norm_before_full": 0.05900024258048931, + "fro_norm_after": 0.04784300306248118, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.037294697016477585, + 0.02770587056875229, + 0.018330058082938194, + 0.014574927277863026, + 0.012064962647855282, + 0.010613040067255497, + 0.009418314322829247, + 0.008811579085886478, + 0.007717924192547798, + 0.007579195313155651, + 0.006844186689704657, + 0.006702117621898651, + 0.006419104058295488, + 0.0060456241481006145, + 0.0058511774986982346, + 0.00536547414958477 + ], + "sigma_after": [ + 0.011811997957781588, + 0.011797141180715589, + 0.011694680678264631, + 0.011801400528216476, + 0.012094847260498446, + 0.012384937981353953, + 0.012415011159966058, + 0.012323835506948808, + 0.012052106114788089, + 0.0120167064740194, + 0.011864340826034214, + 0.011844627608083752, + 0.011816532056703839, + 0.011801524705457582, + 0.011802108658109463, + 0.011816454044901581 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.3.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.032177196903737, + "effective_rank_before": 9.032177196903737, + "effective_rank_after": 15.956582747024054, + "nuclear_norm_before_full": 0.42406582459807396, + "nuclear_norm_before": 0.42406582459807396, + "nuclear_norm_after": 0.42406582459807396, + "fro_norm_before_full": 0.1847281421880677, + "fro_norm_after": 0.10629860583875823, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.16570335626602173, + 0.05555950105190277, + 0.029811406508088112, + 0.025881707668304443, + 0.02188602276146412, + 0.019335512071847916, + 0.01503389235585928, + 0.012730699963867664, + 0.01176160853356123, + 0.011624876409769058, + 0.011216163635253906, + 0.009741120971739292, + 0.009209844283759594, + 0.008563940413296223, + 0.008188351057469845, + 0.007817820645868778 + ], + "sigma_after": [ + 0.02777853216006035, + 0.027642436701873902, + 0.02921214891170835, + 0.028678208147995102, + 0.02795350408772634, + 0.027755451110506958, + 0.027777399814857735, + 0.02736386583197076, + 0.02693205625467355, + 0.026854766750114684, + 0.026597034142511213, + 0.025294720925951426, + 0.024666819881703547, + 0.02377987125251832, + 0.02319954593872329, + 0.022579462685178405 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.3.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.172645055052081, + "effective_rank_before": 10.172645055052081, + "effective_rank_after": 15.995941277943729, + "nuclear_norm_before_full": 0.3086486803367734, + "nuclear_norm_before": 0.3086486803367734, + "nuclear_norm_after": 0.3086486803367734, + "fro_norm_before_full": 0.12621729765979484, + "fro_norm_after": 0.07718172170899884, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11390774697065353, + 0.027803970500826836, + 0.020456327125430107, + 0.017429282888770103, + 0.015126189216971397, + 0.01433140691369772, + 0.01350429654121399, + 0.012507522478699684, + 0.011236614547669888, + 0.010379151441156864, + 0.009634261950850487, + 0.009312608279287815, + 0.008883862756192684, + 0.008503674529492855, + 0.008359459228813648, + 0.007272304967045784 + ], + "sigma_after": [ + 0.019309055117730248, + 0.01948294541380764, + 0.020316727235063812, + 0.019890167532381528, + 0.019461092932327825, + 0.019364910188420743, + 0.019308572757765066, + 0.0192961000257479, + 0.019324657156022804, + 0.0193194474651631, + 0.019253827357563254, + 0.01919756899504938, + 0.01908825705480682, + 0.018952475322277368, + 0.01889016225760825, + 0.018192713525037676 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.3.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 6.020906361453591, + "effective_rank_before": 6.020906361453591, + "effective_rank_after": 15.369864389152609, + "nuclear_norm_before_full": 0.1853878095280379, + "nuclear_norm_before": 0.1853878095280379, + "nuclear_norm_after": 0.1853878095280379, + "fro_norm_before_full": 0.10700834807011537, + "fro_norm_after": 0.048083021446551834, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10412739962339401, + 0.013024860061705112, + 0.01036120392382145, + 0.009633918292820454, + 0.008592403493821621, + 0.006976655218750238, + 0.004760276060551405, + 0.0045044333674013615, + 0.004108319524675608, + 0.0037831347435712814, + 0.003353530541062355, + 0.0031543055083602667, + 0.0026930593885481358, + 0.0023197128903120756, + 0.0020510817412286997, + 0.0019435151480138302 + ], + "sigma_after": [ + 0.01514991929509664, + 0.015218783049555528, + 0.015065680772831448, + 0.01508231939335806, + 0.01506992403117142, + 0.014696850288466557, + 0.012617231242991779, + 0.012215542404340723, + 0.011520795622796072, + 0.010884113393125285, + 0.00995253103736778, + 0.009486465503450755, + 0.008328552290619495, + 0.007315848143433938, + 0.0065492178066232165, + 0.006234035252809206 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.30.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.035450225063277, + "effective_rank_before": 9.035450225063277, + "effective_rank_after": 15.977837365000527, + "nuclear_norm_before_full": 0.43899570778012276, + "nuclear_norm_before": 0.43899570778012276, + "nuclear_norm_after": 0.43899570778012276, + "fro_norm_before_full": 0.19622334238531638, + "fro_norm_after": 0.10989846304677055, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.18134863674640656, + 0.042881373316049576, + 0.03132861480116844, + 0.02325967513024807, + 0.019173333421349525, + 0.017467370256781578, + 0.016405047848820686, + 0.015752702951431274, + 0.013987039215862751, + 0.012967769056558609, + 0.012647605501115322, + 0.012005318887531757, + 0.010815362446010113, + 0.010261272080242634, + 0.009530107490718365, + 0.0091644786298275 + ], + "sigma_after": [ + 0.02822669742344039, + 0.028541355272732247, + 0.029695217819960096, + 0.02843661694626204, + 0.028232804505668757, + 0.028270866681420766, + 0.028271260963878993, + 0.02824366270930416, + 0.027973316913902427, + 0.02762305990685835, + 0.02747472714163873, + 0.027113829024869314, + 0.026192784999240954, + 0.025639025335778096, + 0.024775680111047887, + 0.024284802024119558 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.30.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.989850630251784, + "effective_rank_before": 10.989850630251784, + "effective_rank_after": 15.997899311168219, + "nuclear_norm_before_full": 0.7959231100976467, + "nuclear_norm_before": 0.7959231100976467, + "nuclear_norm_after": 0.7959231100976467, + "fro_norm_before_full": 0.3027344040398882, + "fro_norm_after": 0.19900716316573663, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.26342812180519104, + 0.08181969821453094, + 0.05328580364584923, + 0.04581499844789505, + 0.03956034407019615, + 0.03663180395960808, + 0.03326177969574928, + 0.03231068328022957, + 0.030165022239089012, + 0.029472149908542633, + 0.028036531060934067, + 0.027367644011974335, + 0.025010481476783752, + 0.02449846640229225, + 0.02292032539844513, + 0.02233925648033619 + ], + "sigma_after": [ + 0.04941543270123139, + 0.0494192138310539, + 0.051890800095900365, + 0.05161918277884087, + 0.050380069623156665, + 0.049821496651902296, + 0.049424627630714646, + 0.04937227626823256, + 0.04934195488519809, + 0.049352410862553604, + 0.04938959817634896, + 0.0494075783126634, + 0.04940815067883779, + 0.04938258541641537, + 0.049205311524028406, + 0.0490924206605684 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.30.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.562411298189156, + "effective_rank_before": 9.562411298189156, + "effective_rank_after": 15.997210239334867, + "nuclear_norm_before_full": 0.8397932816296816, + "nuclear_norm_before": 0.8397932816296816, + "nuclear_norm_after": 0.8397932816296816, + "fro_norm_before_full": 0.3595439108417402, + "fro_norm_after": 0.20998460407091962, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.3266907334327698, + 0.0970701351761818, + 0.04851396754384041, + 0.04036979004740715, + 0.035968344658613205, + 0.032881852239370346, + 0.031288888305425644, + 0.029170596972107887, + 0.028507409617304802, + 0.02687068283557892, + 0.025666870176792145, + 0.024834347888827324, + 0.024239659309387207, + 0.024067580699920654, + 0.023306161165237427, + 0.0203462615609169 + ], + "sigma_after": [ + 0.05289033971473445, + 0.05295273939757576, + 0.05427943016701126, + 0.05302178016860237, + 0.05286974584545112, + 0.052932306582176174, + 0.05296003017584019, + 0.052918497176280666, + 0.05287361594237706, + 0.05266689255482624, + 0.05240390714860252, + 0.052154318266671856, + 0.05193751761703189, + 0.05186836231045802, + 0.05152554851214358, + 0.04953825004989845 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.30.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.07875933189293, + "effective_rank_before": 10.07875933189293, + "effective_rank_after": 15.980739008717407, + "nuclear_norm_before_full": 0.17569199413992465, + "nuclear_norm_before": 0.17569199413992465, + "nuclear_norm_after": 0.17569199413992465, + "fro_norm_before_full": 0.0698179441997393, + "fro_norm_after": 0.04397444966204412, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.059911035001277924, + 0.022619495168328285, + 0.01646687090396881, + 0.009640473872423172, + 0.008919989690184593, + 0.008160012774169445, + 0.006667570676654577, + 0.006261559668928385, + 0.006034515798091888, + 0.00586957111954689, + 0.005211625248193741, + 0.0050693172961473465, + 0.0042740413919091225, + 0.0039769988507032394, + 0.003376319073140621, + 0.003232597606256604 + ], + "sigma_after": [ + 0.011218220727042913, + 0.011018073271838461, + 0.011233651883213343, + 0.011561299009901405, + 0.011410242802308948, + 0.011282846846405693, + 0.01122104147437858, + 0.011231785191088214, + 0.011234855648567207, + 0.011233610888091189, + 0.011171247753278232, + 0.011139656423608806, + 0.010774739240176565, + 0.010533200131035269, + 0.009822952270387843, + 0.009604570578601986 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.30.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.68758576327115, + "effective_rank_before": 8.68758576327115, + "effective_rank_after": 15.952251513457615, + "nuclear_norm_before_full": 0.45808779262006283, + "nuclear_norm_before": 0.45808779262006283, + "nuclear_norm_after": 0.45808779262006283, + "fro_norm_before_full": 0.20965585169724044, + "fro_norm_after": 0.11485563879169489, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.19439974427223206, + 0.04799575358629227, + 0.030235014855861664, + 0.024332404136657715, + 0.021310707554221153, + 0.01884414441883564, + 0.017794789746403694, + 0.015157523564994335, + 0.01458499301224947, + 0.01268930546939373, + 0.012495673261582851, + 0.011049492284655571, + 0.00989001989364624, + 0.00957371387630701, + 0.009140676818788052, + 0.00859383586794138 + ], + "sigma_after": [ + 0.030148033407412483, + 0.030277786292850443, + 0.031311108582944835, + 0.030309708040624952, + 0.03014715094419187, + 0.03019089129199847, + 0.030198899897846996, + 0.029929153968704932, + 0.029773241783660247, + 0.02886946446107127, + 0.02873745150874365, + 0.027474660311035674, + 0.026072831184596083, + 0.025625453210824146, + 0.024966285510808203, + 0.024055672222748577 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.30.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.032901001728105, + "effective_rank_before": 11.032901001728105, + "effective_rank_after": 15.997236376924258, + "nuclear_norm_before_full": 0.37523902393877506, + "nuclear_norm_before": 0.37523902393877506, + "nuclear_norm_after": 0.3752390239387751, + "fro_norm_before_full": 0.14237693209967467, + "fro_norm_after": 0.0938261036100823, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12417694926261902, + 0.03541520982980728, + 0.02473832108080387, + 0.02290387824177742, + 0.019722450524568558, + 0.018483730033040047, + 0.016266366466879845, + 0.015988359227776527, + 0.014498557895421982, + 0.013790719211101532, + 0.01367875374853611, + 0.012771474197506905, + 0.011281400918960571, + 0.011034408584237099, + 0.010460824705660343, + 0.010027620010077953 + ], + "sigma_after": [ + 0.023271885419370753, + 0.023233935288900137, + 0.02445766768645026, + 0.024454857064755097, + 0.023964476348050982, + 0.02369834651857486, + 0.02332930028127645, + 0.023301588179103404, + 0.023234083589282258, + 0.023241446164572337, + 0.023244035354464117, + 0.02326828601257545, + 0.02323559967577299, + 0.023208493540836164, + 0.023107327225646433, + 0.022987695589143407 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.30.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.735235988211201, + "effective_rank_before": 8.735235988211201, + "effective_rank_after": 15.846186942685366, + "nuclear_norm_before_full": 0.17857364704832435, + "nuclear_norm_before": 0.17857364704832435, + "nuclear_norm_after": 0.17857364704832435, + "fro_norm_before_full": 0.07720276666420614, + "fro_norm_after": 0.045047633976715726, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06507743149995804, + 0.03432643041014671, + 0.011203182861208916, + 0.009691269136965275, + 0.008517004549503326, + 0.007136459928005934, + 0.006680042017251253, + 0.006404907442629337, + 0.00555712915956974, + 0.004797258414328098, + 0.00418718671426177, + 0.004108002409338951, + 0.0031951263081282377, + 0.002789801685139537, + 0.002579241059720516, + 0.002323173452168703 + ], + "sigma_after": [ + 0.012096874233912092, + 0.012042395926663986, + 0.012617726278417367, + 0.012314687087785076, + 0.012159512137820492, + 0.012148888950553131, + 0.012157494110450362, + 0.012154003582076548, + 0.012042158564221786, + 0.011707034698581718, + 0.011191286933710472, + 0.011104914827316587, + 0.009741956259849386, + 0.00890729794578399, + 0.0084171752362692, + 0.007770240274912159 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.31.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.274590422971823, + "effective_rank_before": 9.274590422971823, + "effective_rank_after": 15.980629004530241, + "nuclear_norm_before_full": 0.4552101595327258, + "nuclear_norm_before": 0.4552101595327258, + "nuclear_norm_after": 0.45521015953272576, + "fro_norm_before_full": 0.19818156805541712, + "fro_norm_after": 0.11393830331867051, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.18084716796875, + 0.04896550625562668, + 0.030760224908590317, + 0.026820065453648567, + 0.021527990698814392, + 0.018820378929376602, + 0.017885219305753708, + 0.015773683786392212, + 0.014173692092299461, + 0.013478686101734638, + 0.012781906872987747, + 0.012086979113519192, + 0.011590654961764812, + 0.010395657271146774, + 0.009729927405714989, + 0.00957241840660572 + ], + "sigma_after": [ + 0.029179885512133186, + 0.02917344698790669, + 0.030613497372977035, + 0.029967060381804023, + 0.02920909663537385, + 0.02919069183212588, + 0.029213799975651808, + 0.029180690218888986, + 0.028923522068056446, + 0.028704434788733458, + 0.028399859297838233, + 0.027997952617529128, + 0.027643790475022415, + 0.02653359766181959, + 0.025742580413535207, + 0.025536253293329845 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.31.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.633289695518314, + "effective_rank_before": 11.633289695518314, + "effective_rank_after": 15.998390840493357, + "nuclear_norm_before_full": 0.7794166970998049, + "nuclear_norm_before": 0.7794166970998049, + "nuclear_norm_after": 0.7794166970998048, + "fro_norm_before_full": 0.2791720019013202, + "fro_norm_after": 0.19487389119988535, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.23249760270118713, + 0.08851859718561172, + 0.05764603242278099, + 0.04255617409944534, + 0.03784477338194847, + 0.035708218812942505, + 0.03405286371707916, + 0.031958840787410736, + 0.03115541860461235, + 0.030104607343673706, + 0.028751928359270096, + 0.028200581669807434, + 0.02663281001150608, + 0.025270361453294754, + 0.02456180937588215, + 0.02395607717335224 + ], + "sigma_after": [ + 0.04802405317548272, + 0.04772510872151225, + 0.04945165084727905, + 0.05063498242731974, + 0.04968465179309695, + 0.04920341123894689, + 0.048874034815210694, + 0.04855948942417869, + 0.04847628686419068, + 0.04840077811689935, + 0.048356571901651074, + 0.048353649074930595, + 0.04837965946636395, + 0.04841937846774268, + 0.04843444625298719, + 0.048438544512012224 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.31.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.53254773724381, + "effective_rank_before": 10.53254773724381, + "effective_rank_after": 15.997843776417847, + "nuclear_norm_before_full": 0.829023290425539, + "nuclear_norm_before": 0.829023290425539, + "nuclear_norm_after": 0.829023290425539, + "fro_norm_before_full": 0.3272189762681138, + "fro_norm_after": 0.20728403461627826, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.28914445638656616, + 0.08625304698944092, + 0.05832010880112648, + 0.0483614020049572, + 0.037662431597709656, + 0.03568374365568161, + 0.03242211043834686, + 0.03165911138057709, + 0.03021727129817009, + 0.02925051748752594, + 0.027442989870905876, + 0.026903601363301277, + 0.025893812999129295, + 0.02441871352493763, + 0.023099226877093315, + 0.022290745750069618 + ], + "sigma_after": [ + 0.05164993145249719, + 0.05161824574092769, + 0.05416719960942885, + 0.053763196062166625, + 0.05180446623620413, + 0.05163465273223713, + 0.05156711832202825, + 0.05158013330443469, + 0.051617342415958764, + 0.051641061233840775, + 0.051646125008774066, + 0.05162928430862307, + 0.05156335388805562, + 0.051359434978214984, + 0.05103279661980117, + 0.05074894851234598 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.31.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.15164966802067, + "effective_rank_before": 12.15164966802067, + "effective_rank_after": 15.996237137840193, + "nuclear_norm_before_full": 0.19400911312550306, + "nuclear_norm_before": 0.19400911312550306, + "nuclear_norm_after": 0.19400911312550306, + "fro_norm_before_full": 0.06412936877436055, + "fro_norm_after": 0.04851373555357671, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04603486508131027, + 0.02685246244072914, + 0.019378703087568283, + 0.013608042150735855, + 0.012317778542637825, + 0.010755383409559727, + 0.009575114585459232, + 0.008036824874579906, + 0.007344361860305071, + 0.006844029761850834, + 0.0065502626821398735, + 0.006084431428462267, + 0.005725009832531214, + 0.005402942653745413, + 0.005005782935768366, + 0.004493117798119783 + ], + "sigma_after": [ + 0.011673586511843831, + 0.011791101093613445, + 0.011989912724523146, + 0.01224657989530302, + 0.012508860659251815, + 0.012692334661087554, + 0.012576794445635808, + 0.012218036458688597, + 0.012095522768020263, + 0.012051816136563974, + 0.012044045564068807, + 0.012051368004495277, + 0.012062341546414132, + 0.012063905045681859, + 0.01203567203536503, + 0.011907235574946502 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.31.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.915059439533842, + "effective_rank_before": 11.915059439533842, + "effective_rank_after": 15.996112757231623, + "nuclear_norm_before_full": 0.4690784104168415, + "nuclear_norm_before": 0.4690784104168415, + "nuclear_norm_after": 0.4690784104168415, + "fro_norm_before_full": 0.16101961640878334, + "fro_norm_after": 0.11729816732234859, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12632794678211212, + 0.05798870697617531, + 0.03892582654953003, + 0.03294891491532326, + 0.02690957300364971, + 0.02426488697528839, + 0.020453425124287605, + 0.019507765769958496, + 0.0184122696518898, + 0.017216520383954048, + 0.01713070459663868, + 0.016017137095332146, + 0.014931697398424149, + 0.013596286997199059, + 0.012982952408492565, + 0.01146379578858614 + ], + "sigma_after": [ + 0.028284351379106856, + 0.028095967828721733, + 0.029176651835588742, + 0.0298855976295536, + 0.030734515059849515, + 0.030494239135026135, + 0.029645160051898854, + 0.02945342561616802, + 0.02928586931808682, + 0.029185005346307222, + 0.029181117201329293, + 0.029165929485361664, + 0.029193175866635467, + 0.02921333171338057, + 0.029186209537413226, + 0.0288978634124138 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.31.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.515365334477467, + "effective_rank_before": 11.515365334477467, + "effective_rank_after": 15.996257153293621, + "nuclear_norm_before_full": 0.3994497759267688, + "nuclear_norm_before": 0.3994497759267688, + "nuclear_norm_after": 0.3994497759267688, + "fro_norm_before_full": 0.14009699223428687, + "fro_norm_after": 0.0998859148771538, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10636675357818604, + 0.06335490196943283, + 0.033199772238731384, + 0.022882919758558273, + 0.022222671657800674, + 0.018448548391461372, + 0.01749904453754425, + 0.017111489549279213, + 0.015475102700293064, + 0.014122987166047096, + 0.013806057162582874, + 0.012661444023251534, + 0.011685230769217014, + 0.01123112067580223, + 0.009731853380799294, + 0.009649878367781639 + ], + "sigma_after": [ + 0.02389296806797736, + 0.024706068358461605, + 0.02485910588613882, + 0.02615000954715829, + 0.026104565866876078, + 0.02538182081802909, + 0.025172820811557504, + 0.025097149629903656, + 0.02487495119469678, + 0.024825830612252145, + 0.024828989382910296, + 0.024860046335419276, + 0.024864173377590035, + 0.02483940571396227, + 0.02451239659670972, + 0.024479473727125826 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.31.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.808730779082001, + "effective_rank_before": 8.808730779082001, + "effective_rank_after": 15.909290961101524, + "nuclear_norm_before_full": 0.18800809141248465, + "nuclear_norm_before": 0.18800809141248465, + "nuclear_norm_after": 0.18800809141248465, + "fro_norm_before_full": 0.08351400333355433, + "fro_norm_after": 0.04725612941715388, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07562658935785294, + 0.023411313071846962, + 0.015112601220607758, + 0.00990578718483448, + 0.008586928248405457, + 0.007672725711017847, + 0.007347191218286753, + 0.0066818916238844395, + 0.006037067621946335, + 0.005118127912282944, + 0.0045431372709572315, + 0.004352557938545942, + 0.003991240169852972, + 0.0036473001819103956, + 0.0031693712808191776, + 0.0028042613994330168 + ], + "sigma_after": [ + 0.012540660484875427, + 0.01255297218705648, + 0.01314400452852965, + 0.012627456737427213, + 0.012534016686061562, + 0.012547688604038525, + 0.012554353792897705, + 0.012539361102242637, + 0.012442614388868095, + 0.012044897427312708, + 0.011571299196379052, + 0.011367824820771243, + 0.01091241493359291, + 0.01038979669234496, + 0.009513453770614454, + 0.00872527605947202 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.32.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.018562470242962, + "effective_rank_before": 8.018562470242962, + "effective_rank_after": 15.95787471964166, + "nuclear_norm_before_full": 0.4632578743621707, + "nuclear_norm_before": 0.4632578743621707, + "nuclear_norm_after": 0.4632578743621707, + "fro_norm_before_full": 0.22767569185841935, + "fro_norm_after": 0.11611585246951543, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.21636319160461426, + 0.04003531485795975, + 0.02813437581062317, + 0.023043334484100342, + 0.017855720594525337, + 0.016932262107729912, + 0.01510426215827465, + 0.014254335314035416, + 0.013584361411631107, + 0.013184217736124992, + 0.011909091845154762, + 0.011482671834528446, + 0.010825566947460175, + 0.01057218387722969, + 0.010452983900904655, + 0.009523999877274036 + ], + "sigma_after": [ + 0.030746371120061428, + 0.03228070261464979, + 0.031074077743483194, + 0.03069121205887, + 0.03067812880948461, + 0.030560587434318727, + 0.030058120672423076, + 0.029663965506511403, + 0.029265820308676957, + 0.028987711508558493, + 0.027880717151039593, + 0.027430237413756622, + 0.026652315991662498, + 0.026324474990390034, + 0.026164788905279356, + 0.02479864213300494 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.32.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.853473327422096, + "effective_rank_before": 11.853473327422096, + "effective_rank_after": 15.997919019090912, + "nuclear_norm_before_full": 0.7869340498000383, + "nuclear_norm_before": 0.7869340498000383, + "nuclear_norm_after": 0.7869340498000384, + "fro_norm_before_full": 0.2776911534359584, + "fro_norm_after": 0.19675925971750208, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.23067204654216766, + 0.0838823914527893, + 0.05503791570663452, + 0.04555528610944748, + 0.038557153195142746, + 0.037311017513275146, + 0.03598843514919281, + 0.03373932093381882, + 0.03263518214225769, + 0.030957991257309914, + 0.029829736799001694, + 0.029304906725883484, + 0.026725908741354942, + 0.02602497674524784, + 0.02576661854982376, + 0.02494516223669052 + ], + "sigma_after": [ + 0.0482774045130267, + 0.04843575968020198, + 0.05020352107041616, + 0.05125413492223138, + 0.0502016295615363, + 0.049912842859911724, + 0.04960969003608054, + 0.049150737799579784, + 0.048970521150321515, + 0.04877166620055245, + 0.04869251669665745, + 0.04867035268077558, + 0.048671073077016465, + 0.04869105185633194, + 0.04869894337917332, + 0.0487222043162252 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.32.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.245257686789245, + "effective_rank_before": 10.245257686789245, + "effective_rank_after": 15.996528003914541, + "nuclear_norm_before_full": 0.8636747170239687, + "nuclear_norm_before": 0.8636747170239687, + "nuclear_norm_after": 0.8636747170239685, + "fro_norm_before_full": 0.3487074675333716, + "fro_norm_after": 0.21596595224894388, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.311125248670578, + 0.0871112197637558, + 0.05770100653171539, + 0.05484382435679436, + 0.044203754514455795, + 0.03804069012403488, + 0.03475959599018097, + 0.032430969178676605, + 0.029310012236237526, + 0.02744254283607006, + 0.026521872729063034, + 0.025746162980794907, + 0.02457614243030548, + 0.024408916011452675, + 0.022944573312997818, + 0.022508185356855392 + ], + "sigma_after": [ + 0.05389095342820832, + 0.05381911998264152, + 0.056710241973027084, + 0.05654974498472631, + 0.054689027559360344, + 0.05389438499393808, + 0.053820957222257805, + 0.053869367045231736, + 0.053906850295288175, + 0.05380501253270982, + 0.053690931378281334, + 0.05355190538254129, + 0.05325322392772923, + 0.05320070922369942, + 0.05262046220251172, + 0.05240182489181627 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.32.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.557203540645983, + "effective_rank_before": 11.557203540645983, + "effective_rank_after": 15.993074142802916, + "nuclear_norm_before_full": 0.17649267823435366, + "nuclear_norm_before": 0.17649267823435366, + "nuclear_norm_after": 0.17649267823435363, + "fro_norm_before_full": 0.0615788823500458, + "fro_norm_after": 0.04414221421845915, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04792246222496033, + 0.02202419377863407, + 0.0184991005808115, + 0.012033980339765549, + 0.010076699778437614, + 0.008468927815556526, + 0.007782868575304747, + 0.007184712681919336, + 0.0069177583791315556, + 0.006675217300653458, + 0.006323863752186298, + 0.0056649139150977135, + 0.005019337870180607, + 0.0047645303420722485, + 0.0036083597224205732, + 0.0035257511772215366 + ], + "sigma_after": [ + 0.010695360410628227, + 0.01065956466804932, + 0.011016488913737955, + 0.011443977288763644, + 0.011647661729128527, + 0.011390280263961095, + 0.011230902246505926, + 0.011122000448849385, + 0.0110895215138913, + 0.011069890541046144, + 0.011057518064691811, + 0.011069437965916881, + 0.011069007314206134, + 0.01104382150593373, + 0.010483988857065923, + 0.010403256501977631 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.32.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.48036014356735, + "effective_rank_before": 10.48036014356735, + "effective_rank_after": 15.99532987389182, + "nuclear_norm_before_full": 0.46791927237063646, + "nuclear_norm_before": 0.46791927237063646, + "nuclear_norm_after": 0.4679192723706364, + "fro_norm_before_full": 0.18011985564062202, + "fro_norm_after": 0.11701375341383641, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15204401314258575, + 0.05660542473196983, + 0.04930043965578079, + 0.02683904953300953, + 0.023347685113549232, + 0.019979124888777733, + 0.018493929877877235, + 0.017836011946201324, + 0.015715479850769043, + 0.015162194147706032, + 0.013847831636667252, + 0.013440138660371304, + 0.012606699950993061, + 0.011865297332406044, + 0.010777845047414303, + 0.010058106854557991 + ], + "sigma_after": [ + 0.029335000214597284, + 0.02909018347553461, + 0.029471261164143094, + 0.03071271270660045, + 0.030000487116837584, + 0.029484598597077652, + 0.029421124874498838, + 0.029422581417475193, + 0.02947063573274187, + 0.029468658417098365, + 0.029372652765179343, + 0.02930368837562463, + 0.029081743617242802, + 0.028771845553476432, + 0.02807801942731625, + 0.027434078915192017 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.32.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.912186782677724, + "effective_rank_before": 10.912186782677724, + "effective_rank_after": 15.998098920345795, + "nuclear_norm_before_full": 0.3597488198429346, + "nuclear_norm_before": 0.3597488198429346, + "nuclear_norm_after": 0.3597488198429346, + "fro_norm_before_full": 0.13652951687313092, + "fro_norm_after": 0.08994790963854764, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11739776283502579, + 0.04102183133363724, + 0.02643468603491783, + 0.019019849598407745, + 0.018321968615055084, + 0.017201555892825127, + 0.015074172988533974, + 0.014162554405629635, + 0.013637731783092022, + 0.012508398853242397, + 0.012329157441854477, + 0.012119466438889503, + 0.011210461147129536, + 0.010676500387489796, + 0.00992913544178009, + 0.008703586645424366 + ], + "sigma_after": [ + 0.02238649694478953, + 0.02229192013560252, + 0.023201504263319905, + 0.02310097612599809, + 0.02294987769281014, + 0.022715679846978035, + 0.02241832582845324, + 0.022377338476287884, + 0.022375261219837694, + 0.022400972667805774, + 0.0224056453659183, + 0.02241015978954274, + 0.02240271852613316, + 0.022361500396778106, + 0.02222790584770462, + 0.02172253671497488 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.32.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.31064466200504, + "effective_rank_before": 7.31064466200504, + "effective_rank_after": 15.60768990365465, + "nuclear_norm_before_full": 0.1581286465516314, + "nuclear_norm_before": 0.1581286465516314, + "nuclear_norm_after": 0.1581286465516314, + "fro_norm_before_full": 0.07877364714074479, + "fro_norm_after": 0.040450615058111594, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.0734366849064827, + 0.018536392599344254, + 0.013888711109757423, + 0.009727267548441887, + 0.00732473423704505, + 0.005593626294285059, + 0.004801691044121981, + 0.004161450080573559, + 0.003336968831717968, + 0.003222732339054346, + 0.0029913419857621193, + 0.002769134007394314, + 0.0025196773931384087, + 0.0022408938966691494, + 0.0018432890065014362, + 0.001734051271341741 + ], + "sigma_after": [ + 0.011878297462421257, + 0.011897594561698924, + 0.012488487497754792, + 0.01202313157068272, + 0.011887403978238483, + 0.011763567396374007, + 0.011396022158329886, + 0.010834202545561451, + 0.009669405500965983, + 0.009465103007104294, + 0.009018351737781838, + 0.008547871305772581, + 0.007971911864243834, + 0.007270140159878277, + 0.006169355188659712, + 0.005847800616163348 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.33.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 6.475210286583287, + "effective_rank_before": 6.475210286583287, + "effective_rank_after": 15.760428734041936, + "nuclear_norm_before_full": 0.44842084497213364, + "nuclear_norm_before": 0.44842084497213364, + "nuclear_norm_after": 0.44842084497213364, + "fro_norm_before_full": 0.2503406133789578, + "fro_norm_after": 0.11374672566331494, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2426074892282486, + 0.03839757665991783, + 0.02463768608868122, + 0.02013537660241127, + 0.014623655006289482, + 0.014007704332470894, + 0.012089387513697147, + 0.011660453863441944, + 0.011095202527940273, + 0.010220988653600216, + 0.009497901424765587, + 0.008892592042684555, + 0.008513075299561024, + 0.008397395722568035, + 0.007029589265584946, + 0.006614770740270615 + ], + "sigma_after": [ + 0.03372268117632471, + 0.03516188029508564, + 0.033557045099643266, + 0.03357471219287258, + 0.03178119578914699, + 0.031310849594669586, + 0.029366763576961233, + 0.028825477624157155, + 0.02804958814584065, + 0.02670669085714857, + 0.02546314929188622, + 0.024329611153369422, + 0.023576219565681948, + 0.0233400966631022, + 0.020324164943621548, + 0.01933071900262195 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.33.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.959628171328536, + "effective_rank_before": 10.959628171328536, + "effective_rank_after": 15.997341715148622, + "nuclear_norm_before_full": 0.8550024591386318, + "nuclear_norm_before": 0.8550024591386318, + "nuclear_norm_after": 0.8550024591386319, + "fro_norm_before_full": 0.32224074178717177, + "fro_norm_after": 0.2137863659457069, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2735406756401062, + 0.10922227054834366, + 0.055937182158231735, + 0.04696356877684593, + 0.04352521523833275, + 0.03750923275947571, + 0.03490939363837242, + 0.03339141234755516, + 0.032169707119464874, + 0.030549874529242516, + 0.029286108911037445, + 0.028300991281867027, + 0.02599041722714901, + 0.02543894574046135, + 0.024902790784835815, + 0.02336467243731022 + ], + "sigma_after": [ + 0.05303989745963231, + 0.051674935853905056, + 0.05592617110262717, + 0.05527112775946369, + 0.05455270247704806, + 0.05342554560747441, + 0.0531853747864283, + 0.053131100392065006, + 0.053126927756473465, + 0.053158697618226974, + 0.05319418745040557, + 0.05321442564090409, + 0.0531655347256149, + 0.05311967257920518, + 0.05305737740752976, + 0.052758780521627996 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.33.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.466826111458468, + "effective_rank_before": 8.466826111458468, + "effective_rank_after": 15.981767337974148, + "nuclear_norm_before_full": 0.9298223182559013, + "nuclear_norm_before": 0.9298223182559013, + "nuclear_norm_after": 0.9298223182559012, + "fro_norm_before_full": 0.43758686232665817, + "fro_norm_after": 0.23271645920837666, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.4097551703453064, + 0.100661501288414, + 0.04753265529870987, + 0.04270476847887039, + 0.03775150701403618, + 0.03624247387051582, + 0.0315401554107666, + 0.030160704627633095, + 0.027421360835433006, + 0.027147946879267693, + 0.025270480662584305, + 0.02461436204612255, + 0.024500524625182152, + 0.02250543422996998, + 0.02181207574903965, + 0.020201196894049644 + ], + "sigma_after": [ + 0.06049628728376011, + 0.06070213798768886, + 0.06054686159819409, + 0.060485932257247396, + 0.06057820238431469, + 0.06055879303291352, + 0.060013013423718994, + 0.059634834824451324, + 0.058461523104875554, + 0.058308937416759896, + 0.05706307658832807, + 0.05654081368089717, + 0.05644536801218757, + 0.05452923595257643, + 0.05375085739479736, + 0.05170644331319023 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.33.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.613172760536209, + "effective_rank_before": 12.613172760536209, + "effective_rank_after": 15.994805975194412, + "nuclear_norm_before_full": 0.17991892853751779, + "nuclear_norm_before": 0.17991892853751779, + "nuclear_norm_after": 0.1799189285375178, + "fro_norm_before_full": 0.0575608716271656, + "fro_norm_after": 0.04499435611374438, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04082523286342621, + 0.021322302520275116, + 0.018657289445400238, + 0.013249728828668594, + 0.010924135334789753, + 0.010252115316689014, + 0.00956723466515541, + 0.008400189690291882, + 0.007462038192898035, + 0.007146164309233427, + 0.0063199978321790695, + 0.006139441858977079, + 0.005405493080615997, + 0.00529837142676115, + 0.00452591385692358, + 0.004423279315233231 + ], + "sigma_after": [ + 0.010845821628057806, + 0.010721246566680179, + 0.010948840224297212, + 0.011187544668055528, + 0.011602604714018748, + 0.011709659604054574, + 0.011745541783194853, + 0.011598639331007521, + 0.011366025658580567, + 0.01129209257828849, + 0.011164736134443258, + 0.011153063639088238, + 0.01115465247457017, + 0.01115825465683015, + 0.01114258936977406, + 0.011127615506576437 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.33.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.677837457120638, + "effective_rank_before": 10.677837457120638, + "effective_rank_after": 15.993520109995917, + "nuclear_norm_before_full": 0.446679531596601, + "nuclear_norm_before": 0.446679531596601, + "nuclear_norm_after": 0.446679531596601, + "fro_norm_before_full": 0.16748751733029893, + "fro_norm_after": 0.11171496759375085, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1352986991405487, + 0.06883861124515533, + 0.03795856237411499, + 0.02619425766170025, + 0.02421550080180168, + 0.020782792940735817, + 0.018263600766658783, + 0.01739686354994774, + 0.01559231523424387, + 0.013959749601781368, + 0.013522975146770477, + 0.012514869682490826, + 0.012188667431473732, + 0.010624759830534458, + 0.010082165710628033, + 0.009245140478014946 + ], + "sigma_after": [ + 0.027546110253932332, + 0.027360927286020062, + 0.02822652956433858, + 0.029479713262869855, + 0.02916689391137287, + 0.028436971527778198, + 0.028109837699908144, + 0.02807474358042731, + 0.028096827166576546, + 0.028113911209765828, + 0.028093171935497548, + 0.02796333321438213, + 0.027888935965343583, + 0.027220526353442843, + 0.026838361110301127, + 0.02606273755464406 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.33.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.473515875770927, + "effective_rank_before": 12.473515875770927, + "effective_rank_after": 15.996149660224457, + "nuclear_norm_before_full": 0.349052669480443, + "nuclear_norm_before": 0.349052669480443, + "nuclear_norm_after": 0.349052669480443, + "fro_norm_before_full": 0.11505437648222874, + "fro_norm_after": 0.0872841987419917, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08904753625392914, + 0.03405269607901573, + 0.03025834448635578, + 0.025563573464751244, + 0.021553130820393562, + 0.01887701265513897, + 0.017452646046876907, + 0.016230646520853043, + 0.015461224131286144, + 0.014103926718235016, + 0.013550611212849617, + 0.012491359375417233, + 0.011331245303153992, + 0.010621227324008942, + 0.01004049088805914, + 0.008416998200118542 + ], + "sigma_after": [ + 0.020817541100207795, + 0.02151433450105227, + 0.02157659199425058, + 0.02173517946713721, + 0.022489632476502402, + 0.022707980112259465, + 0.022557429078584237, + 0.022303299939975336, + 0.022114220749006572, + 0.02180042644221336, + 0.02170073200373202, + 0.021580926144220813, + 0.0215575867644032, + 0.021577599783176395, + 0.021591777082618432, + 0.021427411841102896 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.33.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.453160684244883, + "effective_rank_before": 7.453160684244883, + "effective_rank_after": 15.65277550486897, + "nuclear_norm_before_full": 0.19398830831050873, + "nuclear_norm_before": 0.19398830831050873, + "nuclear_norm_after": 0.19398830831050873, + "fro_norm_before_full": 0.09663420468460528, + "fro_norm_after": 0.04948797028915109, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.0906473770737648, + 0.022275203838944435, + 0.013933688402175903, + 0.011209508404135704, + 0.008426427841186523, + 0.007497397717088461, + 0.0061601814813911915, + 0.005704075563699007, + 0.005227426066994667, + 0.00475929444655776, + 0.004369264934211969, + 0.003230422269552946, + 0.0029691378585994244, + 0.0027186123188585043, + 0.0025204261764883995, + 0.0023398639168590307 + ], + "sigma_after": [ + 0.014191223738591732, + 0.014234876998974244, + 0.014731204850687185, + 0.014259797411882859, + 0.014208403383384551, + 0.014169310403641314, + 0.013770555110687609, + 0.013480188327008343, + 0.013065973668748235, + 0.012535169465495327, + 0.011991223059507335, + 0.009841890310083866, + 0.009230684046514356, + 0.008605026887317891, + 0.008083637041773575, + 0.007589143606210309 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.34.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 5.935225415128159, + "effective_rank_before": 5.935225415128159, + "effective_rank_after": 15.5047963330806, + "nuclear_norm_before_full": 0.4853720245882869, + "nuclear_norm_before": 0.4853720245882869, + "nuclear_norm_after": 0.4853720245882869, + "fro_norm_before_full": 0.2796507715693484, + "fro_norm_after": 0.12502714724816252, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.270490437746048, + 0.05117303505539894, + 0.026942849159240723, + 0.022637799382209778, + 0.015439647249877453, + 0.013382097706198692, + 0.011711042374372482, + 0.010945286601781845, + 0.010349388234317303, + 0.009251892566680908, + 0.008590091951191425, + 0.007884909398853779, + 0.007737028878182173, + 0.007083654869347811, + 0.0062125567346811295, + 0.005540306679904461 + ], + "sigma_after": [ + 0.03987882408443105, + 0.041564967603082, + 0.039708039435868155, + 0.0397273487794835, + 0.03685592381361333, + 0.034581471323469885, + 0.03209306833826612, + 0.03075103548496324, + 0.02961803378576781, + 0.02732967279560185, + 0.025825751639112427, + 0.024123837184818107, + 0.023754276823142177, + 0.022070660501060452, + 0.01970289998394007, + 0.017786213011666722 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.34.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.091467475233607, + "effective_rank_before": 7.091467475233607, + "effective_rank_after": 15.888084460592873, + "nuclear_norm_before_full": 0.9142075423151255, + "nuclear_norm_before": 0.9142075423151255, + "nuclear_norm_after": 0.9142075423151257, + "fro_norm_before_full": 0.4835137970731498, + "fro_norm_after": 0.2301316169799555, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.4640987515449524, + 0.09022760391235352, + 0.045938294380903244, + 0.040252476930618286, + 0.03325250744819641, + 0.02880580723285675, + 0.02788461744785309, + 0.025507252663373947, + 0.022773051634430885, + 0.021577810868620872, + 0.020685182884335518, + 0.020556852221488953, + 0.019792914390563965, + 0.01936178468167782, + 0.017228325828909874, + 0.016264308243989944 + ], + "sigma_after": [ + 0.06464279886656976, + 0.06726899116393484, + 0.06446946505792046, + 0.0645286500527835, + 0.06351579846119407, + 0.06141019356921434, + 0.06077063761802328, + 0.0587431886052135, + 0.05567797271083665, + 0.054074350329575055, + 0.0527684423798129, + 0.052573002441098214, + 0.0513693143213475, + 0.05065949589984312, + 0.04682247416484293, + 0.04491276667291548 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.34.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.150096669423437, + "effective_rank_before": 7.150096669423437, + "effective_rank_after": 15.860029478608471, + "nuclear_norm_before_full": 0.8840320790186524, + "nuclear_norm_before": 0.8840320790186524, + "nuclear_norm_after": 0.8840320790186524, + "fro_norm_before_full": 0.4669430860392185, + "fro_norm_after": 0.2228908715373544, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.44928088784217834, + 0.0760321095585823, + 0.049260690808296204, + 0.03740997239947319, + 0.035107653588056564, + 0.029152484610676765, + 0.027529405429959297, + 0.025966744869947433, + 0.024578286334872246, + 0.023618172854185104, + 0.020575517788529396, + 0.0194267425686121, + 0.018208518624305725, + 0.01714043691754341, + 0.01579374447464943, + 0.014950710348784924 + ], + "sigma_after": [ + 0.06234483497286025, + 0.06541920255444054, + 0.06213825750129063, + 0.062146873551749644, + 0.06190989235067566, + 0.0599916712549718, + 0.059009989635130496, + 0.05783231832435865, + 0.05657772155991102, + 0.05558859972057714, + 0.05175588582736295, + 0.050021483775178836, + 0.048005989966945606, + 0.04608915235521333, + 0.043474100264615075, + 0.04172610540337081 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.34.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.749571444027275, + "effective_rank_before": 10.749571444027275, + "effective_rank_after": 15.987902392142324, + "nuclear_norm_before_full": 0.2304220055229962, + "nuclear_norm_before": 0.2304220055229962, + "nuclear_norm_after": 0.2304220055229962, + "fro_norm_before_full": 0.08695534447611693, + "fro_norm_after": 0.05764833893480486, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07336616516113281, + 0.02709209732711315, + 0.017733341082930565, + 0.01687196083366871, + 0.013442735187709332, + 0.011659040115773678, + 0.010089321061968803, + 0.009152617305517197, + 0.008870990946888924, + 0.008265291340649128, + 0.0069440025836229324, + 0.006374763790518045, + 0.006167379207909107, + 0.0051683527417480946, + 0.004895757883787155, + 0.004328188952058554 + ], + "sigma_after": [ + 0.01445625146733235, + 0.014361354896051512, + 0.014869521450774028, + 0.015029340956043214, + 0.015209249452761638, + 0.01487047668236892, + 0.01457774077935087, + 0.014504834655870557, + 0.014500011085341416, + 0.014508470159695183, + 0.014505671109631491, + 0.014417494013813155, + 0.014360136925195962, + 0.0138101728066265, + 0.013561756762666381, + 0.012879522319473011 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.34.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.674722424606339, + "effective_rank_before": 9.674722424606339, + "effective_rank_after": 15.97410613871729, + "nuclear_norm_before_full": 0.4081218079663813, + "nuclear_norm_before": 0.4081218079663813, + "nuclear_norm_after": 0.4081218079663813, + "fro_norm_before_full": 0.16834699258703897, + "fro_norm_after": 0.10219111742101145, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1471537947654724, + 0.05600440129637718, + 0.028365332633256912, + 0.027010567486286163, + 0.01979709230363369, + 0.016969386488199234, + 0.016065767034888268, + 0.013937752693891525, + 0.013005349785089493, + 0.012439927086234093, + 0.011408518999814987, + 0.010615897364914417, + 0.01015094667673111, + 0.009111366234719753, + 0.008874667808413506, + 0.007211039308458567 + ], + "sigma_after": [ + 0.026126473371595432, + 0.02547267400826804, + 0.02748565851350072, + 0.02744825486211947, + 0.026254989307072905, + 0.026082252470097704, + 0.026096070629815737, + 0.02611943319021492, + 0.02604841771928879, + 0.02595510210655627, + 0.02564700269393323, + 0.025255069724924584, + 0.02494944030044265, + 0.024031305117144748, + 0.023772667426091746, + 0.021376996525314388 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.34.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.321458462812723, + "effective_rank_before": 9.321458462812723, + "effective_rank_after": 15.981778009160204, + "nuclear_norm_before_full": 0.39172953087836504, + "nuclear_norm_before": 0.39172953087836504, + "nuclear_norm_after": 0.3917295308783651, + "fro_norm_before_full": 0.17257767228830537, + "fro_norm_after": 0.09804146237543561, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15997570753097534, + 0.030526405200362206, + 0.025826815515756607, + 0.02425958402454853, + 0.017661510035395622, + 0.017463319003582, + 0.014932169578969479, + 0.014055492356419563, + 0.012799093499779701, + 0.012405868619680405, + 0.011969255283474922, + 0.01156490296125412, + 0.010954995639622211, + 0.00985366478562355, + 0.009565728716552258, + 0.007915018126368523 + ], + "sigma_after": [ + 0.02482416010127675, + 0.02609607853179832, + 0.02593043915954635, + 0.025669916702386903, + 0.024824052144671673, + 0.024823466396174585, + 0.024866979107710414, + 0.024849757397991345, + 0.024711322855222914, + 0.024627070960946823, + 0.024504145150131076, + 0.024359049838978162, + 0.02407552955810598, + 0.023333199281643607, + 0.023083967097810038, + 0.021150396593970132 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.34.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.5566438192827805, + "effective_rank_before": 7.5566438192827805, + "effective_rank_after": 15.76925178977103, + "nuclear_norm_before_full": 0.18930767686106265, + "nuclear_norm_before": 0.18930767686106265, + "nuclear_norm_after": 0.18930767686106267, + "fro_norm_before_full": 0.0935448897346127, + "fro_norm_after": 0.04797942214907784, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08745574206113815, + 0.022641809657216072, + 0.014312400482594967, + 0.010112018324434757, + 0.00775515241548419, + 0.006952270865440369, + 0.0064817070960998535, + 0.0052462066523730755, + 0.0047720023430883884, + 0.0040479316376149654, + 0.003837959375232458, + 0.0037716743536293507, + 0.0036317696794867516, + 0.0030184530187398195, + 0.0028290830086916685, + 0.0024414958897978067 + ], + "sigma_after": [ + 0.013586404417707226, + 0.013587417079824887, + 0.014229473461405387, + 0.01359654532311516, + 0.013601889280023143, + 0.013523493362800018, + 0.013405436861300558, + 0.01268540619081666, + 0.012202609816160104, + 0.011198961957619298, + 0.010844027780349524, + 0.010725879562600654, + 0.010466868753183839, + 0.009177863567912533, + 0.008730340051414681, + 0.007745059394828992 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.35.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.596264893831274, + "effective_rank_before": 7.596264893831274, + "effective_rank_after": 15.672798584911648, + "nuclear_norm_before_full": 0.3810668499208987, + "nuclear_norm_before": 0.3810668499208987, + "nuclear_norm_after": 0.3810668499208986, + "fro_norm_before_full": 0.1819477016571509, + "fro_norm_after": 0.09712315793709848, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1640716791152954, + 0.06130678951740265, + 0.026975391432642937, + 0.022529082372784615, + 0.021088218316435814, + 0.013936001807451248, + 0.011767704039812088, + 0.009331241250038147, + 0.008146887645125389, + 0.007503001019358635, + 0.00739049119874835, + 0.006763772573322058, + 0.006056048441678286, + 0.005110498052090406, + 0.0047429571859538555, + 0.004347085952758789 + ], + "sigma_after": [ + 0.02827378482842577, + 0.02751442268089742, + 0.029470771313587333, + 0.02861146698911025, + 0.028400299338813013, + 0.02819860592200242, + 0.027519925849352405, + 0.025442685767384676, + 0.023754635909419505, + 0.02262721454032187, + 0.022414666485216013, + 0.02114558329348645, + 0.01953983601754023, + 0.017116871040868523, + 0.016093346768801097, + 0.014942733175671657 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.35.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 5.76589820501034, + "effective_rank_before": 5.76589820501034, + "effective_rank_after": 15.499865398979454, + "nuclear_norm_before_full": 0.9356670500710607, + "nuclear_norm_before": 0.9356670500710607, + "nuclear_norm_after": 0.9356670500710607, + "fro_norm_before_full": 0.5542170575548256, + "fro_norm_after": 0.24102598111977805, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.5405022501945496, + 0.08300405740737915, + 0.043288033455610275, + 0.03252043202519417, + 0.031842149794101715, + 0.03100689873099327, + 0.025466516613960266, + 0.02327851392328739, + 0.02109082043170929, + 0.019685151055455208, + 0.016640597954392433, + 0.015509314835071564, + 0.014830735512077808, + 0.013091913424432278, + 0.012039716355502605, + 0.011869948357343674 + ], + "sigma_after": [ + 0.07631460676836177, + 0.07921965075391955, + 0.07578059823454246, + 0.07193129424023163, + 0.07143056896424933, + 0.07076043327756455, + 0.06467542643480637, + 0.0614229225095536, + 0.057665161878876804, + 0.05498252827038817, + 0.048465551838431584, + 0.045805714305680326, + 0.04415094210205943, + 0.03971630810094725, + 0.03690388352102633, + 0.036441458870421495 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.35.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 6.322848583336041, + "effective_rank_before": 6.322848583336041, + "effective_rank_after": 15.74216170837788, + "nuclear_norm_before_full": 0.8020055005326867, + "nuclear_norm_before": 0.8020055005326867, + "nuclear_norm_after": 0.8020055005326867, + "fro_norm_before_full": 0.45399213543387074, + "fro_norm_after": 0.2037044915384723, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.4408718943595886, + 0.06751319020986557, + 0.043786294758319855, + 0.03225450590252876, + 0.028840268030762672, + 0.024990512058138847, + 0.0223261509090662, + 0.019939757883548737, + 0.018639281392097473, + 0.01657813787460327, + 0.016161540523171425, + 0.01537354290485382, + 0.014821785502135754, + 0.014382763765752316, + 0.01337016187608242, + 0.012155712582170963 + ], + "sigma_after": [ + 0.06119735791696246, + 0.06351871176074732, + 0.060885934566711125, + 0.060290148481524454, + 0.05903962501838775, + 0.05644927214606022, + 0.05374932754152411, + 0.0506184890955998, + 0.04861394637030552, + 0.04499726817319288, + 0.044200803588469645, + 0.04263464169699665, + 0.04149198753462513, + 0.040556036834263676, + 0.038308345803855955, + 0.03545360400346 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.35.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.587620133368747, + "effective_rank_before": 10.587620133368747, + "effective_rank_after": 15.99054125208584, + "nuclear_norm_before_full": 0.22547893691807985, + "nuclear_norm_before": 0.22547893691807985, + "nuclear_norm_after": 0.22547893691807985, + "fro_norm_before_full": 0.08433776569990886, + "fro_norm_after": 0.05640283573572725, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06756796687841415, + 0.03144325315952301, + 0.02620631270110607, + 0.01319892518222332, + 0.011633990332484245, + 0.011208994314074516, + 0.009733480401337147, + 0.008075246587395668, + 0.007604256272315979, + 0.006642058491706848, + 0.006167746149003506, + 0.00604893546551466, + 0.005594560410827398, + 0.005146834999322891, + 0.004810988437384367, + 0.004395387135446072 + ], + "sigma_after": [ + 0.013942354221373424, + 0.013714676981059257, + 0.014129080923724953, + 0.014974842828795671, + 0.014695698782379769, + 0.014600470302192671, + 0.014328242931125302, + 0.014265095639323343, + 0.014278343983925471, + 0.014255136666102841, + 0.014173503274334474, + 0.01414139687604011, + 0.013963826941121155, + 0.013686276293621697, + 0.013397898857171908, + 0.012932091415787812 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.35.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 5.571221236433792, + "effective_rank_before": 5.571221236433792, + "effective_rank_after": 15.549634625251409, + "nuclear_norm_before_full": 0.4303470849990845, + "nuclear_norm_before": 0.4303470849990845, + "nuclear_norm_after": 0.4303470849990845, + "fro_norm_before_full": 0.25913162477528295, + "fro_norm_after": 0.11059906716351138, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2529270350933075, + 0.04066447913646698, + 0.019040998071432114, + 0.014569321647286415, + 0.012785453349351883, + 0.011988360434770584, + 0.010927903465926647, + 0.010180036537349224, + 0.0086805559694767, + 0.008000954054296017, + 0.007883859798312187, + 0.007795354351401329, + 0.007090763654559851, + 0.006190895568579435, + 0.005926915444433689, + 0.005694198422133923 + ], + "sigma_after": [ + 0.03588092920308103, + 0.03744788624450029, + 0.03545267455981467, + 0.03332354722428281, + 0.0315304458704062, + 0.030514399145847762, + 0.028944373221171917, + 0.027683813552017208, + 0.024772236191866626, + 0.023286567769199764, + 0.023020401329380658, + 0.022817261346206468, + 0.021140993357053068, + 0.01885360078336424, + 0.018153055901421532, + 0.017524899299470258 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.35.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.166977744040276, + "effective_rank_before": 8.166977744040276, + "effective_rank_after": 15.92833221085896, + "nuclear_norm_before_full": 0.43665161076933146, + "nuclear_norm_before": 0.43665161076933146, + "nuclear_norm_after": 0.4366516107693315, + "fro_norm_before_full": 0.2113163074942155, + "fro_norm_after": 0.10963541283724358, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.20009374618530273, + 0.0362272635102272, + 0.027276089414954185, + 0.024799926206469536, + 0.018081581220030785, + 0.016367046162486076, + 0.014857457019388676, + 0.014291739091277122, + 0.013724235817790031, + 0.012241492047905922, + 0.01180973183363676, + 0.011223400942981243, + 0.010618709959089756, + 0.008716749958693981, + 0.008495789021253586, + 0.007826652377843857 + ], + "sigma_after": [ + 0.02905709706309087, + 0.030566146965170342, + 0.029570041496616968, + 0.029204090646074727, + 0.029068387112741054, + 0.028984674169005575, + 0.028702974537554205, + 0.028523149767123692, + 0.028292620763934944, + 0.02741039407489675, + 0.027066968158442756, + 0.02653123477468537, + 0.025890404212511414, + 0.02324540634243015, + 0.022873618004693317, + 0.021664402680359338 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.35.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 6.2197951703104986, + "effective_rank_before": 6.2197951703104986, + "effective_rank_after": 15.083388693642195, + "nuclear_norm_before_full": 0.21973773767240345, + "nuclear_norm_before": 0.21973773767240345, + "nuclear_norm_after": 0.21973773767240345, + "fro_norm_before_full": 0.11886143828655837, + "fro_norm_after": 0.05786464373449969, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11155975610017776, + 0.03291802853345871, + 0.015456750057637691, + 0.010303793475031853, + 0.008562683127820492, + 0.006198887713253498, + 0.006004102993756533, + 0.0053828004747629166, + 0.004940432496368885, + 0.00401008827611804, + 0.003557087853550911, + 0.0025049729738384485, + 0.002398852491751313, + 0.0021868636831641197, + 0.0019822572357952595, + 0.0017703801859170198 + ], + "sigma_after": [ + 0.018506630836449628, + 0.018528486410816055, + 0.018871257624185494, + 0.018528720995827284, + 0.018354829998254128, + 0.01678108747590437, + 0.016543103707116955, + 0.015650751390046284, + 0.014886754585050323, + 0.012921465668375697, + 0.01179096764835544, + 0.008761929527224894, + 0.008428120526077129, + 0.007747568693665503, + 0.007074377341536463, + 0.006361685243517791 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.4.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.185265307134786, + "effective_rank_before": 9.185265307134786, + "effective_rank_after": 15.982007344098815, + "nuclear_norm_before_full": 0.37408592365682125, + "nuclear_norm_before": 0.37408592365682125, + "nuclear_norm_after": 0.37408592365682125, + "fro_norm_before_full": 0.16162786250677383, + "fro_norm_after": 0.0936254393353216, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1445360779762268, + 0.05032631382346153, + 0.026562098413705826, + 0.02332642674446106, + 0.015059201046824455, + 0.014035666361451149, + 0.012959537096321583, + 0.011457148008048534, + 0.010993530042469501, + 0.010367263108491898, + 0.01019436027854681, + 0.009835767559707165, + 0.009660337120294571, + 0.00867049116641283, + 0.008469505235552788, + 0.007632199674844742 + ], + "sigma_after": [ + 0.023997409321474122, + 0.02374159151395944, + 0.025251473538192244, + 0.02489632436520832, + 0.02398970855694188, + 0.024009536263834924, + 0.023981113145265954, + 0.023735014778098303, + 0.02358320756788251, + 0.023303199536354378, + 0.023208954993272603, + 0.02298796199831221, + 0.02286666812041292, + 0.022004397332907644, + 0.021789748779574004, + 0.02073961384512983 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.4.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.768263264227116, + "effective_rank_before": 9.768263264227116, + "effective_rank_after": 15.972540071359264, + "nuclear_norm_before_full": 0.4540812624618411, + "nuclear_norm_before": 0.4540812624618411, + "nuclear_norm_after": 0.4540812624618411, + "fro_norm_before_full": 0.18971648298534743, + "fro_norm_after": 0.1137075377684482, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.17084625363349915, + 0.04896681755781174, + 0.02869153395295143, + 0.024551501497626305, + 0.024127647280693054, + 0.020757360383868217, + 0.018513359129428864, + 0.017041347920894623, + 0.016818920150399208, + 0.015876201912760735, + 0.014357805252075195, + 0.013957182876765728, + 0.011179526336491108, + 0.010813258588314056, + 0.009698981419205666, + 0.007883564569056034 + ], + "sigma_after": [ + 0.028993138573720934, + 0.028983469367029377, + 0.030303541257781282, + 0.029531749746193044, + 0.02945100767282651, + 0.02901239210160613, + 0.028974639011037067, + 0.029010253385202765, + 0.029014467329810296, + 0.029014026943897278, + 0.02888815592438279, + 0.02881394789558362, + 0.027515945004084576, + 0.027212191017198337, + 0.026051112967244158, + 0.023311224264242952 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.4.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.603902859912777, + "effective_rank_before": 8.603902859912777, + "effective_rank_after": 15.811506394451701, + "nuclear_norm_before_full": 0.4461012654937804, + "nuclear_norm_before": 0.4461012654937804, + "nuclear_norm_after": 0.4461012654937804, + "fro_norm_before_full": 0.19797483177728833, + "fro_norm_after": 0.11274806331951175, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.17703142762184143, + 0.05901842191815376, + 0.03996724262833595, + 0.02880454808473587, + 0.019876576960086823, + 0.019642548635601997, + 0.01748986728489399, + 0.014093524776399136, + 0.012619085609912872, + 0.011061065830290318, + 0.010005375370383263, + 0.00925319455564022, + 0.008389073424041271, + 0.007037862204015255, + 0.006692884955555201, + 0.005118565633893013 + ], + "sigma_after": [ + 0.030754773905858263, + 0.030633982019317003, + 0.03157003484620623, + 0.031948998822238714, + 0.03071712542557001, + 0.030719030323042785, + 0.03076704173059766, + 0.030435790673268907, + 0.029820798584439725, + 0.02864898072887946, + 0.027479230769064878, + 0.02643751795309016, + 0.02501375795442688, + 0.022285635165802245, + 0.021490906067468875, + 0.017377660524508544 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.4.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.374883800177804, + "effective_rank_before": 13.374883800177804, + "effective_rank_after": 15.99448823379194, + "nuclear_norm_before_full": 0.17001828970387578, + "nuclear_norm_before": 0.17001828970387578, + "nuclear_norm_after": 0.17001828970387578, + "fro_norm_before_full": 0.05211499137415647, + "fro_norm_after": 0.04251915487706004, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03681433945894241, + 0.019174611195921898, + 0.01287316158413887, + 0.012105387635529041, + 0.009705890901386738, + 0.009338782168924809, + 0.008705089800059795, + 0.00809958390891552, + 0.007741529494524002, + 0.007393308915197849, + 0.007180756889283657, + 0.006893410813063383, + 0.006335722282528877, + 0.006140343379229307, + 0.00600316422060132, + 0.0055132070556283 + ], + "sigma_after": [ + 0.010168817806358251, + 0.0100395456512473, + 0.010435728178420527, + 0.010465301937026808, + 0.010898759188489262, + 0.010956417943071783, + 0.010997386660180887, + 0.010952365826153963, + 0.01088914886947601, + 0.010808464816483482, + 0.010753557164291268, + 0.010677334845960864, + 0.01054346363345701, + 0.01050667172603993, + 0.010485139732008176, + 0.010440185725210244 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.4.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.601820029997462, + "effective_rank_before": 10.601820029997462, + "effective_rank_after": 15.992722376199447, + "nuclear_norm_before_full": 0.3322834684513509, + "nuclear_norm_before": 0.3322834684513509, + "nuclear_norm_after": 0.3322834684513509, + "fro_norm_before_full": 0.12641531615582394, + "fro_norm_after": 0.08310861290232009, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10434139519929886, + 0.05039580166339874, + 0.02275325544178486, + 0.02111000195145607, + 0.017123902216553688, + 0.01496034860610962, + 0.013033799827098846, + 0.012784642167389393, + 0.011821470223367214, + 0.01142135914415121, + 0.010842371731996536, + 0.0096172159537673, + 0.009281247854232788, + 0.008209511637687683, + 0.0075011588633060455, + 0.007085985969752073 + ], + "sigma_after": [ + 0.020620609092682332, + 0.0201715337554344, + 0.021857324516168425, + 0.02194991817081908, + 0.021399005073003095, + 0.020979869353981974, + 0.02082996441842317, + 0.020828512064152514, + 0.02084634636790134, + 0.020857380599551483, + 0.020865085117841374, + 0.0207795251699124, + 0.020713421371546425, + 0.020307772603772083, + 0.019827550989007566, + 0.019449649787153248 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.4.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.369114158481871, + "effective_rank_before": 11.369114158481871, + "effective_rank_after": 15.998151294223417, + "nuclear_norm_before_full": 0.29813423473387957, + "nuclear_norm_before": 0.29813423473387957, + "nuclear_norm_after": 0.29813423473387957, + "fro_norm_before_full": 0.11017283421555263, + "fro_norm_after": 0.07454223275484502, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09510556608438492, + 0.024495650082826614, + 0.021881762892007828, + 0.020081214606761932, + 0.015986368060112, + 0.013820108957588673, + 0.01312329713255167, + 0.012447652406990528, + 0.012105414643883705, + 0.011248752474784851, + 0.010984212160110474, + 0.010649856179952621, + 0.009953979402780533, + 0.009582058526575565, + 0.008646056987345219, + 0.008022284135222435 + ], + "sigma_after": [ + 0.018475370185907627, + 0.01861021155937844, + 0.019037958822124515, + 0.01933879422781804, + 0.019189643804052334, + 0.01872705992865356, + 0.01860448414105853, + 0.018518630507438637, + 0.018489436019650123, + 0.0184578407065943, + 0.018458405079726795, + 0.018464038876469892, + 0.018483161668558658, + 0.01848965135543034, + 0.018450332081525855, + 0.018339215769491865 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.4.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.705242242244404, + "effective_rank_before": 8.705242242244404, + "effective_rank_after": 15.928216399392316, + "nuclear_norm_before_full": 0.21306528616696596, + "nuclear_norm_before": 0.21306528616696596, + "nuclear_norm_after": 0.21306528616696593, + "fro_norm_before_full": 0.09468120332144732, + "fro_norm_after": 0.05349636786690498, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08498991280794144, + 0.02843773178756237, + 0.018620040267705917, + 0.012553907930850983, + 0.01011628657579422, + 0.0073135062120854855, + 0.006893672049045563, + 0.006351545453071594, + 0.005963687319308519, + 0.005661467090249062, + 0.005525736603885889, + 0.0047892313450574875, + 0.004480264149606228, + 0.004086330998688936, + 0.003932326100766659, + 0.0033496394753456116 + ], + "sigma_after": [ + 0.014209523218749534, + 0.014143032347683618, + 0.01467202429473852, + 0.014529590389216479, + 0.014204557488425074, + 0.014176254571602296, + 0.014103567524720417, + 0.01393193732934855, + 0.013740643473342593, + 0.013544400843100973, + 0.013441444076271554, + 0.012703736325788601, + 0.012296162356935043, + 0.01168598441293651, + 0.011419115422986062, + 0.010263312091120103 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.5.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.028327453753814, + "effective_rank_before": 10.028327453753814, + "effective_rank_after": 15.994247436129355, + "nuclear_norm_before_full": 0.3253189818933606, + "nuclear_norm_before": 0.3253189818933606, + "nuclear_norm_after": 0.3253189818933606, + "fro_norm_before_full": 0.131502690203537, + "fro_norm_after": 0.08135877137861748, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.11485926806926727, + 0.04242187738418579, + 0.024587370455265045, + 0.017427487298846245, + 0.01583867147564888, + 0.013310338370501995, + 0.012760643847286701, + 0.01144292764365673, + 0.01054080855101347, + 0.010270673781633377, + 0.0098782479763031, + 0.009753954596817493, + 0.008967443369328976, + 0.008223423734307289, + 0.007863019593060017, + 0.007172825746238232 + ], + "sigma_after": [ + 0.02051461476754117, + 0.020139343540854383, + 0.02137548634961772, + 0.02096159243394132, + 0.02066893701617001, + 0.02048021885670519, + 0.020486366356661984, + 0.02051576787096087, + 0.020491398864345933, + 0.020466951210526593, + 0.020411314309765765, + 0.020387929351180927, + 0.020160602616546015, + 0.01978873163016936, + 0.01954178403399428, + 0.018927942684379074 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.5.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.178366934349494, + "effective_rank_before": 12.178366934349494, + "effective_rank_after": 15.995790248610696, + "nuclear_norm_before_full": 0.498339825309813, + "nuclear_norm_before": 0.498339825309813, + "nuclear_norm_after": 0.498339825309813, + "fro_norm_before_full": 0.16571086454649747, + "fro_norm_after": 0.12461773045284105, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12256424129009247, + 0.06830450147390366, + 0.04252782464027405, + 0.033438585698604584, + 0.031082339584827423, + 0.026651659980416298, + 0.022094255313277245, + 0.021626625210046768, + 0.02131751738488674, + 0.020459383726119995, + 0.018526360392570496, + 0.01745660975575447, + 0.015104178339242935, + 0.013344451785087585, + 0.012957800179719925, + 0.010883490554988384 + ], + "sigma_after": [ + 0.02976312844051568, + 0.03015247682316427, + 0.030926603672607036, + 0.031776088180530696, + 0.03224805410753285, + 0.032538178911716814, + 0.031673549978442996, + 0.031559292645119814, + 0.03148577673432015, + 0.031296072946349375, + 0.030995780291898735, + 0.030925232297599192, + 0.030953093648922125, + 0.03093609213836262, + 0.03089608003486877, + 0.030214324457861926 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.5.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.558560430713635, + "effective_rank_before": 10.558560430713635, + "effective_rank_after": 15.971722743891231, + "nuclear_norm_before_full": 0.4871499636210501, + "nuclear_norm_before": 0.4871499636210501, + "nuclear_norm_after": 0.4871499636210501, + "fro_norm_before_full": 0.18374083425291465, + "fro_norm_after": 0.12199287214147135, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1490810513496399, + 0.07406128197908401, + 0.04106384515762329, + 0.030058573931455612, + 0.027050459757447243, + 0.02376464195549488, + 0.019966868683695793, + 0.019202275201678276, + 0.017614997923374176, + 0.015465045347809792, + 0.014896238222718239, + 0.013281894847750664, + 0.012829958461225033, + 0.011290987022221088, + 0.010074432007968426, + 0.007447411771863699 + ], + "sigma_after": [ + 0.030425240845050425, + 0.030035416598058806, + 0.031178266326613066, + 0.03259928259856524, + 0.03224807654753669, + 0.0315460642123362, + 0.030985440062536358, + 0.030953539205284836, + 0.03096102825686126, + 0.030997840302936532, + 0.03097531960991578, + 0.030722365965107648, + 0.030580186542768586, + 0.029762711297269637, + 0.028658998065549016, + 0.024520187184660012 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.5.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 13.5139160771834, + "effective_rank_before": 13.5139160771834, + "effective_rank_after": 15.996316252641309, + "nuclear_norm_before_full": 0.1535264141857624, + "nuclear_norm_before": 0.1535264141857624, + "nuclear_norm_after": 0.1535264141857624, + "fro_norm_before_full": 0.0460243367968967, + "fro_norm_after": 0.0383904527788684, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.029788784682750702, + 0.019194886088371277, + 0.013416135683655739, + 0.010663445107638836, + 0.010345681570470333, + 0.008578798733651638, + 0.007822776213288307, + 0.00735116982832551, + 0.00679699145257473, + 0.0065150330774486065, + 0.006434704642742872, + 0.005866487044841051, + 0.005713953636586666, + 0.005393648520112038, + 0.00509481830522418, + 0.00454909959807992 + ], + "sigma_after": [ + 0.009414685718583692, + 0.00922041236731905, + 0.009423078105944855, + 0.009456196032551753, + 0.009489750867976118, + 0.009844186109321676, + 0.00993284885357251, + 0.00991782212846905, + 0.009828243730550426, + 0.009760310223906537, + 0.00973929449621087, + 0.00958760376472892, + 0.00955075354919196, + 0.009486368323620853, + 0.009446797889545538, + 0.009428062024268614 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.5.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.644672614766082, + "effective_rank_before": 11.644672614766082, + "effective_rank_after": 15.996910261565544, + "nuclear_norm_before_full": 0.35588284861296415, + "nuclear_norm_before": 0.35588284861296415, + "nuclear_norm_after": 0.35588284861296415, + "fro_norm_before_full": 0.12316212931665689, + "fro_norm_after": 0.08898797778119713, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08920659869909286, + 0.06193748116493225, + 0.029297878965735435, + 0.020615780726075172, + 0.018532149493694305, + 0.01695866510272026, + 0.01498779933899641, + 0.014189831912517548, + 0.013772214762866497, + 0.012915004976093769, + 0.012276285327970982, + 0.011311112903058529, + 0.010731680318713188, + 0.01048627495765686, + 0.009427016600966454, + 0.009237073361873627 + ], + "sigma_after": [ + 0.02134802664065963, + 0.02212009943798618, + 0.022106539943576496, + 0.02327070092763291, + 0.023083976477073224, + 0.022751954125577064, + 0.022312813094423352, + 0.022187030355145704, + 0.02214030321588344, + 0.022087822522827285, + 0.022082717563753403, + 0.022107013684334296, + 0.02211989874177058, + 0.02212001679414054, + 0.022039527162683054, + 0.022004407925497023 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.5.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.511242385441427, + "effective_rank_before": 12.511242385441427, + "effective_rank_after": 15.994707186955427, + "nuclear_norm_before_full": 0.29058897821232677, + "nuclear_norm_before": 0.29058897821232677, + "nuclear_norm_after": 0.29058897821232677, + "fro_norm_before_full": 0.09530412931844831, + "fro_norm_after": 0.07267122408130416, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07163133472204208, + 0.03669717162847519, + 0.024701451882719994, + 0.017444202676415443, + 0.015265663154423237, + 0.014389012940227985, + 0.013918542303144932, + 0.01291445642709732, + 0.012590502388775349, + 0.011804627254605293, + 0.011317511089146137, + 0.010586898773908615, + 0.010325485840439796, + 0.00975845381617546, + 0.009498742409050465, + 0.007744920905679464 + ], + "sigma_after": [ + 0.01726723736142477, + 0.01729745498988424, + 0.017963010868255476, + 0.018797745267231925, + 0.018889703038110862, + 0.018774984729802126, + 0.018682936206649393, + 0.01844441202673499, + 0.01836360771259551, + 0.018181244313160438, + 0.018088476241766207, + 0.017992410241080586, + 0.017971998052035653, + 0.01795198208964789, + 0.017952352573101415, + 0.017969422500845305 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.5.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.7244449653874545, + "effective_rank_before": 7.7244449653874545, + "effective_rank_after": 15.785471380834718, + "nuclear_norm_before_full": 0.17988162441179156, + "nuclear_norm_before": 0.17988162441179156, + "nuclear_norm_after": 0.17988162441179156, + "fro_norm_before_full": 0.08822006194757473, + "fro_norm_after": 0.045540654184898215, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08273859322071075, + 0.019055118784308434, + 0.014778594486415386, + 0.009024831466376781, + 0.00740125821903348, + 0.006657992489635944, + 0.0058868746273219585, + 0.005578362382948399, + 0.004791978746652603, + 0.004702304024249315, + 0.0042142923921346664, + 0.0037345909513533115, + 0.0033228013198822737, + 0.002870301716029644, + 0.00276622804813087, + 0.0023575015366077423 + ], + "sigma_after": [ + 0.01265955875198764, + 0.012821496098908333, + 0.013334518738207058, + 0.012653478957493304, + 0.012673722393687194, + 0.01261554769166223, + 0.012406407731491867, + 0.012260282189086204, + 0.011675022491704191, + 0.011586193096627267, + 0.011014260876950113, + 0.010297868343946694, + 0.00955563019788015, + 0.008603791386316768, + 0.008365067979976194, + 0.007358777485866372 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.6.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 3.671162466359668, + "effective_rank_before": 3.671162466359668, + "effective_rank_after": 13.089736569700701, + "nuclear_norm_before_full": 0.2269102171412669, + "nuclear_norm_before": 0.2269102171412669, + "nuclear_norm_after": 0.22691021714126694, + "fro_norm_before_full": 0.15840971533943574, + "fro_norm_after": 0.06736939662972208, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15548518300056458, + 0.02533898502588272, + 0.012690712697803974, + 0.005987302400171757, + 0.00433951523154974, + 0.00390644883736968, + 0.003408931428566575, + 0.003350581508129835, + 0.002669375156983733, + 0.002180491341277957, + 0.0018424750305712223, + 0.0014927524607628584, + 0.0012833818327635527, + 0.001191198593005538, + 0.0009083337499760091, + 0.0008345488458871841 + ], + "sigma_after": [ + 0.029551800687311808, + 0.030852322268295273, + 0.029338118266449074, + 0.022196529048345622, + 0.017406531015618985, + 0.015937039137159187, + 0.01415140560254565, + 0.013935544703291256, + 0.011324161909622222, + 0.009357357494719606, + 0.007959405998660714, + 0.0064855053688478515, + 0.005591759207643095, + 0.005195919884655831, + 0.0039736469672633995, + 0.0036531695808373487 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.6.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.577683057014056, + "effective_rank_before": 11.577683057014056, + "effective_rank_after": 15.998078713567583, + "nuclear_norm_before_full": 0.47243193071335554, + "nuclear_norm_before": 0.47243193071335554, + "nuclear_norm_after": 0.4724319307133555, + "fro_norm_before_full": 0.16924891879336787, + "fro_norm_after": 0.11812226376773838, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14069080352783203, + 0.05166473612189293, + 0.03578891232609749, + 0.030921505764126778, + 0.024915795773267746, + 0.022103890776634216, + 0.02102356031537056, + 0.02038966491818428, + 0.018701275810599327, + 0.017469709739089012, + 0.016548877581954002, + 0.01565832830965519, + 0.01517367921769619, + 0.014782984741032124, + 0.01415633037686348, + 0.012441875413060188 + ], + "sigma_after": [ + 0.029053642067951274, + 0.029095550206658875, + 0.0297853732996388, + 0.03065913456212953, + 0.030502203150462995, + 0.02988578352189096, + 0.029657625820109477, + 0.029541925567580073, + 0.029329912068469167, + 0.02927372696176084, + 0.02927755093407878, + 0.029301562203338533, + 0.02931488343974605, + 0.029321704646386747, + 0.029317822755271887, + 0.02911352950788152 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.6.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.42455484898306, + "effective_rank_before": 8.42455484898306, + "effective_rank_after": 15.941253761026509, + "nuclear_norm_before_full": 0.41244708839803934, + "nuclear_norm_before": 0.41244708839803934, + "nuclear_norm_after": 0.4124470883980393, + "fro_norm_before_full": 0.1906087477094309, + "fro_norm_after": 0.10348099370902449, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.17518295347690582, + 0.05276051536202431, + 0.02468976564705372, + 0.024323124438524246, + 0.018151603639125824, + 0.016361728310585022, + 0.01395508088171482, + 0.012944147922098637, + 0.01115428376942873, + 0.010598933324217796, + 0.009785679168999195, + 0.009746779687702656, + 0.009103268384933472, + 0.008608507923781872, + 0.007727352436631918, + 0.007353364024311304 + ], + "sigma_after": [ + 0.027547024992773894, + 0.027593789630104715, + 0.028086233296276283, + 0.02801978873810445, + 0.02756147395258377, + 0.027594280457111414, + 0.027385555215943068, + 0.027096351913157412, + 0.026106982344446063, + 0.02564626411448571, + 0.02481911046283619, + 0.024774767271897393, + 0.02397503238660083, + 0.023272954426808008, + 0.021828453791992775, + 0.021139025402917316 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.6.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 14.342449464247839, + "effective_rank_before": 14.342449464247839, + "effective_rank_after": 15.995227900246334, + "nuclear_norm_before_full": 0.1515137404203415, + "nuclear_norm_before": 0.1515137404203415, + "nuclear_norm_after": 0.1515137404203415, + "fro_norm_before_full": 0.04262668323128309, + "fro_norm_after": 0.037889665017865545, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.024568989872932434, + 0.016358351334929466, + 0.013471143320202827, + 0.01158978696912527, + 0.009505156427621841, + 0.008989079855382442, + 0.008393418975174427, + 0.007886736653745174, + 0.007531437091529369, + 0.007270543836057186, + 0.0066025881096720695, + 0.006469315383583307, + 0.006079559680074453, + 0.0057744975201785564, + 0.00575291458517313, + 0.005270220804959536 + ], + "sigma_after": [ + 0.00927015524756449, + 0.008919506477966748, + 0.009145681293406208, + 0.009275451827127884, + 0.00933222245491886, + 0.009427708208064144, + 0.009568623039333424, + 0.009680923004152547, + 0.009735478533434497, + 0.009755662208331095, + 0.009714888076539611, + 0.009691213308580283, + 0.009599445900878098, + 0.00951352654901401, + 0.009507295232776635, + 0.009375959058252956 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.6.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.996183411286806, + "effective_rank_before": 11.996183411286806, + "effective_rank_after": 15.99637922057554, + "nuclear_norm_before_full": 0.35123754385858774, + "nuclear_norm_before": 0.35123754385858774, + "nuclear_norm_after": 0.35123754385858774, + "fro_norm_before_full": 0.11990520902262285, + "fro_norm_after": 0.08782930274360282, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.09185424447059631, + 0.050679147243499756, + 0.02604808285832405, + 0.02100623957812786, + 0.018938789144158363, + 0.016092214733362198, + 0.01581723988056183, + 0.014547832310199738, + 0.01432038564234972, + 0.013109501451253891, + 0.012834513559937477, + 0.012550758197903633, + 0.011867820285260677, + 0.010768351145088673, + 0.010628068819642067, + 0.010174354538321495 + ], + "sigma_after": [ + 0.020977636345447637, + 0.021360627206209855, + 0.022035505325810327, + 0.02290335433973604, + 0.022882735791193605, + 0.022329518092884918, + 0.0222635874978095, + 0.02198914650476193, + 0.021948989586230518, + 0.02180213150263917, + 0.021785587688332952, + 0.021774795074120192, + 0.021771443594890885, + 0.021801270814549408, + 0.021804419425015785, + 0.021806795068955023 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.6.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.79146480173734, + "effective_rank_before": 12.79146480173734, + "effective_rank_after": 15.994729528703246, + "nuclear_norm_before_full": 0.28657278046011925, + "nuclear_norm_before": 0.28657278046011925, + "nuclear_norm_after": 0.28657278046011925, + "fro_norm_before_full": 0.0921372082742882, + "fro_norm_after": 0.07166670880017924, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06836877018213272, + 0.03543872386217117, + 0.022163528949022293, + 0.01798882707953453, + 0.015682099387049675, + 0.014840024523437023, + 0.013387169688940048, + 0.012960684485733509, + 0.0123281916603446, + 0.012040180154144764, + 0.011672516353428364, + 0.010792975313961506, + 0.010676994919776917, + 0.010275385342538357, + 0.009433033876121044, + 0.008523674681782722 + ], + "sigma_after": [ + 0.01700452211916087, + 0.017022845837166477, + 0.01767911218274935, + 0.018290322604271134, + 0.018615749850014324, + 0.01859827820710145, + 0.018373201167569346, + 0.01827308595788322, + 0.01811356614548477, + 0.018041056957546552, + 0.017952584444155296, + 0.017778859516153756, + 0.01776141443101747, + 0.01771259076521321, + 0.017668910124316255, + 0.017686680150315753 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.6.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.017429392852181, + "effective_rank_before": 11.017429392852181, + "effective_rank_after": 15.991268594217741, + "nuclear_norm_before_full": 0.15941164805553854, + "nuclear_norm_before": 0.15941164805553854, + "nuclear_norm_after": 0.15941164805553854, + "fro_norm_before_full": 0.058204021619482814, + "fro_norm_after": 0.0398744750673379, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.04722258076071739, + 0.02016313374042511, + 0.014502164907753468, + 0.011599376797676086, + 0.00992782972753048, + 0.007837685756385326, + 0.006856925785541534, + 0.00613710330799222, + 0.006009343545883894, + 0.005193253513425589, + 0.00507021090015769, + 0.0045601618476212025, + 0.00422517629340291, + 0.0036060793790966272, + 0.003455184865742922, + 0.0030454369261860847 + ], + "sigma_after": [ + 0.009862931956981, + 0.009716998931217377, + 0.010032168455176045, + 0.010339996082339143, + 0.010569367832794475, + 0.010296275942792586, + 0.010100414953058868, + 0.01003593514847192, + 0.01003295916805054, + 0.010047598649378856, + 0.010049509148658346, + 0.010027339658320013, + 0.009964351497059421, + 0.009669801261152331, + 0.009551318615766484, + 0.009114680754321142 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.7.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.323474176185443, + "effective_rank_before": 7.323474176185443, + "effective_rank_after": 15.899110933214825, + "nuclear_norm_before_full": 0.31175852939486504, + "nuclear_norm_before": 0.31175852939486504, + "nuclear_norm_after": 0.31175852939486504, + "fro_norm_before_full": 0.16245888458939498, + "fro_norm_after": 0.0784251266254206, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.15606780350208282, + 0.025957703590393066, + 0.017127221450209618, + 0.015244729816913605, + 0.012512567453086376, + 0.010699277743697166, + 0.009447365067899227, + 0.008990149945020676, + 0.008854161016643047, + 0.008038125932216644, + 0.0070596178993582726, + 0.006713199429214001, + 0.006595320533961058, + 0.006491646636277437, + 0.005995944608002901, + 0.005963694769889116 + ], + "sigma_after": [ + 0.021706842622546546, + 0.022759181576757825, + 0.021640594892895604, + 0.02164978942842351, + 0.021600299473306796, + 0.021154884257667393, + 0.020460374359393536, + 0.02010419123362383, + 0.019986524722566373, + 0.01916034502946979, + 0.017880578011124942, + 0.0173482535346871, + 0.017157493169352146, + 0.016985669391660765, + 0.01611168412684039, + 0.016051823564548505 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.7.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.663690169924434, + "effective_rank_before": 11.663690169924434, + "effective_rank_after": 15.996450202917218, + "nuclear_norm_before_full": 0.4843954248353839, + "nuclear_norm_before": 0.4843954248353839, + "nuclear_norm_after": 0.4843954248353839, + "fro_norm_before_full": 0.17276100570562403, + "fro_norm_after": 0.12112580294316712, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14241062104701996, + 0.060523226857185364, + 0.028713062405586243, + 0.025815855711698532, + 0.025372236967086792, + 0.02274969033896923, + 0.022244831547141075, + 0.020914940163493156, + 0.019199302420020103, + 0.018755381926894188, + 0.0178852342069149, + 0.017356738448143005, + 0.016199717298150063, + 0.015965640544891357, + 0.015642737969756126, + 0.014646206982433796 + ], + "sigma_after": [ + 0.029669364150588573, + 0.028986481271212075, + 0.03159634170483031, + 0.03131362580198396, + 0.031232381193545763, + 0.030654773760983578, + 0.03054211329830585, + 0.030277860404450553, + 0.030052202343772375, + 0.030019701164136794, + 0.02998640869695203, + 0.02998356077059143, + 0.030007551953415004, + 0.030014626627162616, + 0.03002387597903928, + 0.030034555714413692 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.7.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 9.323098274742415, + "effective_rank_before": 9.323098274742415, + "effective_rank_after": 15.911381812864558, + "nuclear_norm_before_full": 0.46646530646830797, + "nuclear_norm_before": 0.46646530646830797, + "nuclear_norm_after": 0.46646530646830797, + "fro_norm_before_full": 0.19539464430351133, + "fro_norm_after": 0.11723096970002105, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.16826575994491577, + 0.07540794461965561, + 0.028413763269782066, + 0.02461901493370533, + 0.02421722188591957, + 0.022155240178108215, + 0.018645957112312317, + 0.01706269010901451, + 0.01657279022037983, + 0.01413007266819477, + 0.013579363003373146, + 0.011455245316028595, + 0.009796086698770523, + 0.007841203361749649, + 0.007371015381067991, + 0.006931937765330076 + ], + "sigma_after": [ + 0.03085684351370016, + 0.029718446181239956, + 0.03206911996395796, + 0.03130527430835213, + 0.03123214596663367, + 0.030944430461449846, + 0.030851364377726554, + 0.030887903362371582, + 0.030887776531218956, + 0.03061292208581683, + 0.030447864649422333, + 0.029241409799081162, + 0.02748234576196394, + 0.024277633607303086, + 0.02331092723753446, + 0.022338898660535376 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.7.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.648131297800001, + "effective_rank_before": 11.648131297800001, + "effective_rank_after": 15.995170940550805, + "nuclear_norm_before_full": 0.12799991946667433, + "nuclear_norm_before": 0.12799991946667433, + "nuclear_norm_after": 0.12799991946667433, + "fro_norm_before_full": 0.04399795915612631, + "fro_norm_after": 0.03200965568710417, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03208271041512489, + 0.020639779046177864, + 0.011392337270081043, + 0.008595745079219341, + 0.006887887138873339, + 0.006777484901249409, + 0.006151795852929354, + 0.005159545689821243, + 0.004912707023322582, + 0.004505469463765621, + 0.004198877140879631, + 0.004064273089170456, + 0.0036922963336110115, + 0.0032616297248750925, + 0.002969469642266631, + 0.002707911655306816 + ], + "sigma_after": [ + 0.007680804053972288, + 0.007954472445756896, + 0.007962765761114375, + 0.008236839368910433, + 0.008358103646142092, + 0.00834383248733141, + 0.008221524765916815, + 0.008008424033196657, + 0.007978300358347939, + 0.007958078137289071, + 0.007962113456373316, + 0.007966261990919297, + 0.00797079872916257, + 0.007921579485728907, + 0.007819321156809219, + 0.007656699589703036 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.7.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.148633107540903, + "effective_rank_before": 11.148633107540903, + "effective_rank_after": 15.996895466428889, + "nuclear_norm_before_full": 0.3517133938148618, + "nuclear_norm_before": 0.3517133938148618, + "nuclear_norm_after": 0.3517133938148618, + "fro_norm_before_full": 0.12847431227199382, + "fro_norm_after": 0.08794551988237573, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.10301940143108368, + 0.05444863811135292, + 0.023333553224802017, + 0.020315002650022507, + 0.01699143834412098, + 0.016856586560606956, + 0.015039168298244476, + 0.01411092933267355, + 0.012952658347785473, + 0.01258842647075653, + 0.011524784378707409, + 0.01109487283974886, + 0.01077276561409235, + 0.009976442903280258, + 0.009745858609676361, + 0.008942866697907448 + ], + "sigma_after": [ + 0.021367590416267276, + 0.02144727139558364, + 0.022897066378753838, + 0.022962740244114148, + 0.022349090960327295, + 0.022319089630309256, + 0.02197849443703606, + 0.02188075737455747, + 0.021843519623668053, + 0.02184753086801694, + 0.021875731191919708, + 0.021881800704768795, + 0.021878736412664117, + 0.021822090571556055, + 0.0217873096621937, + 0.021574573943125473 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.7.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.222244746265575, + "effective_rank_before": 12.222244746265575, + "effective_rank_after": 15.99522018322619, + "nuclear_norm_before_full": 0.268612798769027, + "nuclear_norm_before": 0.268612798769027, + "nuclear_norm_after": 0.268612798769027, + "fro_norm_before_full": 0.08956808105959332, + "fro_norm_after": 0.0671732508571804, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.06749774515628815, + 0.03583557903766632, + 0.020479684695601463, + 0.018903527408838272, + 0.01494496688246727, + 0.013941553421318531, + 0.012531699612736702, + 0.012374281883239746, + 0.01190995518118143, + 0.010900516994297504, + 0.010338155552744865, + 0.009546958841383457, + 0.008400593884289265, + 0.0076965456828475, + 0.007190255913883448, + 0.0061207786202430725 + ], + "sigma_after": [ + 0.016000074457014388, + 0.016119485846468936, + 0.016707541786082823, + 0.016931880136003406, + 0.0175285859782692, + 0.017459663546415694, + 0.01718619193548659, + 0.017148091817161613, + 0.01703383917319379, + 0.01680736937475112, + 0.01671498118206963, + 0.01664355877109309, + 0.016646566585961507, + 0.016662361620830967, + 0.01664311811804808, + 0.016379488440176136 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.7.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.572537732536098, + "effective_rank_before": 8.572537732536098, + "effective_rank_after": 15.830524899569497, + "nuclear_norm_before_full": 0.14871397952083498, + "nuclear_norm_before": 0.14871397952083498, + "nuclear_norm_after": 0.14871397952083498, + "fro_norm_before_full": 0.06454933273892698, + "fro_norm_after": 0.03754764768023175, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.054776038974523544, + 0.026079094037413597, + 0.015119425021111965, + 0.008020094595849514, + 0.006959802005439997, + 0.005718839820474386, + 0.004752644337713718, + 0.004530976060777903, + 0.003932716324925423, + 0.0037799386773258448, + 0.0032785178627818823, + 0.0028860385064035654, + 0.0025190571323037148, + 0.002362982602789998, + 0.0022725909948349, + 0.00172522256616503 + ], + "sigma_after": [ + 0.010287770964564416, + 0.010006414764036527, + 0.010327130026506925, + 0.010441304799322012, + 0.01031498975347774, + 0.010321603460098524, + 0.010251057673030271, + 0.010190854562993687, + 0.009887010028536503, + 0.00976882442355649, + 0.009239947422912415, + 0.008657693494750695, + 0.007968420484255744, + 0.007631888155565256, + 0.007425170215394194, + 0.005993899291833567 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.8.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.203221096382869, + "effective_rank_before": 8.203221096382869, + "effective_rank_after": 15.94082038730028, + "nuclear_norm_before_full": 0.3286078115925193, + "nuclear_norm_before": 0.3286078115925193, + "nuclear_norm_after": 0.32860781159251934, + "fro_norm_before_full": 0.15524246725593405, + "fro_norm_after": 0.08244782762655731, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.14379988610744476, + 0.041575416922569275, + 0.01991771161556244, + 0.016597965732216835, + 0.014377130195498466, + 0.012569491751492023, + 0.010971778072416782, + 0.009343581274151802, + 0.008717898279428482, + 0.008291559293866158, + 0.008031242527067661, + 0.0077183847315609455, + 0.007501406595110893, + 0.0069787525571882725, + 0.006499180104583502, + 0.005716425832360983 + ], + "sigma_after": [ + 0.02208273973568972, + 0.022114358286650097, + 0.022481235081850667, + 0.02209804567935511, + 0.022104730404800065, + 0.022101954187218182, + 0.021858963736894433, + 0.021111884122794272, + 0.020634351079669117, + 0.02023713855264889, + 0.019964103740902513, + 0.019604103420131365, + 0.019333445710025408, + 0.01860882152029566, + 0.017851284952277082, + 0.016420651381316716 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.8.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.221233979363426, + "effective_rank_before": 11.221233979363426, + "effective_rank_after": 15.998290342923802, + "nuclear_norm_before_full": 0.5780786592513323, + "nuclear_norm_before": 0.5780786592513323, + "nuclear_norm_after": 0.5780786592513324, + "fro_norm_before_full": 0.21895513091653518, + "fro_norm_after": 0.14453523840947374, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1932550072669983, + 0.04510357230901718, + 0.03593818470835686, + 0.030482854694128036, + 0.028573503717780113, + 0.02652563899755478, + 0.026003733277320862, + 0.024103665724396706, + 0.023589981719851494, + 0.02311445213854313, + 0.022424811497330666, + 0.02143135480582714, + 0.02090984582901001, + 0.019987722858786583, + 0.018956774845719337, + 0.017677554860711098 + ], + "sigma_after": [ + 0.03584033013487509, + 0.03661057357534832, + 0.03769387663275199, + 0.036936521058794025, + 0.036525279949088656, + 0.036133579334488845, + 0.036052305074325046, + 0.03584392062535833, + 0.035812806798880645, + 0.03579335724216476, + 0.035779938434540186, + 0.03578606878660634, + 0.03579768798304314, + 0.03582328246237674, + 0.035841635170901935, + 0.03580749598778833 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.8.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.487341756695843, + "effective_rank_before": 11.487341756695843, + "effective_rank_after": 15.996780747333734, + "nuclear_norm_before_full": 0.5731630241498351, + "nuclear_norm_before": 0.5731630241498351, + "nuclear_norm_after": 0.5731630241498351, + "fro_norm_before_full": 0.2061390442389166, + "fro_norm_after": 0.14331976277150152, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1706707626581192, + 0.06732119619846344, + 0.0415571965277195, + 0.034582749009132385, + 0.03186291083693504, + 0.028303071856498718, + 0.026456749066710472, + 0.024376435205340385, + 0.022301429882645607, + 0.02157045342028141, + 0.020746048539876938, + 0.019141368567943573, + 0.01789938472211361, + 0.01673964224755764, + 0.015038629993796349, + 0.01459499541670084 + ], + "sigma_after": [ + 0.03518305713853398, + 0.03481104584533115, + 0.03649442166103604, + 0.037408306145314846, + 0.03723313742081673, + 0.036556120025366595, + 0.03614021346558166, + 0.035752220911845316, + 0.03553571451721345, + 0.035505703540430436, + 0.0354973232408007, + 0.035531538165363044, + 0.03555895930973829, + 0.03553172747701495, + 0.035276870870092805, + 0.035146664415355196 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.8.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.864005770732241, + "effective_rank_before": 11.864005770732241, + "effective_rank_after": 15.994630860435453, + "nuclear_norm_before_full": 0.1333275637589395, + "nuclear_norm_before": 0.1333275637589395, + "nuclear_norm_after": 0.1333275637589395, + "fro_norm_before_full": 0.04552448511952715, + "fro_norm_after": 0.033343109724310926, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.035184379667043686, + 0.016263024881482124, + 0.011726842261850834, + 0.009782752953469753, + 0.008043902926146984, + 0.00784322340041399, + 0.006569176446646452, + 0.005645988509058952, + 0.005138631444424391, + 0.005065150558948517, + 0.004580593202263117, + 0.00442216731607914, + 0.0036077757831662893, + 0.00345206493511796, + 0.0031360345892608166, + 0.0028658548835664988 + ], + "sigma_after": [ + 0.008016087536242611, + 0.00800548444405484, + 0.00830696794103246, + 0.008421563273536981, + 0.008729393886042562, + 0.008744172921727173, + 0.008624359581395971, + 0.008410624974458636, + 0.008329841302609988, + 0.00832254323716094, + 0.008302965792685389, + 0.008305285703963761, + 0.008303559433055396, + 0.008281314827326672, + 0.00818722645761517, + 0.008036172446030955 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.8.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.691808482784955, + "effective_rank_before": 10.691808482784955, + "effective_rank_after": 15.994989432237427, + "nuclear_norm_before_full": 0.38260423857718706, + "nuclear_norm_before": 0.38260423857718706, + "nuclear_norm_after": 0.38260423857718706, + "fro_norm_before_full": 0.14680260292708486, + "fro_norm_after": 0.09568099882315416, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.12621185183525085, + 0.044262297451496124, + 0.02728571742773056, + 0.025297624990344048, + 0.0207512229681015, + 0.01901896856725216, + 0.016554372385144234, + 0.014171255752444267, + 0.013577940873801708, + 0.012668179348111153, + 0.011862569488584995, + 0.011657141149044037, + 0.010860331356525421, + 0.01054734829813242, + 0.009318338707089424, + 0.008559077978134155 + ], + "sigma_after": [ + 0.023835716500059584, + 0.023724155831439657, + 0.024888956994903275, + 0.025089244529385095, + 0.02465944444952016, + 0.024291421498042073, + 0.0238991338379941, + 0.02383215256764925, + 0.023848021125469066, + 0.023865230937487037, + 0.023843041844301204, + 0.0238273509487787, + 0.023710000811402433, + 0.023633325208099694, + 0.023105274759364514, + 0.022551766733291182 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.8.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 11.996866334139822, + "effective_rank_before": 11.996866334139822, + "effective_rank_after": 15.995977548756377, + "nuclear_norm_before_full": 0.3032745160162449, + "nuclear_norm_before": 0.3032745160162449, + "nuclear_norm_after": 0.3032745160162449, + "fro_norm_before_full": 0.10362256081545441, + "fro_norm_after": 0.07583765986452096, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.08071409910917282, + 0.0401131771504879, + 0.02119273692369461, + 0.020378364250063896, + 0.017970407381653786, + 0.01461851317435503, + 0.014095893129706383, + 0.013561293482780457, + 0.012525763362646103, + 0.011359215714037418, + 0.011249915696680546, + 0.010210863314568996, + 0.009973185136914253, + 0.0089256651699543, + 0.008317864499986172, + 0.008067558519542217 + ], + "sigma_after": [ + 0.018178538213561746, + 0.01811416020613035, + 0.019255190178351474, + 0.01942000594188405, + 0.019780026102782483, + 0.01944093130845857, + 0.019317165779436686, + 0.019189200980499387, + 0.018969413758960555, + 0.018818664379979852, + 0.018811125014168047, + 0.018790707577805273, + 0.018795930782749146, + 0.018821267866263296, + 0.018799472891096385, + 0.018772715034117607 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.8.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.151143300780227, + "effective_rank_before": 8.151143300780227, + "effective_rank_after": 15.71924595337533, + "nuclear_norm_before_full": 0.14305305352900177, + "nuclear_norm_before": 0.14305305352900177, + "nuclear_norm_after": 0.14305305352900177, + "fro_norm_before_full": 0.06601509037348245, + "fro_norm_after": 0.036340797375643503, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.05960961431264877, + 0.021054750308394432, + 0.010397881269454956, + 0.007574109360575676, + 0.006901764310896397, + 0.005981450900435448, + 0.004941149614751339, + 0.0049042971804738045, + 0.003854799084365368, + 0.0036156668793410063, + 0.0033926162868738174, + 0.0028509506955742836, + 0.0025536594912409782, + 0.0020507201552391052, + 0.001838377327658236, + 0.0015312463510781527 + ], + "sigma_after": [ + 0.010098921204477751, + 0.009948144412727462, + 0.010605114123404362, + 0.01013631588939645, + 0.010093899704802673, + 0.01010517328100974, + 0.010054493742711975, + 0.010047619275207526, + 0.009557709842567314, + 0.009341686274863523, + 0.009096762939955582, + 0.008311130033111524, + 0.007758290391824432, + 0.006623622786735642, + 0.006071704557472168, + 0.005202465068733645 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.9.mlp.down_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 7.1388219985346515, + "effective_rank_before": 7.1388219985346515, + "effective_rank_after": 15.844995575585155, + "nuclear_norm_before_full": 0.36304950108751655, + "nuclear_norm_before": 0.36304950108751655, + "nuclear_norm_after": 0.36304950108751655, + "fro_norm_before_full": 0.18872685740407077, + "fro_norm_after": 0.09163760460267224, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1796736717224121, + 0.03638778254389763, + 0.02909057028591633, + 0.014938433654606342, + 0.013658040203154087, + 0.011941100470721722, + 0.010164943523705006, + 0.008897265419363976, + 0.008654636330902576, + 0.008421051315963268, + 0.0075560216791927814, + 0.007196747697889805, + 0.006945119705051184, + 0.006679678335785866, + 0.006570850964635611, + 0.006273587234318256 + ], + "sigma_after": [ + 0.026085909967152876, + 0.027017842787032127, + 0.02729754841988624, + 0.026033535848526735, + 0.02584775879084673, + 0.025228377287937313, + 0.02393928906932596, + 0.022507471586849927, + 0.022179422930218354, + 0.021846603286739714, + 0.02046639514650992, + 0.01982410185351228, + 0.019350076955797903, + 0.01882853153519211, + 0.018608357312926836, + 0.017988278309061564 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.9.mlp.gate_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.987447763830989, + "effective_rank_before": 10.987447763830989, + "effective_rank_after": 15.998049410236456, + "nuclear_norm_before_full": 0.5952203702181578, + "nuclear_norm_before": 0.5952203702181578, + "nuclear_norm_after": 0.5952203702181577, + "fro_norm_before_full": 0.22641733580305404, + "fro_norm_after": 0.14882342124388195, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.19711431860923767, + 0.058936506509780884, + 0.04304024204611778, + 0.03666553646326065, + 0.028804047033190727, + 0.025631163269281387, + 0.02432318963110447, + 0.02307462878525257, + 0.021761277690529823, + 0.021415313705801964, + 0.021047471091151237, + 0.01996377296745777, + 0.019879858940839767, + 0.018927615135908127, + 0.018489647656679153, + 0.016145780682563782 + ], + "sigma_after": [ + 0.03699083547026161, + 0.03696093267202653, + 0.038435134935401925, + 0.038888407494361864, + 0.03755092864720259, + 0.03706028170662658, + 0.036964709518095246, + 0.03693354220568848, + 0.03694908521072436, + 0.036958071628733706, + 0.03696843092976097, + 0.03699435466647406, + 0.03699551894376125, + 0.03699153152241912, + 0.036974954285124914, + 0.036603650381494525 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.9.mlp.up_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.11348260414068, + "effective_rank_before": 10.11348260414068, + "effective_rank_after": 15.99452922483867, + "nuclear_norm_before_full": 0.6067985892295837, + "nuclear_norm_before": 0.6067985892295837, + "nuclear_norm_after": 0.6067985892295837, + "fro_norm_before_full": 0.24740486376208543, + "fro_norm_after": 0.15175123177746486, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.2212173491716385, + 0.06448053568601608, + 0.04073980450630188, + 0.03145817667245865, + 0.029901180416345596, + 0.028249135240912437, + 0.02603982761502266, + 0.023549407720565796, + 0.021510016173124313, + 0.02003409154713154, + 0.01901382952928543, + 0.0185005534440279, + 0.01731439121067524, + 0.016489842906594276, + 0.014481141231954098, + 0.013819306157529354 + ], + "sigma_after": [ + 0.038130943397981676, + 0.03811342173318969, + 0.040125573241737884, + 0.03871757890126503, + 0.03845012681920659, + 0.03823824993339857, + 0.038096689387864646, + 0.03810544248675669, + 0.0381498402011563, + 0.03811758000279425, + 0.03802149131607582, + 0.037940488142171415, + 0.03764532233963562, + 0.03733231181504808, + 0.036096835894362585, + 0.03551669361693895 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.9.self_attn.k_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.128181409623297, + "effective_rank_before": 12.128181409623297, + "effective_rank_after": 15.995067528558705, + "nuclear_norm_before_full": 0.13295034994371235, + "nuclear_norm_before": 0.13295034994371235, + "nuclear_norm_after": 0.13295034994371235, + "fro_norm_before_full": 0.04454007756896348, + "fro_norm_after": 0.033247844749719035, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.03353556618094444, + 0.017698442563414574, + 0.011283362284302711, + 0.008807043544948101, + 0.007978582754731178, + 0.0071045695804059505, + 0.006529573816806078, + 0.005717776715755463, + 0.005345203913748264, + 0.005065406672656536, + 0.0049185859970748425, + 0.004507352132350206, + 0.004109788220375776, + 0.003836696967482567, + 0.003422982059419155, + 0.003089416539296508 + ], + "sigma_after": [ + 0.007930131903489113, + 0.00797934356962457, + 0.008245401994344799, + 0.008511463248147194, + 0.00865595317329248, + 0.008672453375358985, + 0.008585173686856244, + 0.008393786264961806, + 0.008316617044322745, + 0.008275157514890302, + 0.008260346020866836, + 0.008244113200744356, + 0.008252955270600139, + 0.008258608191104166, + 0.008230655948307092, + 0.008138189536801519 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.9.self_attn.o_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 10.85737906031727, + "effective_rank_before": 10.85737906031727, + "effective_rank_after": 15.995872111850174, + "nuclear_norm_before_full": 0.35414919070899487, + "nuclear_norm_before": 0.35414919070899487, + "nuclear_norm_after": 0.35414919070899487, + "fro_norm_before_full": 0.13170686492655537, + "fro_norm_after": 0.08856029804062629, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.1055346205830574, + 0.05765315145254135, + 0.02292776107788086, + 0.021427474915981293, + 0.01829078234732151, + 0.01723315939307213, + 0.014500275254249573, + 0.013234179466962814, + 0.012515353038907051, + 0.012005377560853958, + 0.01120215468108654, + 0.010786264203488827, + 0.01025986298918724, + 0.00914775300770998, + 0.008856971748173237, + 0.008574048988521099 + ], + "sigma_after": [ + 0.02154225560058798, + 0.02179006469683297, + 0.023186345683754467, + 0.02320008426691814, + 0.022731596546805535, + 0.02249814640548569, + 0.022068887308681288, + 0.022029046942053494, + 0.022042004664724245, + 0.022056567056050722, + 0.022066920641154942, + 0.02205525699335354, + 0.022011314184610643, + 0.02175028806556954, + 0.021630868388699615, + 0.021489543263712083 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.9.self_attn.q_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 12.040007151617852, + "effective_rank_before": 12.040007151617852, + "effective_rank_after": 15.994789098560595, + "nuclear_norm_before_full": 0.28956578532233834, + "nuclear_norm_before": 0.28956578532233834, + "nuclear_norm_after": 0.2895657853223383, + "fro_norm_before_full": 0.0979591290208377, + "fro_norm_after": 0.07241503661855421, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.07476378232240677, + 0.03911648690700531, + 0.022166693583130836, + 0.01919700764119625, + 0.016987895593047142, + 0.015766002237796783, + 0.013286152854561806, + 0.012822582386434078, + 0.012058934196829796, + 0.011336061172187328, + 0.010385829955339432, + 0.009923744946718216, + 0.009461155161261559, + 0.008610131219029427, + 0.007454072590917349, + 0.006229252554476261 + ], + "sigma_after": [ + 0.017305934103643034, + 0.017408545638180107, + 0.018073265806045336, + 0.018587947638713003, + 0.018918560554332404, + 0.018914791873459725, + 0.01846625145318032, + 0.01835403439019583, + 0.018184849145077127, + 0.01806234494038764, + 0.017978854375957868, + 0.017969941631804293, + 0.017976437862140905, + 0.018000116093950577, + 0.01792798855014248, + 0.017435921265127656 + ], + "saved_update_relative_error": 0.0 + }, + "base_model.model.model.layers.9.self_attn.v_proj": { + "rank_before_full": 16, + "rank_after": 16, + "effective_rank_before_full": 8.471001310628166, + "effective_rank_before": 8.471001310628166, + "effective_rank_after": 15.743837722444868, + "nuclear_norm_before_full": 0.14770925662014633, + "nuclear_norm_before": 0.14770925662014633, + "nuclear_norm_after": 0.14770925662014633, + "fro_norm_before_full": 0.064774868375983, + "fro_norm_after": 0.0374721900018868, + "preserve_nuclear_norm": true, + "hns_strength": 1.0, + "fast_steps": 2, + "stable_steps": 1, + "fast_coefficients": [ + 3.4445, + -4.775, + 2.0315 + ], + "stable_coefficients": [ + 2.0, + -1.5, + 0.5 + ], + "sigma_before": [ + 0.05579628422856331, + 0.02373361587524414, + 0.016659531742334366, + 0.006641983520239592, + 0.006473241373896599, + 0.006091481074690819, + 0.005090781021863222, + 0.004781774710863829, + 0.004251533187925816, + 0.0038440015632659197, + 0.0035032033920288086, + 0.0030096585396677256, + 0.00232863100245595, + 0.002036745660007, + 0.0019134307513013482, + 0.0015533589757978916 + ], + "sigma_after": [ + 0.010393218168584288, + 0.009993069624002383, + 0.010390219807699148, + 0.010385859497055379, + 0.010385851579671859, + 0.010393458879423629, + 0.010382211949938838, + 0.010333907266712622, + 0.010149063173487651, + 0.009886757735315423, + 0.009566089821240098, + 0.008910731056164996, + 0.007594754677918236, + 0.006880623060173717, + 0.006552659858003349, + 0.005510780464754724 + ], + "saved_update_relative_error": 0.0 + } + } +}