Upload EEE results (en1_en2_codeswitching_134k)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- eval_results/eee/en1_en2_codeswitching_134k_eclektic/eclektic/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f46b08ce-4bbb-4d62-a8ef-078128288e46.json +98 -0
- eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_0_ar_0_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/5888c7a7-d9a9-4ab8-8c31-dabad5834f3c.json +82 -0
- eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_0_ar_1000_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f7cb7b77-4b87-4f31-857b-3201102b013e.json +82 -0
- eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_1000_ar_0_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/91009629-515f-4767-ab1b-682d8ae1d860.json +82 -0
- eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_1000_ar_1000_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a56badcb-e7f1-41ee-b6b0-d7d1bf6f6b35.json +82 -0
- eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/7c926f4d-a402-4075-823b-5c9ac507b490.json +57 -0
- eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_business/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/909c8cad-2e72-4f9f-968d-7e52fe2b1b47.json +59 -0
- eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_humanities/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/4435a853-564a-4c05-b56c-984b9be5387d.json +59 -0
- eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_medical/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/2d2743ec-7fe5-4393-bfb2-7298fec67607.json +59 -0
- eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_other/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/088071d8-f67a-45a5-8757-6ecee3708cc0.json +59 -0
- eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_social_sciences/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/23e1b161-f466-4179-ab1c-cf610616d8b5.json +59 -0
- eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_stem/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f0f44275-3260-47a6-9c05-7488e4da27a7.json +59 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_cloze/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/dc095e1b-8f5b-4364-a188-2f58eddaaee9.json +107 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/d0d66fb7-318c-48bd-8f57-4d28c9d55efd.json +107 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_acm_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1eb9496f-16b4-44c1-a74f-79768c126f81.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_acq_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a85063d4-99d6-4598-93b0-10458863594e.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_aeb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/bb806ef6-ccba-4f38-abc2-00b73d6ea968.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_afb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/5bb99884-6b9b-45bc-93e2-9b3f01941d43.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_als_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/80aa9f7f-9766-4507-8eec-c6d35354d72a.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_amh_ethi/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/b101eb56-82c2-42af-97c3-2b5c4f08eb48.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_jord/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a177e0c6-8b19-426a-8242-20c70c33a691.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_leba/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/6777b630-006a-4367-9af7-6059a360659b.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_pale/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/7f4d9aa0-9b28-4d75-9456-2f841417650c.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_syri/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/0bc560fb-b190-4316-a7ba-eeeb858b1a9a.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_arb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ce969175-055f-4845-a1f4-d68d6511d312.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_arq_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/10c59c16-bf7f-4eed-8ce4-357247e49601.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ars_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f20de927-88de-440c-8dab-151884632007.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ary_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ff60ec2c-2662-4850-994d-733cbbf8402d.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_arz_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ea096d50-6e8e-4cfd-90d0-d3a048839b1d.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_asm_beng/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/c97b5c42-c9ae-44ab-80c9-86693feed2cd.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_azj_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/203b9f07-5f99-475c-9e96-06d3a5815bbb.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bam_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/cf9985bc-7f53-43b5-b61d-e569cae40157.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bcc_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/9ede2730-ec07-4fe1-9642-5290c17bc798.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bel_cyrl/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/da343631-9b59-4997-a7b4-b835cbd099c0.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ben_beng/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/e19e6702-fda2-4626-8e98-2cbe999f25a4.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ben_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/7cbaa339-cfc1-49f7-b4ca-2843c99711d5.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bgc_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/367511d2-a584-4739-9a35-750efb5a5474.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bho_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/8ec8c036-743c-4ffa-b4dd-ab90ce9a6378.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bos_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/3199788f-bab0-429e-857b-e33e1dc4c7b7.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bra_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a08efb06-c824-4b18-8146-34501af45688.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bsk_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f86660b3-6295-4d9f-bfdc-44d184272d47.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_btx_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ecfd4ddf-b071-4b49-b3e5-7ea649363569.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bul_cyrl/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/518529aa-ce65-45ed-86d2-1c486b5cd0c1.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cat_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/e47055c1-5d91-4159-853d-cc2921752fb1.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ces_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/e1f63e4f-41dc-44b5-90e9-d87c9f65a30e.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ckb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/44d3a3b5-fd16-444a-9ab8-437ba6cb6a3f.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ckm_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/824a6ce4-714c-4719-b880-853575c8ff7c.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cls_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/9e475b8b-140c-4067-a728-acfbc15a81d6.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cmn_hans/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/d1c092ad-0ef8-4c72-8af6-80dbfd23264b.json +113 -0
- eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cmn_hant/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/d5d4b001-6d7f-4f89-a87e-35f664734587.json +113 -0
eval_results/eee/en1_en2_codeswitching_134k_eclektic/eclektic/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f46b08ce-4bbb-4d62-a8ef-078128288e46.json
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "eclektic//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468568.402025",
|
| 4 |
+
"evaluation_timestamp": "1786467721",
|
| 5 |
+
"retrieved_timestamp": "1786468568.402025",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "eclektic",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "eclektic",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786467721",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "eclektic_transfer",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.12245337837975227,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"num_samples": 4608
|
| 49 |
+
}
|
| 50 |
+
},
|
| 51 |
+
"generation_config": {
|
| 52 |
+
"generation_args": {
|
| 53 |
+
"temperature": 0.0,
|
| 54 |
+
"max_tokens": 64,
|
| 55 |
+
"max_attempts": 1
|
| 56 |
+
},
|
| 57 |
+
"additional_details": {
|
| 58 |
+
"until": "[\"\\n\"]",
|
| 59 |
+
"do_sample": "false",
|
| 60 |
+
"num_fewshot": "0"
|
| 61 |
+
}
|
| 62 |
+
}
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"evaluation_name": "eclektic",
|
| 66 |
+
"source_data": {
|
| 67 |
+
"dataset_name": "eclektic",
|
| 68 |
+
"source_type": "other"
|
| 69 |
+
},
|
| 70 |
+
"evaluation_timestamp": "1786467721",
|
| 71 |
+
"metric_config": {
|
| 72 |
+
"evaluation_description": "eclektic_overall",
|
| 73 |
+
"lower_is_better": false,
|
| 74 |
+
"score_type": "continuous",
|
| 75 |
+
"min_score": 0.0,
|
| 76 |
+
"max_score": 1.0
|
| 77 |
+
},
|
| 78 |
+
"score_details": {
|
| 79 |
+
"score": 0.006872514817612356,
|
| 80 |
+
"uncertainty": {
|
| 81 |
+
"num_samples": 4608
|
| 82 |
+
}
|
| 83 |
+
},
|
| 84 |
+
"generation_config": {
|
| 85 |
+
"generation_args": {
|
| 86 |
+
"temperature": 0.0,
|
| 87 |
+
"max_tokens": 64,
|
| 88 |
+
"max_attempts": 1
|
| 89 |
+
},
|
| 90 |
+
"additional_details": {
|
| 91 |
+
"until": "[\"\\n\"]",
|
| 92 |
+
"do_sample": "false",
|
| 93 |
+
"num_fewshot": "0"
|
| 94 |
+
}
|
| 95 |
+
}
|
| 96 |
+
}
|
| 97 |
+
]
|
| 98 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_0_ar_0_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/5888c7a7-d9a9-4ab8-8c31-dabad5834f3c.json
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "fictive_en_0_ar_0_mcq_en//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468569.151101",
|
| 4 |
+
"evaluation_timestamp": "1786468307",
|
| 5 |
+
"retrieved_timestamp": "1786468569.151101",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "fictive_en_0_ar_0_mcq_en",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "fictive_en_0_ar_0_mcq_en",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786468307",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "acc",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.6469230769230769,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"standard_error": {
|
| 49 |
+
"value": 0.00937471014234386,
|
| 50 |
+
"method": "bootstrap"
|
| 51 |
+
},
|
| 52 |
+
"num_samples": 2600
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"evaluation_name": "fictive_en_0_ar_0_mcq_en",
|
| 58 |
+
"source_data": {
|
| 59 |
+
"dataset_name": "fictive_en_0_ar_0_mcq_en",
|
| 60 |
+
"source_type": "other"
|
| 61 |
+
},
|
| 62 |
+
"evaluation_timestamp": "1786468307",
|
| 63 |
+
"metric_config": {
|
| 64 |
+
"evaluation_description": "acc_norm",
|
| 65 |
+
"lower_is_better": false,
|
| 66 |
+
"score_type": "continuous",
|
| 67 |
+
"min_score": 0.0,
|
| 68 |
+
"max_score": 1.0
|
| 69 |
+
},
|
| 70 |
+
"score_details": {
|
| 71 |
+
"score": 0.6053846153846154,
|
| 72 |
+
"uncertainty": {
|
| 73 |
+
"standard_error": {
|
| 74 |
+
"value": 0.009587371512181838,
|
| 75 |
+
"method": "bootstrap"
|
| 76 |
+
},
|
| 77 |
+
"num_samples": 2600
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
}
|
| 81 |
+
]
|
| 82 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_0_ar_1000_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f7cb7b77-4b87-4f31-857b-3201102b013e.json
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "fictive_en_0_ar_1000_mcq_en//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468569.151365",
|
| 4 |
+
"evaluation_timestamp": "1786468307",
|
| 5 |
+
"retrieved_timestamp": "1786468569.151365",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "fictive_en_0_ar_1000_mcq_en",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "fictive_en_0_ar_1000_mcq_en",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786468307",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "acc",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.8638461538461538,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"standard_error": {
|
| 49 |
+
"value": 0.006727133080376297,
|
| 50 |
+
"method": "bootstrap"
|
| 51 |
+
},
|
| 52 |
+
"num_samples": 2600
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"evaluation_name": "fictive_en_0_ar_1000_mcq_en",
|
| 58 |
+
"source_data": {
|
| 59 |
+
"dataset_name": "fictive_en_0_ar_1000_mcq_en",
|
| 60 |
+
"source_type": "other"
|
| 61 |
+
},
|
| 62 |
+
"evaluation_timestamp": "1786468307",
|
| 63 |
+
"metric_config": {
|
| 64 |
+
"evaluation_description": "acc_norm",
|
| 65 |
+
"lower_is_better": false,
|
| 66 |
+
"score_type": "continuous",
|
| 67 |
+
"min_score": 0.0,
|
| 68 |
+
"max_score": 1.0
|
| 69 |
+
},
|
| 70 |
+
"score_details": {
|
| 71 |
+
"score": 0.7861538461538462,
|
| 72 |
+
"uncertainty": {
|
| 73 |
+
"standard_error": {
|
| 74 |
+
"value": 0.00804269045113095,
|
| 75 |
+
"method": "bootstrap"
|
| 76 |
+
},
|
| 77 |
+
"num_samples": 2600
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
}
|
| 81 |
+
]
|
| 82 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_1000_ar_0_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/91009629-515f-4767-ab1b-682d8ae1d860.json
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "fictive_en_1000_ar_0_mcq_en//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468569.151295",
|
| 4 |
+
"evaluation_timestamp": "1786468307",
|
| 5 |
+
"retrieved_timestamp": "1786468569.151295",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "fictive_en_1000_ar_0_mcq_en",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "fictive_en_1000_ar_0_mcq_en",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786468307",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "acc",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.72,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"standard_error": {
|
| 49 |
+
"value": 0.008807286500294434,
|
| 50 |
+
"method": "bootstrap"
|
| 51 |
+
},
|
| 52 |
+
"num_samples": 2600
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"evaluation_name": "fictive_en_1000_ar_0_mcq_en",
|
| 58 |
+
"source_data": {
|
| 59 |
+
"dataset_name": "fictive_en_1000_ar_0_mcq_en",
|
| 60 |
+
"source_type": "other"
|
| 61 |
+
},
|
| 62 |
+
"evaluation_timestamp": "1786468307",
|
| 63 |
+
"metric_config": {
|
| 64 |
+
"evaluation_description": "acc_norm",
|
| 65 |
+
"lower_is_better": false,
|
| 66 |
+
"score_type": "continuous",
|
| 67 |
+
"min_score": 0.0,
|
| 68 |
+
"max_score": 1.0
|
| 69 |
+
},
|
| 70 |
+
"score_details": {
|
| 71 |
+
"score": 0.6653846153846154,
|
| 72 |
+
"uncertainty": {
|
| 73 |
+
"standard_error": {
|
| 74 |
+
"value": 0.0092556343705144,
|
| 75 |
+
"method": "bootstrap"
|
| 76 |
+
},
|
| 77 |
+
"num_samples": 2600
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
}
|
| 81 |
+
]
|
| 82 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_fictive/fictive_en_1000_ar_1000_mcq_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a56badcb-e7f1-41ee-b6b0-d7d1bf6f6b35.json
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "fictive_en_1000_ar_1000_mcq_en//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468569.151411",
|
| 4 |
+
"evaluation_timestamp": "1786468307",
|
| 5 |
+
"retrieved_timestamp": "1786468569.151411",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "fictive_en_1000_ar_1000_mcq_en",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "fictive_en_1000_ar_1000_mcq_en",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786468307",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "acc",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.916923076923077,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"standard_error": {
|
| 49 |
+
"value": 0.00541381627574876,
|
| 50 |
+
"method": "bootstrap"
|
| 51 |
+
},
|
| 52 |
+
"num_samples": 2600
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"evaluation_name": "fictive_en_1000_ar_1000_mcq_en",
|
| 58 |
+
"source_data": {
|
| 59 |
+
"dataset_name": "fictive_en_1000_ar_1000_mcq_en",
|
| 60 |
+
"source_type": "other"
|
| 61 |
+
},
|
| 62 |
+
"evaluation_timestamp": "1786468307",
|
| 63 |
+
"metric_config": {
|
| 64 |
+
"evaluation_description": "acc_norm",
|
| 65 |
+
"lower_is_better": false,
|
| 66 |
+
"score_type": "continuous",
|
| 67 |
+
"min_score": 0.0,
|
| 68 |
+
"max_score": 1.0
|
| 69 |
+
},
|
| 70 |
+
"score_details": {
|
| 71 |
+
"score": 0.8492307692307692,
|
| 72 |
+
"uncertainty": {
|
| 73 |
+
"standard_error": {
|
| 74 |
+
"value": 0.007018851925209192,
|
| 75 |
+
"method": "bootstrap"
|
| 76 |
+
},
|
| 77 |
+
"num_samples": 2600
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
}
|
| 81 |
+
]
|
| 82 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/7c926f4d-a402-4075-823b-5c9ac507b490.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_mmlu_en//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468570.241986",
|
| 4 |
+
"evaluation_timestamp": "1786467210",
|
| 5 |
+
"retrieved_timestamp": "1786468570.241986",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_mmlu_en",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_mmlu_en",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786467210",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "acc",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.26,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"standard_error": {
|
| 49 |
+
"value": 0.02189143420210505,
|
| 50 |
+
"method": "bootstrap"
|
| 51 |
+
},
|
| 52 |
+
"num_samples": 400
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
]
|
| 57 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_business/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/909c8cad-2e72-4f9f-968d-7e52fe2b1b47.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_mmlu_en_business//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468570.241604",
|
| 4 |
+
"evaluation_timestamp": "1786467210",
|
| 5 |
+
"retrieved_timestamp": "1786468570.241604",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_mmlu_en_business",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_mmlu_en_business",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467210",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.27586206896551724,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05919966036406566,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 58
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
]
|
| 59 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_humanities/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/4435a853-564a-4c05-b56c-984b9be5387d.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_mmlu_en_humanities//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468570.241789",
|
| 4 |
+
"evaluation_timestamp": "1786467210",
|
| 5 |
+
"retrieved_timestamp": "1786468570.241789",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_mmlu_en_humanities",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_mmlu_en_humanities",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467210",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.22549019607843138,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04158307533083289,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 102
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
]
|
| 59 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_medical/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/2d2743ec-7fe5-4393-bfb2-7298fec67607.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_mmlu_en_medical//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468570.241843",
|
| 4 |
+
"evaluation_timestamp": "1786467210",
|
| 5 |
+
"retrieved_timestamp": "1786468570.241843",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_mmlu_en_medical",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_mmlu_en_medical",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467210",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.19444444444444445,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.0668977476599572,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 36
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
]
|
| 59 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_other/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/088071d8-f67a-45a5-8757-6ecee3708cc0.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_mmlu_en_other//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468570.241885",
|
| 4 |
+
"evaluation_timestamp": "1786467210",
|
| 5 |
+
"retrieved_timestamp": "1786468570.241885",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_mmlu_en_other",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_mmlu_en_other",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467210",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.375,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.06527912098338669,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 56
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
]
|
| 59 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_social_sciences/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/23e1b161-f466-4179-ab1c-cf610616d8b5.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_mmlu_en_social_sciences//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468570.241958",
|
| 4 |
+
"evaluation_timestamp": "1786467210",
|
| 5 |
+
"retrieved_timestamp": "1786468570.241958",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_mmlu_en_social_sciences",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_mmlu_en_social_sciences",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467210",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.21568627450980393,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04092563958237658,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 102
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
]
|
| 59 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_mmlu_en/global_mmlu_en_stem/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f0f44275-3260-47a6-9c05-7488e4da27a7.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_mmlu_en_stem//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468570.241914",
|
| 4 |
+
"evaluation_timestamp": "1786467210",
|
| 5 |
+
"retrieved_timestamp": "1786468570.241914",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_mmlu_en_stem",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_mmlu_en_stem",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467210",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.32608695652173914,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.06988152725357213,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 46
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
]
|
| 59 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_cloze/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/dc095e1b-8f5b-4364-a188-2f58eddaaee9.json
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_cloze//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.637387",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.637387",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_cloze",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_cloze",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786467255",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "acc",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.3439869405288952,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"standard_error": {
|
| 49 |
+
"value": 0.002750968916335563,
|
| 50 |
+
"method": "bootstrap"
|
| 51 |
+
},
|
| 52 |
+
"num_samples": 27091
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"evaluation_name": "global_piqa_cloze",
|
| 58 |
+
"source_data": {
|
| 59 |
+
"dataset_name": "global_piqa_cloze",
|
| 60 |
+
"source_type": "other"
|
| 61 |
+
},
|
| 62 |
+
"evaluation_timestamp": "1786467255",
|
| 63 |
+
"metric_config": {
|
| 64 |
+
"evaluation_description": "acc_norm",
|
| 65 |
+
"lower_is_better": false,
|
| 66 |
+
"score_type": "continuous",
|
| 67 |
+
"min_score": 0.0,
|
| 68 |
+
"max_score": 1.0
|
| 69 |
+
},
|
| 70 |
+
"score_details": {
|
| 71 |
+
"score": 0.37141325193935343,
|
| 72 |
+
"uncertainty": {
|
| 73 |
+
"standard_error": {
|
| 74 |
+
"value": 0.0027908258568468485,
|
| 75 |
+
"method": "bootstrap"
|
| 76 |
+
},
|
| 77 |
+
"num_samples": 27091
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"evaluation_name": "global_piqa_cloze",
|
| 83 |
+
"source_data": {
|
| 84 |
+
"dataset_name": "global_piqa_cloze",
|
| 85 |
+
"source_type": "other"
|
| 86 |
+
},
|
| 87 |
+
"evaluation_timestamp": "1786467255",
|
| 88 |
+
"metric_config": {
|
| 89 |
+
"evaluation_description": "acc_bytes",
|
| 90 |
+
"lower_is_better": false,
|
| 91 |
+
"score_type": "continuous",
|
| 92 |
+
"min_score": 0.0,
|
| 93 |
+
"max_score": 1.0
|
| 94 |
+
},
|
| 95 |
+
"score_details": {
|
| 96 |
+
"score": 0.3710917946818136,
|
| 97 |
+
"uncertainty": {
|
| 98 |
+
"standard_error": {
|
| 99 |
+
"value": 0.0027875606507870377,
|
| 100 |
+
"method": "bootstrap"
|
| 101 |
+
},
|
| 102 |
+
"num_samples": 27091
|
| 103 |
+
}
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
]
|
| 107 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/d0d66fb7-318c-48bd-8f57-4d28c9d55efd.json
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.637343",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.637343",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze",
|
| 35 |
+
"source_type": "other"
|
| 36 |
+
},
|
| 37 |
+
"evaluation_timestamp": "1786467255",
|
| 38 |
+
"metric_config": {
|
| 39 |
+
"evaluation_description": "acc",
|
| 40 |
+
"lower_is_better": false,
|
| 41 |
+
"score_type": "continuous",
|
| 42 |
+
"min_score": 0.0,
|
| 43 |
+
"max_score": 1.0
|
| 44 |
+
},
|
| 45 |
+
"score_details": {
|
| 46 |
+
"score": 0.47419117647058817,
|
| 47 |
+
"uncertainty": {
|
| 48 |
+
"standard_error": {
|
| 49 |
+
"value": 0.004236343339452992,
|
| 50 |
+
"method": "bootstrap"
|
| 51 |
+
},
|
| 52 |
+
"num_samples": 13600
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"evaluation_name": "global_piqa_nonparallel_cloze",
|
| 58 |
+
"source_data": {
|
| 59 |
+
"dataset_name": "global_piqa_nonparallel_cloze",
|
| 60 |
+
"source_type": "other"
|
| 61 |
+
},
|
| 62 |
+
"evaluation_timestamp": "1786467255",
|
| 63 |
+
"metric_config": {
|
| 64 |
+
"evaluation_description": "acc_norm",
|
| 65 |
+
"lower_is_better": false,
|
| 66 |
+
"score_type": "continuous",
|
| 67 |
+
"min_score": 0.0,
|
| 68 |
+
"max_score": 1.0
|
| 69 |
+
},
|
| 70 |
+
"score_details": {
|
| 71 |
+
"score": 0.514264705882353,
|
| 72 |
+
"uncertainty": {
|
| 73 |
+
"standard_error": {
|
| 74 |
+
"value": 0.004270349391532596,
|
| 75 |
+
"method": "bootstrap"
|
| 76 |
+
},
|
| 77 |
+
"num_samples": 13600
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"evaluation_name": "global_piqa_nonparallel_cloze",
|
| 83 |
+
"source_data": {
|
| 84 |
+
"dataset_name": "global_piqa_nonparallel_cloze",
|
| 85 |
+
"source_type": "other"
|
| 86 |
+
},
|
| 87 |
+
"evaluation_timestamp": "1786467255",
|
| 88 |
+
"metric_config": {
|
| 89 |
+
"evaluation_description": "acc_bytes",
|
| 90 |
+
"lower_is_better": false,
|
| 91 |
+
"score_type": "continuous",
|
| 92 |
+
"min_score": 0.0,
|
| 93 |
+
"max_score": 1.0
|
| 94 |
+
},
|
| 95 |
+
"score_details": {
|
| 96 |
+
"score": 0.5145588235294118,
|
| 97 |
+
"uncertainty": {
|
| 98 |
+
"standard_error": {
|
| 99 |
+
"value": 0.004265743005000673,
|
| 100 |
+
"method": "bootstrap"
|
| 101 |
+
},
|
| 102 |
+
"num_samples": 13600
|
| 103 |
+
}
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
]
|
| 107 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_acm_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1eb9496f-16b4-44c1-a74f-79768c126f81.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_acm_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.630985",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.630985",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_acm_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_acm_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.44,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.049888765156985884,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_acm_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_acm_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.49,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_acm_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_acm_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.5,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.050251890762960605,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_acq_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a85063d4-99d6-4598-93b0-10458863594e.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_acq_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631028",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631028",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_acq_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_acq_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.5,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.050251890762960605,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_acq_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_acq_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.49,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_acq_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_acq_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.49,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_aeb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/bb806ef6-ccba-4f38-abc2-00b73d6ea968.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_aeb_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631072",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631072",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_aeb_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_aeb_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.43,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.049756985195624305,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_aeb_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_aeb_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.5,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.050251890762960605,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_aeb_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_aeb_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.5,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.050251890762960605,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_afb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/5bb99884-6b9b-45bc-93e2-9b3f01941d43.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_afb_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.63111",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.63111",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_afb_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_afb_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.5,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.050251890762960605,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_afb_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_afb_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.49,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_afb_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_afb_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.49,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_als_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/80aa9f7f-9766-4507-8eec-c6d35354d72a.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_als_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631153",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631153",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_als_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_als_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.55,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_als_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_als_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.5,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.050251890762960605,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_als_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_als_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.49,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_amh_ethi/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/b101eb56-82c2-42af-97c3-2b5c4f08eb48.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_amh_ethi//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631203",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631203",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_amh_ethi",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_amh_ethi",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.3,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04605661864718382,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_amh_ethi",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_amh_ethi",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.53,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05016135580465919,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_amh_ethi",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_amh_ethi",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.52,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05021167315686783,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_jord/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a177e0c6-8b19-426a-8242-20c70c33a691.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_apc_arab_jord//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631248",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631248",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_jord",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_jord",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.45,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_jord",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_jord",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.51,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_jord",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_jord",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.49,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_leba/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/6777b630-006a-4367-9af7-6059a360659b.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_apc_arab_leba//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631286",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631286",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_leba",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_leba",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.36,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.048241815132442176,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_leba",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_leba",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.54,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05009082659620331,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_leba",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_leba",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.53,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05016135580465919,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_pale/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/7f4d9aa0-9b28-4d75-9456-2f841417650c.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_apc_arab_pale//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631329",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631329",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_pale",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_pale",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.41,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04943110704237104,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_pale",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_pale",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.38,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.04878317312145634,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_pale",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_pale",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.36,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.048241815132442176,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_apc_arab_syri/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/0bc560fb-b190-4316-a7ba-eeeb858b1a9a.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_apc_arab_syri//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.63137",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.63137",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_syri",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_syri",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.43,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.049756985195624305,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_syri",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_syri",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.43,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.049756985195624305,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_apc_arab_syri",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_apc_arab_syri",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.42,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.04960449637488583,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_arb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ce969175-055f-4845-a1f4-d68d6511d312.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_arb_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631411",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631411",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arb_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arb_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.42,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04960449637488583,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arb_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arb_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.43,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.049756985195624305,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arb_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arb_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.45,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_arq_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/10c59c16-bf7f-4eed-8ce4-357247e49601.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_arq_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631454",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631454",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arq_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arq_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.4,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.0492365963917331,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arq_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arq_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.51,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arq_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arq_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.49,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ars_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f20de927-88de-440c-8dab-151884632007.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_ars_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631497",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631497",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ars_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ars_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.47,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05016135580465919,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ars_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ars_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.49,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ars_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ars_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.48,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05021167315686783,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ary_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ff60ec2c-2662-4850-994d-733cbbf8402d.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_ary_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631537",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631537",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ary_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ary_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.41,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04943110704237104,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ary_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ary_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.54,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05009082659620331,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ary_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ary_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.52,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05021167315686783,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_arz_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ea096d50-6e8e-4cfd-90d0-d3a048839b1d.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_arz_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631578",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631578",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arz_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arz_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.53,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05016135580465919,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arz_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arz_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.51,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_arz_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_arz_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.49,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_asm_beng/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/c97b5c42-c9ae-44ab-80c9-86693feed2cd.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_asm_beng//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631619",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631619",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_asm_beng",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_asm_beng",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.4,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.0492365963917331,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_asm_beng",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_asm_beng",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.55,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_asm_beng",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_asm_beng",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.55,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_azj_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/203b9f07-5f99-475c-9e96-06d3a5815bbb.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_azj_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631663",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631663",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_azj_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_azj_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.55,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_azj_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_azj_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.54,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05009082659620331,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_azj_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_azj_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.51,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bam_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/cf9985bc-7f53-43b5-b61d-e569cae40157.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bam_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631702",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631702",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bam_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bam_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.49,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05024183937956913,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bam_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bam_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.49,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05024183937956913,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bam_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bam_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.51,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bcc_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/9ede2730-ec07-4fe1-9642-5290c17bc798.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bcc_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631746",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631746",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bcc_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bcc_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.46,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05009082659620332,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bcc_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bcc_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.54,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05009082659620331,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bcc_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bcc_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.54,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05009082659620331,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bel_cyrl/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/da343631-9b59-4997-a7b4-b835cbd099c0.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bel_cyrl//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631806",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631806",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bel_cyrl",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bel_cyrl",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.59,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04943110704237104,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bel_cyrl",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bel_cyrl",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.62,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.04878317312145634,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bel_cyrl",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bel_cyrl",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.62,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.04878317312145634,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ben_beng/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/e19e6702-fda2-4626-8e98-2cbe999f25a4.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_ben_beng//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.631957",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.631957",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ben_beng",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ben_beng",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.32,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04688261722621507,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ben_beng",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ben_beng",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.53,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05016135580465919,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ben_beng",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ben_beng",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.51,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ben_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/7cbaa339-cfc1-49f7-b4ca-2843c99711d5.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_ben_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632005",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632005",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ben_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ben_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.52,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05021167315686783,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ben_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ben_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.48,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05021167315686783,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ben_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ben_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.48,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05021167315686783,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bgc_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/367511d2-a584-4739-9a35-750efb5a5474.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bgc_deva//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632045",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632045",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bgc_deva",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bgc_deva",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.36,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.048241815132442176,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bgc_deva",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bgc_deva",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.41,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.04943110704237104,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bgc_deva",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bgc_deva",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.38,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.04878317312145634,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bho_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/8ec8c036-743c-4ffa-b4dd-ab90ce9a6378.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bho_deva//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632091",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632091",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bho_deva",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bho_deva",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.44,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.049888765156985884,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bho_deva",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bho_deva",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.52,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05021167315686783,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bho_deva",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bho_deva",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.49,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bos_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/3199788f-bab0-429e-857b-e33e1dc4c7b7.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bos_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.63213",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.63213",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bos_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bos_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.47,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05016135580465919,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bos_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bos_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.46,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05009082659620332,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bos_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bos_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.43,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.049756985195624305,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bra_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/a08efb06-c824-4b18-8146-34501af45688.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bra_deva//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632173",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632173",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bra_deva",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bra_deva",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.42,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04960449637488583,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bra_deva",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bra_deva",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.48,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05021167315686783,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bra_deva",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bra_deva",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.47,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05016135580465919,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bsk_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/f86660b3-6295-4d9f-bfdc-44d184272d47.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bsk_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632218",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632218",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bsk_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bsk_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.51,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05024183937956913,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bsk_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bsk_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.53,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05016135580465919,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bsk_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bsk_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.52,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05021167315686783,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_btx_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/ecfd4ddf-b071-4b49-b3e5-7ea649363569.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_btx_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632259",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632259",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_btx_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_btx_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.37,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.048523658709390974,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_btx_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_btx_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.56,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.049888765156985884,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_btx_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_btx_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.56,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.049888765156985884,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_bul_cyrl/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/518529aa-ce65-45ed-86d2-1c486b5cd0c1.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_bul_cyrl//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632303",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632303",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bul_cyrl",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bul_cyrl",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.49,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05024183937956913,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bul_cyrl",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bul_cyrl",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.47,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05016135580465919,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_bul_cyrl",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_bul_cyrl",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.47,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05016135580465919,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cat_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/e47055c1-5d91-4159-853d-cc2921752fb1.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_cat_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632345",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632345",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cat_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cat_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.47,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05016135580465919,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cat_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cat_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.54,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05009082659620331,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cat_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cat_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.51,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05024183937956913,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ces_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/e1f63e4f-41dc-44b5-90e9-d87c9f65a30e.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_ces_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632384",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632384",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ces_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ces_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.46,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05009082659620332,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ces_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ces_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.55,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ces_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ces_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.52,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05021167315686783,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ckb_arab/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/44d3a3b5-fd16-444a-9ab8-437ba6cb6a3f.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_ckb_arab//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.63243",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.63243",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ckb_arab",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ckb_arab",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.44,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.049888765156985884,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ckb_arab",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ckb_arab",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.44,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.049888765156985884,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ckb_arab",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ckb_arab",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.45,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_ckm_latn/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/824a6ce4-714c-4719-b880-853575c8ff7c.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_ckm_latn//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.63247",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.63247",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ckm_latn",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ckm_latn",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.39,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04902071300001973,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ckm_latn",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ckm_latn",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.41,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.04943110704237104,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_ckm_latn",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_ckm_latn",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.45,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cls_deva/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/9e475b8b-140c-4067-a728-acfbc15a81d6.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_cls_deva//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632525",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632525",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cls_deva",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cls_deva",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.59,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.04943110704237104,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cls_deva",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cls_deva",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.56,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.049888765156985884,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cls_deva",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cls_deva",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.56,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.049888765156985884,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cmn_hans/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/d1c092ad-0ef8-4c72-8af6-80dbfd23264b.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_cmn_hans//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632562",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632562",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cmn_hans",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cmn_hans",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.49,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05024183937956913,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cmn_hans",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cmn_hans",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.48,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05021167315686783,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cmn_hans",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cmn_hans",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.48,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05021167315686783,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|
eval_results/eee/en1_en2_codeswitching_134k_piqa/global_piqa_nonparallel_cloze_cmn_hant/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/d5d4b001-6d7f-4f89-a87e-35f664734587.json
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": "0.2.2",
|
| 3 |
+
"evaluation_id": "global_piqa_nonparallel_cloze_cmn_hant//gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k/1786468571.632606",
|
| 4 |
+
"evaluation_timestamp": "1786467255",
|
| 5 |
+
"retrieved_timestamp": "1786468571.632606",
|
| 6 |
+
"source_metadata": {
|
| 7 |
+
"source_name": "lm-evaluation-harness",
|
| 8 |
+
"source_type": "evaluation_run",
|
| 9 |
+
"source_organization_name": "unknown",
|
| 10 |
+
"evaluator_relationship": "third_party"
|
| 11 |
+
},
|
| 12 |
+
"eval_library": {
|
| 13 |
+
"name": "lm_eval",
|
| 14 |
+
"version": "0.4.13.dev0"
|
| 15 |
+
},
|
| 16 |
+
"model_info": {
|
| 17 |
+
"name": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 18 |
+
"id": "/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k",
|
| 19 |
+
"developer": "",
|
| 20 |
+
"inference_engine": {
|
| 21 |
+
"name": "transformers"
|
| 22 |
+
},
|
| 23 |
+
"additional_details": {
|
| 24 |
+
"num_parameters": "6291689472",
|
| 25 |
+
"dtype": "torch.bfloat16",
|
| 26 |
+
"revision": "main",
|
| 27 |
+
"model_args": "{'pretrained': '/gpfs/ess6000-1/proj/dmfexp/trAr/torchtitan-mulitlingual/outputs/hf_export/llama3_7b_en1_en2_codeswitching_134k', 'dtype': 'bfloat16'}"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"evaluation_results": [
|
| 31 |
+
{
|
| 32 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cmn_hant",
|
| 33 |
+
"source_data": {
|
| 34 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cmn_hant",
|
| 35 |
+
"source_type": "hf_dataset",
|
| 36 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 37 |
+
"hf_split": "test"
|
| 38 |
+
},
|
| 39 |
+
"evaluation_timestamp": "1786467255",
|
| 40 |
+
"metric_config": {
|
| 41 |
+
"evaluation_description": "acc",
|
| 42 |
+
"lower_is_better": false,
|
| 43 |
+
"score_type": "continuous",
|
| 44 |
+
"min_score": 0.0,
|
| 45 |
+
"max_score": 1.0
|
| 46 |
+
},
|
| 47 |
+
"score_details": {
|
| 48 |
+
"score": 0.54,
|
| 49 |
+
"uncertainty": {
|
| 50 |
+
"standard_error": {
|
| 51 |
+
"value": 0.05009082659620331,
|
| 52 |
+
"method": "bootstrap"
|
| 53 |
+
},
|
| 54 |
+
"num_samples": 100
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cmn_hant",
|
| 60 |
+
"source_data": {
|
| 61 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cmn_hant",
|
| 62 |
+
"source_type": "hf_dataset",
|
| 63 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 64 |
+
"hf_split": "test"
|
| 65 |
+
},
|
| 66 |
+
"evaluation_timestamp": "1786467255",
|
| 67 |
+
"metric_config": {
|
| 68 |
+
"evaluation_description": "acc_norm",
|
| 69 |
+
"lower_is_better": false,
|
| 70 |
+
"score_type": "continuous",
|
| 71 |
+
"min_score": 0.0,
|
| 72 |
+
"max_score": 1.0
|
| 73 |
+
},
|
| 74 |
+
"score_details": {
|
| 75 |
+
"score": 0.47,
|
| 76 |
+
"uncertainty": {
|
| 77 |
+
"standard_error": {
|
| 78 |
+
"value": 0.05016135580465919,
|
| 79 |
+
"method": "bootstrap"
|
| 80 |
+
},
|
| 81 |
+
"num_samples": 100
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"evaluation_name": "global_piqa_nonparallel_cloze_cmn_hant",
|
| 87 |
+
"source_data": {
|
| 88 |
+
"dataset_name": "global_piqa_nonparallel_cloze_cmn_hant",
|
| 89 |
+
"source_type": "hf_dataset",
|
| 90 |
+
"hf_repo": "mrlbenchmarks/global-piqa-nonparallel",
|
| 91 |
+
"hf_split": "test"
|
| 92 |
+
},
|
| 93 |
+
"evaluation_timestamp": "1786467255",
|
| 94 |
+
"metric_config": {
|
| 95 |
+
"evaluation_description": "acc_bytes",
|
| 96 |
+
"lower_is_better": false,
|
| 97 |
+
"score_type": "continuous",
|
| 98 |
+
"min_score": 0.0,
|
| 99 |
+
"max_score": 1.0
|
| 100 |
+
},
|
| 101 |
+
"score_details": {
|
| 102 |
+
"score": 0.47,
|
| 103 |
+
"uncertainty": {
|
| 104 |
+
"standard_error": {
|
| 105 |
+
"value": 0.05016135580465919,
|
| 106 |
+
"method": "bootstrap"
|
| 107 |
+
},
|
| 108 |
+
"num_samples": 100
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
}
|