enable env: START_REPETITION_PENALTY

llm_toolkit/eval.py

@@ -28,6 +28,7 @@ results_path = os.getenv("RESULTS_PATH")
 batch_size = int(os.getenv("BATCH_SIZE", 1))
 use_english_datasets = os.getenv("USE_ENGLISH_DATASETS") == "true"
 max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", 2048))
+start_repetition_penalty = float(os.getenv("START_REPETITION_PENALTY", 1.0))
 
 print(
     model_name,
@@ -93,7 +94,7 @@ evaluate_model_with_repetition_penalty(
     model_name,
     datasets["test"],
     on_repetition_penalty_step_completed,
-    start_repetition_penalty=1.0,
+    start_repetition_penalty=start_repetition_penalty,
     end_repetition_penalty=1.3,
     step_repetition_penalty=0.02,
     batch_size=batch_size,

results/mac-results_metrics.csv

@@ -1,77 +1,80 @@
 model,rpp,meteor,bleu_1,rouge_l,ews_score,repetition_score,total_repetitions,rap,num_max_output_tokens
-01-ai/Yi-1.5-9B-Chat,1.00,0.3463725436435439,0.09312113035602035,0.33279900782244626,0.0,0.35127978817299205,0.35127978817299205,0.34125573890735983,2
-01-ai/Yi-1.5-9B-Chat,1.02,0.3471185374158656,0.09126513887574451,0.3329223332114953,0.0,0.264783759929391,0.264783759929391,0.3432230230787291,4
-01-ai/Yi-1.5-9B-Chat,1.04,0.3471882673119874,0.09019886552461354,0.3321870466419108,0.0,0.37775816416593117,0.37775816416593117,0.3416859125059273,8
-01-ai/Yi-1.5-9B-Chat,1.06,0.3475947948648639,0.09004996084071014,0.33143152244770613,0.0,0.46866725507502205,0.46866725507502205,0.34081549880066125,9
-01-ai/Yi-1.5-9B-Chat,1.08,0.34751102711658816,0.09004837948083254,0.33169125948525535,0.0,0.3115622241835834,0.3115622241835834,0.3429415142848335,4
-01-ai/Yi-1.5-9B-Chat,1.10,0.3483026982644252,0.08832819752923171,0.3313149728213199,0.0,0.265666372462489,0.265666372462489,0.3443811795403635,6
-01-ai/Yi-1.5-9B-Chat,1.12,0.34415914233475586,0.08286056438796492,0.3267174273303975,0.0,0.4315975286849073,0.4315975286849073,0.3379573469517685,11
-01-ai/Yi-1.5-9B-Chat,1.14,0.3423084527203913,0.07890758839182645,0.3253386227189031,0.06090026478375993,0.30185348631950576,0.3627537511032657,0.33709189557055336,16
-01-ai/Yi-1.5-9B-Chat,1.16,0.34089943422352015,0.07400825605871394,0.3224006234779672,0.02294792586054722,0.2833186231244484,0.3062665489849956,0.33649094177763084,22
-01-ai/Yi-1.5-9B-Chat,1.18,0.3365273541015462,0.06786555450924157,0.3143386280928703,0.0,0.30979699911738745,0.30979699911738745,0.33212663825955735,26
-01-ai/Yi-1.5-9B-Chat,1.20,0.3355307477803475,0.06314678954328107,0.31123860048630714,0.0,0.22241835834068843,0.22241835834068843,0.33235553904085485,36
-Qwen/Qwen2-72B-Instruct,1.00,0.3928168861285181,0.12345162681603773,0.38390142579747066,0.0,0.17563989408649602,0.17563989408649602,0.389868803763904,0
-Qwen/Qwen2-72B-Instruct,1.02,0.3936651928828143,0.12446659906815814,0.3839114141179235,0.0,0.147396293027361,0.147396293027361,0.39117939588436124,0
-Qwen/Qwen2-72B-Instruct,1.04,0.39263683565035906,0.12496255366843562,0.38459581775094975,0.0,0.15798764342453664,0.15798764342453664,0.38998196316138,0
-Qwen/Qwen2-72B-Instruct,1.06,0.39300072547277504,0.12419625082296233,0.3842780423867672,0.0,0.15798764342453664,0.15798764342453664,0.3903433924885219,0
-Qwen/Qwen2-72B-Instruct,1.08,0.39260920223934465,0.12159917926987915,0.38447418699052216,0.0,0.15798764342453664,0.15798764342453664,0.3899545165977931,0
-Qwen/Qwen2-72B-Instruct,1.10,0.3908754537278073,0.12158997920112467,0.382023720181386,0.0,0.1650485436893204,0.1650485436893204,0.38811615859747245,0
-Qwen/Qwen2-72B-Instruct,1.12,0.3901405252446773,0.11689725142339027,0.3820602123452673,0.0,0.19505736981465135,0.19505736981465135,0.38689460563454847,1
-Qwen/Qwen2-72B-Instruct,1.14,0.38779002122304185,0.1157566203933617,0.3781701938094464,0.0,0.1703442188879082,0.1703442188879082,0.38496604428211745,0
-Qwen/Qwen2-72B-Instruct,1.16,0.3867065727555076,0.11470275348285906,0.37688474492298285,0.0,0.15975286849073256,0.15975286849073256,0.3840630117198038,1
-Qwen/Qwen2-72B-Instruct,1.18,0.3836165246347915,0.10917281839005354,0.3747478193448276,0.0,0.14916151809355693,0.14916151809355693,0.3811655674825621,3
-Qwen/Qwen2-72B-Instruct,1.20,0.37971211487547984,0.10498466771923821,0.37197960114090106,0.0,0.293909973521624,0.293909973521624,0.37499454632380763,6
-Qwen/Qwen2-72B-Instruct,1.22,0.3774442827665068,0.10330617330273109,0.3699012784671952,0.06001765225066196,0.14563106796116504,0.205648720211827,0.37413668409339734,4
-Qwen/Qwen2-72B-Instruct,1.24,0.37236689663431144,0.0975677163987442,0.3660038409634332,0.0,0.14210061782877317,0.14210061782877317,0.37009896173470336,7
-Qwen/Qwen2-72B-Instruct,1.26,0.36709984654252126,0.09446497792017897,0.36287607522182885,0.00529567519858782,0.18623124448367168,0.1915269196822595,0.3640999339946495,9
-Qwen/Qwen2-72B-Instruct,1.28,0.363672126987302,0.08776847871058446,0.35837939967274784,0.0,0.1262135922330097,0.1262135922330097,0.36170190642085254,11
-Qwen/Qwen2-72B-Instruct,1.30,0.36072372289732685,0.07886593555129406,0.35434208671286505,0.3000882612533098,0.22065313327449249,0.5207413945278023,0.3529426277694367,27
-Qwen/Qwen2-7B-Instruct,1.00,0.3788852766816091,0.11779757565648481,0.36942355750319633,0.0,0.0997352162400706,0.0997352162400706,0.37725929362438315,0
-Qwen/Qwen2-7B-Instruct,1.02,0.37747961314136774,0.11676496093416945,0.3685378471451668,0.0,0.0997352162400706,0.0997352162400706,0.37585966247769537,0
-Qwen/Qwen2-7B-Instruct,1.04,0.37726610964345525,0.1156236824251826,0.3673369615606667,0.0,0.09090909090909091,0.09090909090909091,0.37578914794963475,0
-Qwen/Qwen2-7B-Instruct,1.06,0.37811123052366835,0.11425460970355292,0.3674510486875733,0.0,0.09002647837599294,0.09002647837599294,0.37664521213491803,0
-Qwen/Qwen2-7B-Instruct,1.08,0.37410786991499057,0.11100236068028714,0.3629286781065867,0.0,0.0820829655781112,0.0820829655781112,0.37278438583748985,0
-Qwen/Qwen2-7B-Instruct,1.10,0.3726776797510745,0.10913952265827548,0.35984560125194764,0.0,0.09090909090909091,0.09090909090909091,0.37121868133307606,0
-Qwen/Qwen2-7B-Instruct,1.12,0.37111350769392976,0.10686948327720774,0.3573554465690655,0.0,0.09090909090909091,0.09090909090909091,0.36966063286390244,0
-Qwen/Qwen2-7B-Instruct,1.14,0.36899468281467784,0.10532730577953538,0.3565920877170038,0.0,0.09090909090909091,0.09090909090909091,0.3675501029867792,0
-Qwen/Qwen2-7B-Instruct,1.16,0.36740747078226954,0.10220163431022722,0.3538496075663,0.0,0.0997352162400706,0.0997352162400706,0.36583074463491855,0
-Qwen/Qwen2-7B-Instruct,1.18,0.36291646275476663,0.09906644982386541,0.3499896954704415,0.0,0.0997352162400706,0.0997352162400706,0.36135900973153,0
-Qwen/Qwen2-7B-Instruct,1.20,0.3602807930821352,0.09532425997250199,0.34611214566424575,0.0,0.11032656663724624,0.11032656663724624,0.3585721310217351,0
-Qwen/Qwen2-7B-Instruct,1.22,0.35715128912133703,0.0920447399321579,0.3441700414615685,0.0,0.08561341571050309,0.08561341571050309,0.3558338791174631,0
-Qwen/Qwen2-7B-Instruct,1.24,0.352755026120472,0.08591470945904531,0.3394417789236487,0.0,0.08561341571050309,0.08561341571050309,0.3514538324401377,0
-Qwen/Qwen2-7B-Instruct,1.26,0.3483233677173315,0.07972359456247886,0.33446894013409734,0.0,0.08561341571050309,0.08561341571050309,0.3470385209221742,0
-Qwen/Qwen2-7B-Instruct,1.28,0.34450122231539704,0.07518096876457613,0.33039776348046035,0.0,0.09179170344218888,0.09179170344218888,0.34313954918633316,1
-Qwen/Qwen2-7B-Instruct,1.30,0.3401098279932269,0.07026740554261787,0.3254126623075102,0.0,0.09002647837599294,0.09002647837599294,0.3387911491977248,3
-shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.3815423445635067,0.11524878188694271,0.36996911815693134,0.0,0.1968225948808473,0.1968225948808473,0.37833975022913946,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.3579680086793429,0.10207096308148353,0.3459640175229714,0.0,0.19240953221535745,0.19240953221535745,0.3550294775004645,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.3572226770743513,0.10061303169730976,0.3448975271352682,0.0,0.1615180935569285,0.1615180935569285,0.3547540871288482,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.04,0.35670586983276636,0.10074138007196803,0.3449144632118436,0.0,0.1615180935569285,0.1615180935569285,0.3542408512875192,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.06,0.35549318326656437,0.0998891248706679,0.34322366122312886,0.0,0.15975286849073256,0.15975286849073256,0.35306299977874456,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.08,0.3549933805160392,0.09858894278315135,0.3430005301668489,0.0,0.1526919682259488,0.1526919682259488,0.35267236961581055,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.10,0.3534792705039357,0.09604337437044752,0.3415984757584558,0.0,0.12886142983230361,0.12886142983230361,0.35152457252197683,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.12,0.35134989369086755,0.09466593964355864,0.3402757462396958,0.0,0.13592233009708737,0.13592233009708737,0.3493018413002178,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.14,0.3523467471502627,0.09455136235619709,0.34011481374007896,0.0,0.15710503089143865,0.15710503089143865,0.34997741984243846,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.16,0.35026382260485167,0.09291738095604976,0.33774387946289774,0.0,0.17299205648720212,0.17299205648720212,0.34767410066010357,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.18,0.3489231946755186,0.09267866809703615,0.33706232254061985,0.0,0.18711385701676964,0.18711385701676964,0.3461363951386201,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.20,0.3481931091877492,0.09121903225057944,0.33581387922683614,0.0,0.2003530450132392,0.2003530450132392,0.34521897747455255,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.22,0.34604714296451533,0.08945165053230478,0.33477451607196596,0.0,0.18711385701676964,0.18711385701676964,0.34328331404034373,1
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.24,0.3441084154272239,0.0880200303756021,0.33240744238972986,0.0,0.20123565754633715,0.20123565754633715,0.34115646351180073,1
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.26,0.3434534163683513,0.08571979267389605,0.33010684388262634,0.0,0.17828773168578993,0.17828773168578993,0.34083758204748993,0
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.28,0.34008394315191964,0.08346595677194628,0.3274872658281857,0.0,0.19858781994704325,0.19858781994704325,0.33720420576281357,2
-shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.30,0.3385373237572206,0.08244181010811574,0.32639975491275247,0.0,0.18005295675198588,0.18005295675198588,0.33593382411969774,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.3256642047768536,0.08331314362646546,0.3158704402692302,0.0,0.16857899382171226,0.16857899382171226,0.3233168382824646,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.3261638331201866,0.08437219278343962,0.3157031443159532,0.0,0.17210944395410416,0.17210944395410416,0.3237644072634129,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.3261267542205407,0.0841026780937562,0.31557218541916426,0.0,0.9267431597528685,0.9267431597528685,0.3140391563308914,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.32610191030444663,0.08440911364941035,0.315363371759167,0.0,0.7917034421888791,0.7917034421888791,0.3156568252183852,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.32519072627069395,0.08573531403311445,0.3153935939954504,0.0,0.7917034421888791,0.7917034421888791,0.31477482652351235,1
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.32510929376904546,0.08572184129459336,0.31430790316573815,0.0,0.2921447484554281,0.2921447484554281,0.3210937323218641,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.12,0.325321692973156,0.08501006133800607,0.3134505738999941,0.0,0.294792586054722,0.294792586054722,0.3212680607332789,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.14,0.3224620858016468,0.08389328832417228,0.3130539552486071,0.0,0.12268314210061783,0.12268314210061783,0.3207634371908326,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.16,0.32354623636120206,0.08389983318570625,0.3135227768775155,0.0,0.12268314210061783,0.12268314210061783,0.32184187671977993,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.18,0.3227464993995023,0.08237511984991769,0.31232054358058636,0.0,0.10326566637246248,0.10326566637246248,0.3213128743065604,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.20,0.3213479416591043,0.08021470447158471,0.31080818824701156,0.0,0.08473080317740513,0.08473080317740513,0.3201747222670157,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.22,0.31939727082775615,0.08027275774782588,0.308833370619165,0.0,0.10061782877316858,0.10061782877316858,0.3180145628216866,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.24,0.3188662188138966,0.07877965659256216,0.30709225454106126,0.0,0.0820829655781112,0.0820829655781112,0.31773816351917916,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.26,0.31805084189335,0.07777595035895293,0.30653839142082673,0.0,0.07325684024713151,0.07325684024713151,0.31704583595098484,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.28,0.31564132115319793,0.07471248687074669,0.30482678714954914,0.0,0.05736981465136805,0.05736981465136805,0.3148590790692337,0
-shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.30,0.31448483374273595,0.07484673889486904,0.3033930752633869,0.0,0.06796116504854369,0.06796116504854369,0.3135624797736094,0
+01-ai/Yi-1.5-9B-Chat,1.00,0.3463725436435439,0.09312113035602035,0.33287597095291194,0.0,0.35127978817299205,0.35127978817299205,0.34125573890735983,2
+01-ai/Yi-1.5-9B-Chat,1.02,0.3471185374158656,0.09126513887574451,0.3325894211716421,0.0,0.264783759929391,0.264783759929391,0.3432230230787291,4
+01-ai/Yi-1.5-9B-Chat,1.04,0.3471882673119874,0.09019886552461354,0.33194600115482237,0.0,0.37775816416593117,0.37775816416593117,0.3416859125059273,8
+01-ai/Yi-1.5-9B-Chat,1.06,0.3475947948648639,0.09004996084071014,0.33128175611085137,0.0,0.46866725507502205,0.46866725507502205,0.34081549880066125,9
+01-ai/Yi-1.5-9B-Chat,1.08,0.34751102711658816,0.09004837948083254,0.33142704269248113,0.0,0.3115622241835834,0.3115622241835834,0.3429415142848335,4
+01-ai/Yi-1.5-9B-Chat,1.10,0.3483026982644252,0.08832819752923171,0.3310725986559182,0.0,0.265666372462489,0.265666372462489,0.3443811795403635,6
+01-ai/Yi-1.5-9B-Chat,1.12,0.34415914233475586,0.08286056438796492,0.32639196028762685,0.0,0.4315975286849073,0.4315975286849073,0.3379573469517685,11
+01-ai/Yi-1.5-9B-Chat,1.14,0.3423084527203913,0.07890758839182645,0.32510226519758223,0.06090026478375993,0.30185348631950576,0.3627537511032657,0.33709189557055336,16
+01-ai/Yi-1.5-9B-Chat,1.16,0.34089943422352015,0.07400825605871394,0.3220659569751716,0.02294792586054722,0.2833186231244484,0.3062665489849956,0.33649094177763084,22
+01-ai/Yi-1.5-9B-Chat,1.18,0.3365273541015462,0.06786555450924157,0.31409006189011535,0.0,0.30979699911738745,0.30979699911738745,0.33212663825955735,26
+01-ai/Yi-1.5-9B-Chat,1.20,0.3355307477803475,0.06314678954328107,0.3110853370928648,0.0,0.22241835834068843,0.22241835834068843,0.33235553904085485,36
+01-ai/Yi-1.5-9B-Chat,1.22,0.33363375306882515,0.06214712430276763,0.3086913255065605,0.11827007943512798,0.25772285966460723,0.3715798764342454,0.32842981018186884,33
+Qwen/Qwen2-72B-Instruct,1.00,0.3928168861285181,0.12345162681603773,0.3843593208981698,0.0,0.17563989408649602,0.17563989408649602,0.389868803763904,0
+Qwen/Qwen2-72B-Instruct,1.02,0.3936651928828143,0.12446659906815814,0.3844415446718956,0.0,0.147396293027361,0.147396293027361,0.39117939588436124,0
+Qwen/Qwen2-72B-Instruct,1.04,0.39263683565035906,0.12496255366843562,0.38481746782098636,0.0,0.15798764342453664,0.15798764342453664,0.38998196316138,0
+Qwen/Qwen2-72B-Instruct,1.06,0.39300072547277504,0.12419625082296233,0.3845159808902676,0.0,0.15798764342453664,0.15798764342453664,0.3903433924885219,0
+Qwen/Qwen2-72B-Instruct,1.08,0.39260920223934465,0.12159917926987915,0.38481620894368174,0.0,0.15798764342453664,0.15798764342453664,0.3899545165977931,0
+Qwen/Qwen2-72B-Instruct,1.10,0.3908754537278073,0.12158997920112467,0.3824226422124425,0.0,0.1650485436893204,0.1650485436893204,0.38811615859747245,0
+Qwen/Qwen2-72B-Instruct,1.12,0.3901405252446773,0.11689725142339027,0.38250566812232806,0.0,0.19505736981465135,0.19505736981465135,0.38689460563454847,1
+Qwen/Qwen2-72B-Instruct,1.14,0.38779002122304185,0.1157566203933617,0.37857957862490155,0.0,0.1703442188879082,0.1703442188879082,0.38496604428211745,0
+Qwen/Qwen2-72B-Instruct,1.16,0.3867065727555076,0.11470275348285906,0.3775981974046914,0.0,0.15975286849073256,0.15975286849073256,0.3840630117198038,1
+Qwen/Qwen2-72B-Instruct,1.18,0.3836165246347915,0.10917281839005354,0.37528224157206047,0.0,0.14916151809355693,0.14916151809355693,0.3811655674825621,3
+Qwen/Qwen2-72B-Instruct,1.20,0.37971211487547984,0.10498466771923821,0.3725613062501543,0.0,0.293909973521624,0.293909973521624,0.37499454632380763,6
+Qwen/Qwen2-72B-Instruct,1.22,0.3774442827665068,0.10330617330273109,0.37054031476113913,0.06001765225066196,0.14563106796116504,0.205648720211827,0.37413668409339734,4
+Qwen/Qwen2-72B-Instruct,1.24,0.37236689663431144,0.0975677163987442,0.3669036472865035,0.0,0.14210061782877317,0.14210061782877317,0.37009896173470336,7
+Qwen/Qwen2-72B-Instruct,1.26,0.36709984654252126,0.09446497792017897,0.3636041937619996,0.00529567519858782,0.18623124448367168,0.1915269196822595,0.3640999339946495,9
+Qwen/Qwen2-72B-Instruct,1.28,0.363672126987302,0.08776847871058446,0.3589890433500369,0.0,0.1262135922330097,0.1262135922330097,0.36170190642085254,11
+Qwen/Qwen2-72B-Instruct,1.30,0.36072372289732685,0.07886593555129406,0.3550654391413709,0.3000882612533098,0.22065313327449249,0.5207413945278023,0.3529426277694367,27
+Qwen/Qwen2-7B-Instruct,1.00,0.3788852766816091,0.11779757565648481,0.3698175713986759,0.0,0.0997352162400706,0.0997352162400706,0.37725929362438315,0
+Qwen/Qwen2-7B-Instruct,1.02,0.37747961314136774,0.11676496093416945,0.36883439076652713,0.0,0.0997352162400706,0.0997352162400706,0.37585966247769537,0
+Qwen/Qwen2-7B-Instruct,1.04,0.37726610964345525,0.1156236824251826,0.3675624036484201,0.0,0.09090909090909091,0.09090909090909091,0.37578914794963475,0
+Qwen/Qwen2-7B-Instruct,1.06,0.37811123052366835,0.11425460970355292,0.36784542445645985,0.0,0.09002647837599294,0.09002647837599294,0.37664521213491803,0
+Qwen/Qwen2-7B-Instruct,1.08,0.37410786991499057,0.11100236068028714,0.3633474533737226,0.0,0.0820829655781112,0.0820829655781112,0.37278438583748985,0
+Qwen/Qwen2-7B-Instruct,1.10,0.3726776797510745,0.10913952265827548,0.36035372033185764,0.0,0.09090909090909091,0.09090909090909091,0.37121868133307606,0
+Qwen/Qwen2-7B-Instruct,1.12,0.37111350769392976,0.10686948327720774,0.35786498961907043,0.0,0.09090909090909091,0.09090909090909091,0.36966063286390244,0
+Qwen/Qwen2-7B-Instruct,1.14,0.36899468281467784,0.10532730577953538,0.3570710521542533,0.0,0.09090909090909091,0.09090909090909091,0.3675501029867792,0
+Qwen/Qwen2-7B-Instruct,1.16,0.36740747078226954,0.10220163431022722,0.3542070351262723,0.0,0.0997352162400706,0.0997352162400706,0.36583074463491855,0
+Qwen/Qwen2-7B-Instruct,1.18,0.36291646275476663,0.09906644982386541,0.35051557091700036,0.0,0.0997352162400706,0.0997352162400706,0.36135900973153,0
+Qwen/Qwen2-7B-Instruct,1.20,0.3602807930821352,0.09532425997250199,0.34681080047647195,0.0,0.11032656663724624,0.11032656663724624,0.3585721310217351,0
+Qwen/Qwen2-7B-Instruct,1.22,0.35715128912133703,0.0920447399321579,0.3449249735672454,0.0,0.08561341571050309,0.08561341571050309,0.3558338791174631,0
+Qwen/Qwen2-7B-Instruct,1.24,0.352755026120472,0.08591470945904531,0.3399352759086753,0.0,0.08561341571050309,0.08561341571050309,0.3514538324401377,0
+Qwen/Qwen2-7B-Instruct,1.26,0.3483233677173315,0.07972359456247886,0.3352291660383133,0.0,0.08561341571050309,0.08561341571050309,0.3470385209221742,0
+Qwen/Qwen2-7B-Instruct,1.28,0.34450122231539704,0.07518096876457613,0.33099576010918924,0.0,0.09179170344218888,0.09179170344218888,0.34313954918633316,1
+Qwen/Qwen2-7B-Instruct,1.30,0.3401098279932269,0.07026740554261787,0.32623150769341913,0.0,0.09002647837599294,0.09002647837599294,0.3387911491977248,3
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.00,0.3815423445635067,0.11524878188694271,0.37042646286690667,0.0,0.1968225948808473,0.1968225948808473,0.37833975022913946,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.02,0.3814400195917603,0.11481993983759356,0.3699786095406088,0.0,0.2118270079435128,0.2118270079435128,0.3779989256169545,0
+shenzhi-wang/Llama3.1-70B-Chinese-Chat,1.04,0.3804874897233306,0.11332999451398112,0.36959666465385377,0.0,0.1879964695498676,0.1879964695498676,0.3774345028777672,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.00,0.3579680086793429,0.10207096308148353,0.3460132814937531,0.0,0.19240953221535745,0.19240953221535745,0.3550294775004645,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.02,0.3572226770743513,0.10061303169730976,0.3450507994469454,0.0,0.1615180935569285,0.1615180935569285,0.3547540871288482,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.04,0.35670586983276636,0.10074138007196803,0.3450245802338977,0.0,0.1615180935569285,0.1615180935569285,0.3542408512875192,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.06,0.35549318326656437,0.0998891248706679,0.3434279258982189,0.0,0.15975286849073256,0.15975286849073256,0.35306299977874456,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.08,0.3549933805160392,0.09858894278315135,0.34312451283209056,0.0,0.1526919682259488,0.1526919682259488,0.35267236961581055,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.10,0.3534792705039357,0.09604337437044752,0.3418333251931206,0.0,0.12886142983230361,0.12886142983230361,0.35152457252197683,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.12,0.35134989369086755,0.09466593964355864,0.3406575067978035,0.0,0.13592233009708737,0.13592233009708737,0.3493018413002178,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.14,0.3523467471502627,0.09455136235619709,0.3405868200118059,0.0,0.15710503089143865,0.15710503089143865,0.34997741984243846,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.16,0.35026382260485167,0.09291738095604976,0.3381480886207706,0.0,0.17299205648720212,0.17299205648720212,0.34767410066010357,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.18,0.3489231946755186,0.09267866809703615,0.33749580508212734,0.0,0.18711385701676964,0.18711385701676964,0.3461363951386201,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.20,0.3481931091877492,0.09121903225057944,0.3363167460865901,0.0,0.2003530450132392,0.2003530450132392,0.34521897747455255,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.22,0.34604714296451533,0.08945165053230478,0.3352830183155636,0.0,0.18711385701676964,0.18711385701676964,0.34328331404034373,1
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.24,0.3441084154272239,0.0880200303756021,0.3327376500632496,0.0,0.20123565754633715,0.20123565754633715,0.34115646351180073,1
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.26,0.3434534163683513,0.08571979267389605,0.33067392953084385,0.0,0.17828773168578993,0.17828773168578993,0.34083758204748993,0
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.28,0.34008394315191964,0.08346595677194628,0.3280102626306619,0.0,0.19858781994704325,0.19858781994704325,0.33720420576281357,2
+shenzhi-wang/Llama3.1-8B-Chinese-Chat,1.30,0.3385373237572206,0.08244181010811574,0.32684753756927853,0.0,0.18005295675198588,0.18005295675198588,0.33593382411969774,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.00,0.3256642047768536,0.08331314362646546,0.3161580747851038,0.0,0.16857899382171226,0.16857899382171226,0.3233168382824646,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.02,0.3261638331201866,0.08437219278343962,0.3160165886106982,0.0,0.17210944395410416,0.17210944395410416,0.3237644072634129,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.04,0.3261267542205407,0.0841026780937562,0.31586790820617083,0.0,0.9267431597528685,0.9267431597528685,0.3140391563308914,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.06,0.32610191030444663,0.08440911364941035,0.3156161826682502,0.0,0.7917034421888791,0.7917034421888791,0.3156568252183852,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.08,0.32519072627069395,0.08573531403311445,0.31555219872015555,0.0,0.7917034421888791,0.7917034421888791,0.31477482652351235,1
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.10,0.32510929376904546,0.08572184129459336,0.3143439978950341,0.0,0.2921447484554281,0.2921447484554281,0.3210937323218641,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.12,0.325321692973156,0.08501006133800607,0.3136378009708979,0.0,0.294792586054722,0.294792586054722,0.3212680607332789,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.14,0.3224620858016468,0.08389328832417228,0.3130154415556936,0.0,0.12268314210061783,0.12268314210061783,0.3207634371908326,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.16,0.32354623636120206,0.08389983318570625,0.31361563891120947,0.0,0.12268314210061783,0.12268314210061783,0.32184187671977993,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.18,0.3227464993995023,0.08237511984991769,0.31237472571694164,0.0,0.10326566637246248,0.10326566637246248,0.3213128743065604,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.20,0.3213479416591043,0.08021470447158471,0.31081287893463483,0.0,0.08473080317740513,0.08473080317740513,0.3201747222670157,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.22,0.31939727082775615,0.08027275774782588,0.3089835285121734,0.0,0.10061782877316858,0.10061782877316858,0.3180145628216866,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.24,0.3188662188138966,0.07877965659256216,0.30728620231759696,0.0,0.0820829655781112,0.0820829655781112,0.31773816351917916,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.26,0.31805084189335,0.07777595035895293,0.30667694159970027,0.0,0.07325684024713151,0.07325684024713151,0.31704583595098484,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.28,0.31564132115319793,0.07471248687074669,0.3046949012325021,0.0,0.05736981465136805,0.05736981465136805,0.3148590790692337,0
+shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat,1.30,0.31448483374273595,0.07484673889486904,0.3035257437090866,0.0,0.06796116504854369,0.06796116504854369,0.3135624797736094,0

scripts/eval-4gpu.sh

@@ -17,6 +17,7 @@ grep MemTotal /proc/meminfo
 
 #./scripts/eval-model.sh Qwen/Qwen2-72B-Instruct
 
+export START_REPETITION_PENALTY=1.06
 ./scripts/eval-model.sh shenzhi-wang/Llama3.1-70B-Chinese-Chat
 
-./scripts/eval-model.sh 01-ai/Yi-1.5-34B-Chat
+# ./scripts/eval-model.sh 01-ai/Yi-1.5-34B-Chat