| {"data": {"SlimPajama": {"x": [0.0, 2.0971520000000003, 4.194304000000001, 6.291456, 8.388608000000001, 10.48576, 12.582912, 14.680064000000002, 16.777216000000003, 18.874368, 20.97152, 23.068672000000003, 25.165824, 27.262976000000002, 29.360128000000003, 31.45728, 33.554432000000006, 35.651584, 37.748736, 39.845888, 41.94304, 44.040192000000005, 46.137344000000006, 48.234496, 50.331648, 52.4288, 54.525952000000004, 56.623104000000005, 58.720256000000006, 60.817408, 62.91456, 65.011712, 67.10886400000001, 69.206016, 71.303168, 73.40032000000001, 75.497472, 77.59462400000001, 79.691776, 81.788928, 83.88608, 85.983232, 88.08038400000001, 90.177536, 92.27468800000001, 94.37184, 96.468992, 98.56614400000001, 100.663296, 102.76044800000001, 104.8576, 106.95475200000001, 109.05190400000001, 111.149056, 113.24620800000001, 115.34336, 117.44051200000001, 119.537664, 121.634816, 123.73196800000001, 125.82912, 127.92627200000001, 130.023424, 132.120576, 134.21772800000002, 136.31488000000002, 138.412032, 140.509184, 142.606336, 144.70348800000002, 146.80064000000002, 148.897792, 150.994944, 153.092096, 155.18924800000002, 157.28640000000001, 159.383552, 161.480704, 163.577856, 165.67500800000002, 167.77216, 169.869312, 171.966464, 174.06361600000002, 176.16076800000002, 178.25792, 180.355072, 182.452224, 184.54937600000002, 186.64652800000002, 188.74368, 190.840832, 192.937984, 195.03513600000002, 197.13228800000002, 199.22944, 201.326592, 203.423744, 205.52089600000002, 207.61804800000002, 209.7152, 211.812352, 213.90950400000003, 216.00665600000002, 218.10380800000001, 220.20096, 222.298112, 224.39526400000003, 226.49241600000002, 228.589568, 230.68672, 232.783872, 234.88102400000002, 236.97817600000002, 239.075328, 241.17248, 243.269632, 245.36678400000002, 247.46393600000002, 249.561088, 251.65824, 253.75539200000003, 255.85254400000002, 257.949696, 260.046848, 262.144, 264.241152, 266.338304, 268.43545600000004, 270.53260800000004, 272.62976000000003, 274.726912, 276.824064, 278.921216, 281.018368, 283.11552, 285.212672, 287.309824, 289.40697600000004, 291.50412800000004, 293.60128000000003, 295.698432, 297.795584, 299.892736, 301.989888, 304.08704, 306.184192, 308.28134400000005, 310.37849600000004, 312.47564800000004, 314.57280000000003, 316.669952, 318.767104, 320.864256, 322.961408, 325.05856, 327.155712, 329.25286400000005, 331.35001600000004, 333.44716800000003, 335.54432, 337.641472, 339.738624, 341.835776, 343.932928, 346.03008, 348.12723200000005, 350.22438400000004], "y": [0.2497200071811676, 0.2592614293098449, 0.2593538165092468, 0.2662509083747864, 0.2664803266525268, 0.2686671912670135, 0.2750212550163269, 0.2728220522403717, 0.2735992670059204, 0.2816755771636963, 0.2807474434375763, 0.2832716107368469, 0.2859686017036438, 0.2851547598838806, 0.2844336628913879, 0.2834415435791015, 0.2903566062450409, 0.2867975533008575, 0.2869687974452972, 0.2880666553974151, 0.2895745635032654, 0.2932893931865692, 0.285281777381897, 0.29486083984375, 0.2966246604919433, 0.2939338088035583, 0.2906226515769958, 0.2981550693511963, 0.2983705997467041, 0.2941931188106537, 0.3010403513908386, 0.2986909449100494, 0.3023008406162262, 0.2999930381774902, 0.2975163459777832, 0.3010782599449157, 0.3005977869033813, 0.3014611005783081, 0.2960115671157837, 0.3022516369819641, 0.3078178465366363, 0.3019892573356628, 0.3054529130458832, 0.3049662113189697, 0.3028279542922973, 0.3042507171630859, 0.3116106986999511, 0.3049386143684387, 0.3076794445514679, 0.3053653836250305, 0.3102109432220459, 0.3084518015384674, 0.3062224984169006, 0.3118025958538055, 0.3052020072937011, 0.3125185668468475, 0.3075071275234222, 0.3104312717914581, 0.3031999468803406, 0.3096194863319397, 0.3074706494808197, 0.3123086094856262, 0.3070628046989441, 0.3116572499275207, 0.3127453923225403, 0.3144129812717438, 0.3129480183124542, 0.3091934025287628, 0.3163215219974518, 0.3153833448886871, 0.3132557868957519, 0.3166911005973816, 0.3108917474746704, 0.3084513247013092, 0.3118407726287842, 0.3133653402328491, 0.3157584071159363, 0.3147788345813751, 0.311055988073349, 0.3152942359447479, 0.3187046945095062, 0.3157975673675537, 0.3121104836463928, 0.3179425001144409, 0.3151332139968872, 0.3168922960758209, 0.3146162927150726, 0.3181720376014709, 0.3161299228668213, 0.3202162981033325, 0.3161057233810425, 0.3180809915065765, 0.3162356913089752, 0.315767765045166, 0.3190236985683441, 0.3171856999397278, 0.3202316761016845, 0.3133728802204132, 0.3163894414901733, 0.31822270154953, 0.3266949653625488, 0.3198533058166504, 0.3233769834041595, 0.3238007426261902, 0.3205596804618835, 0.3247207403182983, 0.3206443786621094, 0.323225736618042, 0.3217288851737976, 0.3235573768615722, 0.3252238631248474, 0.324187159538269, 0.3246301114559173, 0.3234524428844452, 0.3228626251220703, 0.3241204023361206, 0.3248985707759857, 0.3271089494228363, 0.3260233104228973, 0.3256067335605621, 0.3253726661205292, 0.3233819603919983, 0.3251070082187652, 0.3276388943195343, 0.3218710720539093, 0.3246103525161743, 0.3265250921249389, 0.3275103867053985, 0.3297024071216583, 0.3277328908443451, 0.3267576098442077, 0.327478289604187, 0.3304429948329925, 0.3273082375526428, 0.3264918923377991, 0.3236933350563049, 0.3272844851016998, 0.3272388875484466, 0.3295789361000061, 0.3278802335262298, 0.3284499049186706, 0.3294344544410705, 0.3298918604850769, 0.3306445777416229, 0.3281697332859039, 0.3339054882526397, 0.3293753862380981, 0.3298616707324981, 0.3280701637268066, 0.3316899836063385, 0.3303491175174713, 0.330327957868576, 0.3314228653907776, 0.3317765891551971, 0.3307021856307983, 0.330121636390686, 0.3283750414848327, 0.3317910432815552, 0.3326679468154907, 0.3311441838741302, 0.330735981464386, 0.3313741981983185, 0.3320714235305786, 0.3333134055137634, 0.3337414860725403, 0.3286249935626983, 0.3344342410564422, 0.3337143063545227], "label": "SlimPajama"}, "RefinedWeb": {"x": [0.0, 2.0971520000000003, 4.194304000000001, 6.291456, 8.388608000000001, 10.48576, 12.582912, 14.680064000000002, 16.777216000000003, 18.874368, 20.97152, 23.068672000000003, 25.165824, 27.262976000000002, 29.360128000000003, 31.45728, 33.554432000000006, 35.651584, 37.748736, 39.845888, 41.94304, 44.040192000000005, 46.137344000000006, 48.234496, 50.331648, 52.4288, 54.525952000000004, 56.623104000000005, 58.720256000000006, 60.817408, 62.91456, 65.011712, 67.10886400000001, 69.206016, 71.303168, 73.40032000000001, 75.497472, 77.59462400000001, 79.691776, 81.788928, 83.88608, 85.983232, 88.08038400000001, 90.177536, 92.27468800000001, 94.37184, 96.468992, 98.56614400000001, 100.663296, 102.76044800000001, 104.8576, 106.95475200000001, 109.05190400000001, 111.149056, 113.24620800000001, 115.34336, 117.44051200000001, 119.537664, 121.634816, 123.73196800000001, 125.82912, 127.92627200000001, 130.023424, 132.120576, 134.21772800000002, 136.31488000000002, 138.412032, 140.509184, 142.606336, 144.70348800000002, 146.80064000000002, 148.897792, 150.994944, 153.092096, 155.18924800000002, 157.28640000000001, 159.383552, 161.480704, 163.577856, 165.67500800000002, 167.77216, 169.869312, 171.966464, 174.06361600000002, 176.16076800000002, 178.25792, 180.355072, 182.452224, 184.54937600000002, 186.64652800000002, 188.74368, 190.840832, 192.937984, 195.03513600000002, 197.13228800000002, 199.22944, 201.326592, 203.423744, 205.52089600000002, 207.61804800000002, 209.7152, 211.812352, 213.90950400000003, 216.00665600000002, 218.10380800000001, 220.20096, 222.298112, 224.39526400000003, 226.49241600000002, 228.589568, 230.68672, 232.783872, 234.88102400000002, 236.97817600000002, 239.075328, 241.17248, 243.269632, 245.36678400000002, 247.46393600000002, 249.561088, 251.65824, 253.75539200000003, 255.85254400000002, 257.949696, 260.046848, 262.144, 264.241152, 266.338304, 268.43545600000004, 270.53260800000004, 272.62976000000003, 274.726912, 276.824064, 278.921216, 281.018368, 283.11552, 285.212672, 287.309824, 289.40697600000004, 291.50412800000004, 293.60128000000003, 295.698432, 297.795584, 299.892736, 301.989888, 304.08704, 306.184192, 308.28134400000005, 310.37849600000004, 312.47564800000004, 314.57280000000003, 316.669952, 318.767104, 320.864256, 322.961408, 325.05856, 327.155712, 329.25286400000005, 331.35001600000004, 333.44716800000003, 335.54432, 337.641472, 339.738624, 341.835776, 343.932928, 346.03008, 348.12723200000005, 350.22438400000004], "y": [0.2501466572284698, 0.2528519630432129, 0.2616856694221496, 0.2665999829769134, 0.2683407664299011, 0.2742894291877746, 0.2762066125869751, 0.2807516455650329, 0.2767378389835357, 0.2807380557060241, 0.2788906991481781, 0.2844051718711853, 0.2856102883815765, 0.2883394360542297, 0.2890409529209137, 0.2894668281078338, 0.2883355319499969, 0.2872501015663147, 0.291619062423706, 0.2900333702564239, 0.2962473034858703, 0.2962896525859833, 0.297355443239212, 0.2932226359844208, 0.2886744439601898, 0.29665008187294, 0.2976542115211487, 0.2991503179073334, 0.3004479110240936, 0.3044549524784088, 0.2976194322109222, 0.3014707863330841, 0.3048252463340759, 0.3039425611495971, 0.303354948759079, 0.3027459383010864, 0.2999922931194305, 0.3050121665000915, 0.2998814284801483, 0.2978588044643402, 0.3041949570178985, 0.3010904192924499, 0.3022017180919647, 0.2997751235961914, 0.3015910983085632, 0.3096485137939453, 0.3012076020240783, 0.3065535724163055, 0.3042872548103332, 0.3104783594608307, 0.2997980415821075, 0.3051296770572662, 0.303458571434021, 0.3088337182998657, 0.3145398199558258, 0.3032208085060119, 0.310806930065155, 0.3075874149799347, 0.3101692199707031, 0.310107946395874, 0.3066047430038452, 0.3109066784381866, 0.3081336915493011, 0.3084586262702942, 0.3086149394512176, 0.3085348606109619, 0.3136637806892395, 0.3110873103141784, 0.31076380610466, 0.3084572553634643, 0.3133681714534759, 0.3125792145729065, 0.3124453127384186, 0.3097185790538788, 0.3106793165206909, 0.3089564740657806, 0.3111244142055511, 0.3123694658279419, 0.3144859969615936, 0.3135123550891876, 0.311982125043869, 0.3142133951187134, 0.3122903704643249, 0.3147654831409454, 0.3078767359256744, 0.314947634935379, 0.3171303570270538, 0.3129573762416839, 0.3154936134815216, 0.3158208429813385, 0.3153132200241089, 0.3141326904296875, 0.3163397014141083, 0.3166318237781524, 0.3168410360813141, 0.3198235332965851, 0.3201336860656738, 0.3212967813014984, 0.3191385567188263, 0.3178017139434814, 0.3192791938781738, 0.323061466217041, 0.320336639881134, 0.3165886104106903, 0.3206393420696258, 0.3167395293712616, 0.3135207295417785, 0.315539002418518, 0.3191742599010467, 0.321073055267334, 0.3222262561321258, 0.3193058371543884, 0.3213480710983276, 0.3198905289173126, 0.3219239711761474, 0.3211614489555359, 0.318855881690979, 0.3177095353603363, 0.324197381734848, 0.3208906352519989, 0.3264936804771423, 0.3245965242385864, 0.3231639564037323, 0.3221887946128845, 0.3277338445186615, 0.3227696120738983, 0.3263820111751556, 0.3258577883243561, 0.3264622390270233, 0.3222362995147705, 0.3286814987659454, 0.3235024213790893, 0.32446950674057, 0.3311836123466491, 0.328130304813385, 0.3271634578704834, 0.3250012993812561, 0.3309800624847412, 0.3274554014205932, 0.3273015916347503, 0.3261759579181671, 0.32697594165802, 0.3303172886371612, 0.3282814025878906, 0.3289586305618286, 0.3260826468467712, 0.3258011937141418, 0.3297208249568939, 0.3254813551902771, 0.3287739753723144, 0.3287097811698913, 0.3275279700756073, 0.3293041586875915, 0.3314100801944732, 0.3287808299064636, 0.3251930773258209, 0.3288172781467438, 0.3265027701854706, 0.3275215625762939, 0.3290774822235107, 0.3261331617832184, 0.3299777805805206, 0.331955999135971, 0.3305029273033142, 0.3274719417095184, 0.3235560953617096, 0.3269940316677093, 0.3323083519935608], "label": "RefinedWeb"}, "The Pile": {"x": [0.0, 2.0971520000000003, 4.194304000000001, 6.291456, 8.388608000000001, 10.48576, 12.582912, 14.680064000000002, 16.777216000000003, 18.874368, 20.97152, 23.068672000000003, 25.165824, 27.262976000000002, 29.360128000000003, 31.45728, 33.554432000000006, 35.651584, 37.748736, 39.845888, 41.94304, 44.040192000000005, 46.137344000000006, 48.234496, 50.331648, 52.4288, 54.525952000000004, 56.623104000000005, 58.720256000000006, 60.817408, 62.91456, 65.011712, 67.10886400000001, 69.206016, 71.303168, 73.40032000000001, 75.497472, 77.59462400000001, 79.691776, 81.788928, 83.88608, 85.983232, 88.08038400000001, 90.177536, 92.27468800000001, 94.37184, 96.468992, 98.56614400000001, 100.663296, 102.76044800000001, 104.8576, 106.95475200000001, 109.05190400000001, 111.149056, 113.24620800000001, 115.34336, 117.44051200000001, 119.537664, 121.634816, 123.73196800000001, 125.82912, 127.92627200000001, 130.023424, 132.120576, 134.21772800000002, 136.31488000000002, 138.412032, 140.509184, 142.606336, 144.70348800000002, 146.80064000000002, 148.897792, 150.994944, 153.092096, 155.18924800000002, 157.28640000000001, 159.383552, 161.480704, 163.577856, 165.67500800000002, 167.77216, 169.869312, 171.966464, 174.06361600000002, 176.16076800000002, 178.25792, 180.355072, 182.452224, 184.54937600000002, 186.64652800000002, 188.74368, 190.840832, 192.937984, 195.03513600000002, 197.13228800000002, 199.22944, 201.326592, 203.423744, 205.52089600000002, 207.61804800000002, 209.7152, 211.812352, 213.90950400000003, 216.00665600000002, 218.10380800000001, 220.20096, 222.298112, 224.39526400000003, 226.49241600000002, 228.589568, 230.68672, 232.783872, 234.88102400000002, 236.97817600000002, 239.075328, 241.17248, 243.269632, 245.36678400000002, 247.46393600000002, 249.561088, 251.65824, 253.75539200000003, 255.85254400000002, 257.949696, 260.046848, 262.144, 264.241152, 266.338304, 268.43545600000004, 270.53260800000004, 272.62976000000003, 274.726912, 276.824064, 278.921216, 281.018368, 283.11552, 285.212672, 287.309824, 289.40697600000004, 291.50412800000004, 293.60128000000003, 295.698432, 297.795584, 299.892736, 301.989888, 304.08704, 306.184192, 308.28134400000005, 310.37849600000004, 312.47564800000004, 314.57280000000003, 316.669952, 318.767104, 320.864256, 322.961408, 325.05856, 327.155712, 329.25286400000005, 331.35001600000004, 333.44716800000003, 335.54432, 337.641472, 339.738624, 341.835776, 343.932928, 346.03008, 348.12723200000005, 350.22438400000004], "y": [0.2498730421066284, 0.2534083127975464, 0.2561326622962951, 0.2599403262138366, 0.2633565366268158, 0.2649387121200561, 0.2713600993156433, 0.2703712880611419, 0.2707636058330536, 0.2733668386936188, 0.2729981541633606, 0.274116575717926, 0.2803106009960174, 0.2809702754020691, 0.2779591083526611, 0.2805380225181579, 0.2798897624015808, 0.2806138396263122, 0.2803314626216888, 0.2810186445713043, 0.2816281020641327, 0.2882067859172821, 0.2882601618766784, 0.2830324470996856, 0.2890165150165558, 0.2814686000347137, 0.2897007167339325, 0.2888377606868744, 0.2834174335002899, 0.2923084199428558, 0.2913220524787903, 0.2947664260864258, 0.2914148271083832, 0.2925300896167755, 0.293608158826828, 0.2889452278614044, 0.2899258136749267, 0.2933099865913391, 0.2945047914981842, 0.2932035624980926, 0.2960488498210907, 0.2930898070335388, 0.2955847680568695, 0.2957557141780853, 0.2971082925796509, 0.2952709197998047, 0.2950723767280578, 0.2981964945793152, 0.2964051365852356, 0.2939505577087402, 0.2963071465492248, 0.2983180284500122, 0.2972628474235534, 0.3001365661621094, 0.2960390746593475, 0.2977366149425506, 0.2980498075485229, 0.2993811666965484, 0.2978217303752899, 0.2964333891868591, 0.3007307946681976, 0.3038485944271087, 0.3062177002429962, 0.2997282147407532, 0.3027545511722564, 0.3020226955413818, 0.3046002388000488, 0.3044104874134063, 0.3070093989372253, 0.2984016835689544, 0.301089197397232, 0.3019088506698608, 0.2994609773159027, 0.3090873956680298, 0.3015709221363067, 0.3070562481880188, 0.3078455030918121, 0.3082370460033417, 0.3078760802745819, 0.3083541393280029, 0.3041301965713501, 0.3008038699626922, 0.308231920003891, 0.3079721331596374, 0.3082239031791687, 0.3112189173698425, 0.3071774840354919, 0.303611695766449, 0.3100490272045135, 0.306540310382843, 0.3090290427207947, 0.3081201016902923, 0.3089599609375, 0.3034833371639251, 0.3090586364269256, 0.3152794539928436, 0.3124137222766876, 0.308320552110672, 0.3110654950141907, 0.3116510510444641, 0.3129254281520843, 0.312163382768631, 0.3099710345268249, 0.3123080432415008, 0.3135911822319031, 0.3122925460338592, 0.3133870661258697, 0.3189542889595032, 0.3170333206653595, 0.3164204955101013, 0.3185184895992279, 0.319377452135086, 0.3195942044258117, 0.3197914958000183, 0.315343827009201, 0.3192422688007355, 0.3192358016967773, 0.3179579675197601, 0.319799929857254, 0.3205091953277588, 0.3259644210338592, 0.3236323595046997, 0.3209564685821533, 0.3220179975032806, 0.3186767101287842, 0.3223333358764648, 0.3227059245109558, 0.3206779062747955, 0.3189719617366791, 0.324008584022522, 0.3231253325939178, 0.3192954957485199, 0.3221996128559112, 0.3230528831481933, 0.3278523981571197, 0.3217717707157135, 0.3207707703113556, 0.3236245810985565, 0.3222698867321014, 0.3228896558284759, 0.3253422081470489, 0.3267556726932525, 0.3249134719371795, 0.32914799451828, 0.3244758546352386, 0.327859878540039, 0.3285762071609497, 0.3294639587402344, 0.3258695304393768, 0.3260534405708313, 0.3286381065845489, 0.3287994265556335, 0.3297498822212219, 0.3312559723854065, 0.3358747959136963, 0.3314234912395477, 0.3303252458572387, 0.3299272060394287, 0.3301239311695099, 0.3315372467041015, 0.330515444278717, 0.3301299214363098, 0.331067830324173, 0.3263126313686371, 0.3315630555152893, 0.331708014011383, 0.3312946856021881, 0.3308100700378418], "label": "The Pile"}, "FineWeb (ours)": {"x": [0.0, 2.0971520000000003, 4.194304000000001, 6.291456, 8.388608000000001, 10.48576, 12.582912, 14.680064000000002, 16.777216000000003, 18.874368, 20.97152, 23.068672000000003, 25.165824, 27.262976000000002, 29.360128000000003, 31.45728, 33.554432000000006, 35.651584, 37.748736, 39.845888, 41.94304, 44.040192000000005, 46.137344000000006, 48.234496, 50.331648, 52.4288, 54.525952000000004, 56.623104000000005, 58.720256000000006, 60.817408, 62.91456, 65.011712, 67.10886400000001, 69.206016, 71.303168, 73.40032000000001, 75.497472, 77.59462400000001, 79.691776, 81.788928, 83.88608, 85.983232, 88.08038400000001, 90.177536, 92.27468800000001, 94.37184, 96.468992, 98.56614400000001, 100.663296, 102.76044800000001, 104.8576, 106.95475200000001, 109.05190400000001, 111.149056, 113.24620800000001, 115.34336, 117.44051200000001, 119.537664, 121.634816, 123.73196800000001, 125.82912, 127.92627200000001, 130.023424, 132.120576, 134.21772800000002, 136.31488000000002, 138.412032, 140.509184, 142.606336, 144.70348800000002, 146.80064000000002, 148.897792, 150.994944, 153.092096, 155.18924800000002, 157.28640000000001, 159.383552, 161.480704, 163.577856, 165.67500800000002, 167.77216, 169.869312, 171.966464, 174.06361600000002, 176.16076800000002, 178.25792, 180.355072, 182.452224, 184.54937600000002, 186.64652800000002, 188.74368, 190.840832, 192.937984, 195.03513600000002, 197.13228800000002, 199.22944, 201.326592, 203.423744, 205.52089600000002, 207.61804800000002, 209.7152, 211.812352, 213.90950400000003, 216.00665600000002, 218.10380800000001, 220.20096, 222.298112, 224.39526400000003, 226.49241600000002, 228.589568, 230.68672, 232.783872, 234.88102400000002, 236.97817600000002, 239.075328, 241.17248, 243.269632, 245.36678400000002, 247.46393600000002, 249.561088, 251.65824, 253.75539200000003, 255.85254400000002, 257.949696, 260.046848, 262.144, 264.241152, 266.338304, 268.43545600000004, 270.53260800000004, 272.62976000000003, 274.726912, 276.824064, 278.921216, 281.018368, 283.11552, 285.212672, 287.309824, 289.40697600000004, 291.50412800000004, 293.60128000000003, 295.698432, 297.795584, 299.892736, 301.989888, 304.08704, 306.184192, 308.28134400000005, 310.37849600000004, 312.47564800000004, 314.57280000000003, 316.669952, 318.767104, 320.864256, 322.961408, 325.05856, 327.155712, 329.25286400000005, 331.35001600000004, 333.44716800000003, 335.54432, 337.641472, 339.738624, 341.835776, 343.932928, 346.03008, 348.12723200000005, 350.22438400000004], "y": [0.2501466572284698, 0.2562687695026397, 0.264194518327713, 0.2659797668457031, 0.2690401375293731, 0.2707462012767792, 0.2736803293228149, 0.2808477580547333, 0.2819793820381164, 0.2818062305450439, 0.2852273285388946, 0.2852552533149719, 0.293150246143341, 0.2869345247745514, 0.2926198840141296, 0.2911646664142608, 0.2883031964302063, 0.2938489317893982, 0.2923268675804138, 0.2927436530590057, 0.2957125902175903, 0.2942458391189575, 0.2957732379436493, 0.2933609783649444, 0.2939628064632416, 0.2984270751476288, 0.2989151179790497, 0.3007727265357971, 0.2968312501907348, 0.2969468235969543, 0.3013020753860473, 0.3045085966587066, 0.3018752634525299, 0.3014349043369293, 0.2988792657852173, 0.3034284710884094, 0.3015728890895843, 0.3065252900123596, 0.3021449446678161, 0.3043071627616882, 0.303546279668808, 0.3056059181690216, 0.2971993386745453, 0.3057574033737182, 0.3057517111301422, 0.3124973773956299, 0.3139103651046753, 0.3144983947277069, 0.3126215636730194, 0.3140240907669067, 0.3113631308078766, 0.3124240636825561, 0.3126817643642425, 0.3123457431793213, 0.3111095428466797, 0.3113269805908203, 0.3142518699169159, 0.3163851797580719, 0.3134008049964905, 0.3138530254364013, 0.3171449303627014, 0.3119543790817261, 0.3147956132888794, 0.3138984441757202, 0.3178529143333435, 0.3162296414375305, 0.315980851650238, 0.3123161196708679, 0.3166452944278717, 0.3140694200992584, 0.3176922798156738, 0.3176673054695129, 0.3150016367435455, 0.3161586821079254, 0.3222477436065674, 0.3194025754928589, 0.3176416158676147, 0.3159928619861603, 0.3169592320919037, 0.3135637938976288, 0.3155058920383453, 0.3215300440788269, 0.3201274275779724, 0.3192023932933807, 0.3156079053878784, 0.3212503492832184, 0.3163617849349975, 0.3223940432071686, 0.3191330432891845, 0.3194314539432525, 0.3221519589424133, 0.3211863040924072, 0.3197937309741974, 0.3174488544464111, 0.3159596025943756, 0.3157133460044861, 0.3193388879299164, 0.3163386285305023, 0.3202225565910339, 0.3163421154022217, 0.3212694227695465, 0.3187369704246521, 0.3203508555889129, 0.3224054872989654, 0.3207881152629852, 0.3219418525695801, 0.3197605609893799, 0.3255409598350525, 0.3253240585327148, 0.319698303937912, 0.3250498473644256, 0.3228228390216827, 0.3213794529438019, 0.3219127357006073, 0.3214426934719085, 0.3238218128681183, 0.3229665458202362, 0.3220484256744385, 0.3240038454532623, 0.3246393501758575, 0.3237775564193725, 0.3258441984653473, 0.322843462228775, 0.3241913020610809, 0.324148565530777, 0.3238157927989959, 0.3248989582061767, 0.3280864655971527, 0.3288898766040802, 0.3265794515609741, 0.3277602791786194, 0.3231202363967895, 0.3224002718925476, 0.323845773935318, 0.3278093039989471, 0.3247094452381134, 0.3289697468280792, 0.3272296786308288, 0.3275051414966583, 0.3271359801292419, 0.3280861675739288, 0.3281281590461731, 0.327859491109848, 0.3281152546405792, 0.3282515406608581, 0.3258990049362182, 0.3271094560623169, 0.3259278535842895, 0.3258941769599914, 0.3278749883174896, 0.3300504386425018, 0.326113760471344, 0.3242938220500946, 0.3262194991111755, 0.3263693153858185, 0.3274452090263366, 0.3254594206809997, 0.3287247717380523, 0.3250340223312378, 0.3270816206932068, 0.3275731801986694, 0.3282500207424164, 0.3257671594619751, 0.3272948265075683, 0.3274084031581878, 0.3302212655544281, 0.3322067260742187, 0.3296935856342315], "label": "FineWeb (ours)"}, "RedPajama2": {"x": [0.0, 2.0971520000000003, 4.194304000000001, 6.291456, 8.388608000000001, 10.48576, 12.582912, 14.680064000000002, 16.777216000000003, 18.874368, 20.97152, 23.068672000000003, 25.165824, 27.262976000000002, 29.360128000000003, 31.45728, 33.554432000000006, 35.651584, 37.748736, 39.845888, 41.94304, 44.040192000000005, 46.137344000000006, 48.234496, 50.331648, 52.4288, 54.525952000000004, 56.623104000000005, 58.720256000000006, 60.817408, 62.91456, 65.011712, 67.10886400000001, 69.206016, 71.303168, 73.40032000000001, 75.497472, 77.59462400000001, 79.691776, 81.788928, 83.88608, 85.983232, 88.08038400000001, 90.177536, 92.27468800000001, 94.37184, 96.468992, 98.56614400000001, 100.663296, 102.76044800000001, 104.8576, 106.95475200000001, 109.05190400000001, 111.149056, 113.24620800000001, 115.34336, 117.44051200000001, 119.537664, 121.634816, 123.73196800000001, 125.82912, 127.92627200000001, 130.023424, 132.120576, 134.21772800000002, 136.31488000000002, 138.412032, 140.509184, 142.606336, 144.70348800000002, 146.80064000000002, 148.897792, 150.994944, 153.092096, 155.18924800000002, 157.28640000000001, 159.383552, 161.480704, 163.577856, 165.67500800000002, 167.77216, 169.869312, 171.966464, 174.06361600000002, 176.16076800000002, 178.25792, 180.355072, 182.452224, 184.54937600000002, 186.64652800000002, 188.74368, 190.840832, 192.937984, 195.03513600000002, 197.13228800000002, 199.22944, 201.326592, 203.423744, 205.52089600000002, 207.61804800000002, 209.7152, 211.812352, 213.90950400000003, 216.00665600000002, 218.10380800000001, 220.20096, 222.298112, 224.39526400000003, 226.49241600000002, 228.589568, 230.68672, 232.783872, 234.88102400000002, 236.97817600000002, 239.075328, 241.17248, 243.269632, 245.36678400000002, 247.46393600000002, 249.561088, 251.65824, 253.75539200000003, 255.85254400000002, 257.949696, 260.046848, 262.144, 264.241152, 266.338304, 268.43545600000004, 270.53260800000004, 272.62976000000003, 274.726912, 276.824064, 278.921216, 281.018368, 283.11552, 285.212672, 287.309824, 289.40697600000004, 291.50412800000004, 293.60128000000003, 295.698432, 297.795584, 299.892736, 301.989888, 304.08704, 306.184192, 308.28134400000005, 310.37849600000004, 312.47564800000004, 314.57280000000003, 316.669952, 318.767104, 320.864256, 322.961408, 325.05856, 327.155712, 329.25286400000005, 331.35001600000004, 333.44716800000003, 335.54432, 337.641472, 339.738624, 341.835776, 343.932928, 346.03008, 348.12723200000005, 350.22438400000004], "y": [0.2496780008077621, 0.2519060671329498, 0.2591428160667419, 0.2637551724910736, 0.2671225368976593, 0.2742190659046173, 0.2761570811271667, 0.27329882979393, 0.2755842506885528, 0.2790464162826538, 0.2869725823402405, 0.2838374674320221, 0.2864839732646942, 0.2891924679279327, 0.2910299301147461, 0.2891930639743805, 0.288548856973648, 0.29207244515419, 0.2911308705806732, 0.2965781390666961, 0.2966488599777221, 0.2910915017127991, 0.2942662537097931, 0.2906062006950378, 0.2968285381793976, 0.299734503030777, 0.2920202910900116, 0.3011439740657806, 0.2991106510162353, 0.2955383658409118, 0.2984142899513244, 0.3006328046321869, 0.3036990761756897, 0.2976612746715545, 0.2990829646587372, 0.2943736910820007, 0.300097107887268, 0.2956410348415375, 0.3071651458740234, 0.3055790960788727, 0.3042990863323211, 0.3054490685462951, 0.3014840185642242, 0.298349380493164, 0.3029196262359619, 0.3046568930149078, 0.306732028722763, 0.3014123737812042, 0.3036067485809326, 0.3066047132015228, 0.3052354753017425, 0.3075886964797973, 0.3091740012168884, 0.3017406165599823, 0.3043143153190613, 0.3051626980304718, 0.3052559792995453, 0.3032524585723877, 0.3126200735569, 0.3073466420173645, 0.3121364712715149, 0.3114152550697326, 0.3056653439998626, 0.3071228861808777, 0.3121611177921295, 0.3070071935653686, 0.3095386028289795, 0.3122327923774719, 0.3104317188262939, 0.3108241260051727, 0.3109523057937622, 0.3170003592967987, 0.3106639087200165, 0.3124147951602936, 0.3132331371307373, 0.3180687129497528, 0.3111820816993713, 0.3153944313526153, 0.3127918243408203, 0.3113606572151184, 0.3180998861789703, 0.3133571743965149, 0.314390480518341, 0.3146277070045471, 0.3148570358753204, 0.3144837021827698, 0.3187994956970215, 0.3204345703125, 0.3150869607925415, 0.3201290965080261, 0.3202804625034332, 0.319104790687561, 0.3177886009216308, 0.3194527924060821, 0.3167977631092071, 0.3182326853275299, 0.3169693946838379, 0.3235675394535064, 0.3185839354991913, 0.3215724229812622, 0.3182428181171417, 0.3219003081321716, 0.3199219107627868, 0.3217185437679291, 0.3206124603748321, 0.3205193877220154, 0.318211942911148, 0.3197448849678039, 0.3191796839237213, 0.3248491883277893, 0.3221067786216736, 0.3249030709266662, 0.3219638168811798, 0.3252624571323395, 0.3218883872032165, 0.3220428824424743, 0.3214907944202423, 0.3213936984539032, 0.3221090137958526, 0.3221961259841919, 0.3210761547088623, 0.3239326179027557, 0.321384847164154, 0.3226822912693023, 0.3249708116054535, 0.3271316289901733, 0.3257546424865722, 0.323205828666687, 0.3232316970825195, 0.3275502622127533, 0.3237724900245666, 0.3249053359031677, 0.3257182836532593, 0.3244708180427551, 0.3246290981769562, 0.3252749741077423, 0.3240469694137573, 0.329105406999588, 0.3294115364551544, 0.3245800733566284, 0.3285538256168365, 0.3260121047496795, 0.3243294656276703, 0.3270638883113861, 0.3259481191635132, 0.3246479034423828, 0.321680337190628, 0.3224166929721832, 0.3243135511875152, 0.3283822238445282, 0.3236311376094818, 0.327635109424591, 0.3274770975112915, 0.3242181539535522, 0.3290536999702453, 0.3260405361652374, 0.3252047896385193, 0.3275159001350403, 0.3269789516925812, 0.3300522267818451, 0.3290919959545135, 0.3258809447288513, 0.3274780511856079, 0.3302557468414306, 0.3261787891387939, 0.3282874226570129, 0.3261036276817322, 0.3293801844120025], "label": "RedPajama2"}, "Dolma": {"x": [0.0, 2.0971520000000003, 4.194304000000001, 6.291456, 8.388608000000001, 10.48576, 12.582912, 14.680064000000002, 16.777216000000003, 18.874368, 20.97152, 23.068672000000003, 25.165824, 27.262976000000002, 29.360128000000003, 31.45728, 33.554432000000006, 35.651584, 37.748736, 39.845888, 41.94304, 44.040192000000005, 46.137344000000006, 48.234496, 50.331648, 52.4288, 54.525952000000004, 56.623104000000005, 58.720256000000006, 60.817408, 62.91456, 65.011712, 67.10886400000001, 69.206016, 71.303168, 73.40032000000001, 75.497472, 77.59462400000001, 79.691776, 81.788928, 83.88608, 85.983232, 88.08038400000001, 90.177536, 92.27468800000001, 94.37184, 96.468992, 98.56614400000001, 100.663296, 102.76044800000001, 104.8576, 106.95475200000001, 109.05190400000001, 111.149056, 113.24620800000001, 115.34336, 117.44051200000001, 119.537664, 121.634816, 123.73196800000001, 125.82912, 127.92627200000001, 130.023424, 132.120576, 134.21772800000002, 136.31488000000002, 138.412032, 140.509184, 142.606336, 144.70348800000002, 146.80064000000002, 148.897792, 150.994944, 153.092096, 155.18924800000002, 157.28640000000001, 159.383552, 161.480704, 163.577856, 165.67500800000002, 167.77216, 169.869312, 171.966464, 174.06361600000002, 176.16076800000002, 178.25792, 180.355072, 182.452224, 184.54937600000002, 186.64652800000002, 188.74368, 190.840832, 192.937984, 195.03513600000002, 197.13228800000002, 199.22944, 201.326592, 203.423744, 205.52089600000002, 207.61804800000002, 209.7152, 211.812352, 213.90950400000003, 216.00665600000002, 218.10380800000001, 220.20096, 222.298112, 224.39526400000003, 226.49241600000002, 228.589568, 230.68672, 232.783872, 234.88102400000002, 236.97817600000002, 239.075328, 241.17248, 243.269632, 245.36678400000002, 247.46393600000002, 249.561088, 251.65824, 253.75539200000003, 255.85254400000002, 257.949696, 260.046848, 262.144, 264.241152, 266.338304, 268.43545600000004, 270.53260800000004, 272.62976000000003, 274.726912, 276.824064, 278.921216, 281.018368, 283.11552, 285.212672, 287.309824, 289.40697600000004, 291.50412800000004, 293.60128000000003, 295.698432, 297.795584, 299.892736, 301.989888, 304.08704, 306.184192, 308.28134400000005, 310.37849600000004, 312.47564800000004, 314.57280000000003, 316.669952, 318.767104, 320.864256, 322.961408, 325.05856, 327.155712, 329.25286400000005, 331.35001600000004, 333.44716800000003, 335.54432, 337.641472, 339.738624, 341.835776, 343.932928, 346.03008, 348.12723200000005, 350.22438400000004], "y": [0.2501270473003387, 0.2560833990573883, 0.2599925696849823, 0.2675598859786987, 0.2684114277362823, 0.2716337144374847, 0.2740766704082489, 0.2775039076805115, 0.2783404588699341, 0.2796013355255127, 0.2820949554443359, 0.2810406684875488, 0.2823644876480102, 0.2833253145217895, 0.2857703566551208, 0.2808239161968231, 0.2917547225952148, 0.2897486090660095, 0.2875275015830993, 0.2943058311939239, 0.2891780138015747, 0.2897033989429474, 0.2903319895267486, 0.2924950718879699, 0.2875644862651825, 0.2924458384513855, 0.293954849243164, 0.2929113805294037, 0.2915636599063873, 0.2917861938476562, 0.2942286729812622, 0.2967639863491058, 0.291913092136383, 0.2953178882598877, 0.2972874641418457, 0.2956210374832153, 0.3018575012683868, 0.2929699420928955, 0.2950209975242615, 0.3003560602664947, 0.2974177300930023, 0.2950251400470733, 0.296758621931076, 0.2972809970378876, 0.2982426285743713, 0.3019979298114776, 0.3008206784725189, 0.3005954325199127, 0.2938346862792969, 0.3009699583053589, 0.2989449501037597, 0.2996537983417511, 0.3019669651985168, 0.3000748455524444, 0.3000148236751556, 0.2995675504207611, 0.298966109752655, 0.3004475831985473, 0.3023184835910797, 0.3034453392028808, 0.3016394674777984, 0.3060395121574402, 0.3066911995410919, 0.3065873384475708, 0.29949951171875, 0.3031920790672302, 0.3026251792907715, 0.3046470880508423, 0.3025145232677459, 0.306982010602951, 0.302643358707428, 0.304037868976593, 0.3015348613262176, 0.299553781747818, 0.3041835427284241, 0.3027640879154205, 0.3082228600978851, 0.3047288656234741, 0.3039765655994415, 0.3048267066478729, 0.3035992980003357, 0.3056430518627167, 0.3037824630737304, 0.3048160970211029, 0.3040882349014282, 0.3052836060523987, 0.3065180480480194, 0.307971179485321, 0.3091272115707397, 0.3034843504428863, 0.3075712919235229, 0.3104844093322754, 0.3063389658927917, 0.3032608032226562, 0.3080332279205322, 0.3079279065132141, 0.3062038123607635, 0.3072481155395508, 0.3117794990539551, 0.3067246973514557, 0.3082475662231445, 0.3099625706672668, 0.3070119023323059, 0.3101015090942383, 0.3109049797058105, 0.3087223470211029, 0.3101778626441955, 0.3094301521778106, 0.3144752383232116, 0.3116049468517303, 0.3116113245487213, 0.3137989044189453, 0.3153277635574341, 0.3151018321514129, 0.3132036030292511, 0.3145381212234497, 0.3129039704799652, 0.3149133026599884, 0.3102412819862366, 0.3117950558662414, 0.3154202997684479, 0.3132332563400268, 0.3113301992416382, 0.3101116418838501, 0.3130112290382385, 0.3137075006961822, 0.315510481595993, 0.3132544159889221, 0.3161831498146057, 0.3116595447063446, 0.3104106485843658, 0.3173929154872894, 0.314871996641159, 0.316310316324234, 0.3159449696540832, 0.3161650598049164, 0.3156299889087677, 0.3161895871162414, 0.3156271278858185, 0.3160959482192993, 0.317343145608902, 0.3176217675209045, 0.3190700709819793, 0.3196708261966705, 0.3158564865589142, 0.3173984587192535, 0.316327154636383, 0.3143565356731415, 0.3159370124340057, 0.3192657828330993, 0.3172151446342468, 0.3148190677165985, 0.3171719908714294, 0.3181072175502777, 0.3178979456424713, 0.3210897445678711, 0.3147720694541931, 0.3171736001968384, 0.3176901936531067, 0.3162892162799835, 0.3163918852806091, 0.3174804747104645, 0.3201273679733276, 0.3196343183517456, 0.3189445436000824, 0.3197543323040008, 0.3184814155101776, 0.3209713697433471], "label": "Dolma"}, "C4": {"x": [0.0, 2.0971520000000003, 4.194304000000001, 6.291456, 8.388608000000001, 10.48576, 12.582912, 14.680064000000002, 16.777216000000003, 18.874368, 20.97152, 23.068672000000003, 25.165824, 27.262976000000002, 29.360128000000003, 31.45728, 33.554432000000006, 35.651584, 37.748736, 39.845888, 41.94304, 44.040192000000005, 46.137344000000006, 48.234496, 50.331648, 52.4288, 54.525952000000004, 56.623104000000005, 58.720256000000006, 60.817408, 62.91456, 65.011712, 67.10886400000001, 69.206016, 71.303168, 73.40032000000001, 75.497472, 77.59462400000001, 79.691776, 81.788928, 83.88608, 85.983232, 88.08038400000001, 90.177536, 92.27468800000001, 94.37184, 96.468992, 98.56614400000001, 100.663296, 102.76044800000001, 104.8576, 106.95475200000001, 109.05190400000001, 111.149056, 113.24620800000001, 115.34336, 117.44051200000001, 119.537664, 121.634816, 123.73196800000001, 125.82912, 127.92627200000001, 130.023424, 132.120576, 134.21772800000002, 136.31488000000002, 138.412032, 140.509184, 142.606336, 144.70348800000002, 146.80064000000002, 148.897792, 150.994944, 153.092096, 155.18924800000002, 157.28640000000001, 159.383552, 161.480704, 163.577856, 165.67500800000002, 167.77216, 169.869312, 171.966464, 174.06361600000002, 176.16076800000002, 178.25792, 180.355072, 182.452224, 184.54937600000002, 186.64652800000002, 188.74368, 190.840832, 192.937984, 195.03513600000002, 197.13228800000002, 199.22944, 201.326592, 203.423744, 205.52089600000002, 207.61804800000002, 209.7152, 211.812352, 213.90950400000003, 216.00665600000002, 218.10380800000001, 220.20096, 222.298112, 224.39526400000003, 226.49241600000002, 228.589568, 230.68672, 232.783872, 234.88102400000002, 236.97817600000002, 239.075328, 241.17248, 243.269632, 245.36678400000002, 247.46393600000002, 249.561088, 251.65824, 253.75539200000003, 255.85254400000002, 257.949696, 260.046848, 262.144, 264.241152, 266.338304, 268.43545600000004, 270.53260800000004, 272.62976000000003, 274.726912, 276.824064, 278.921216, 281.018368, 283.11552, 285.212672, 287.309824, 289.40697600000004, 291.50412800000004, 293.60128000000003, 295.698432, 297.795584, 299.892736, 301.989888, 304.08704, 306.184192, 308.28134400000005, 310.37849600000004, 312.47564800000004, 314.57280000000003, 316.669952, 318.767104, 320.864256, 322.961408, 325.05856, 327.155712, 329.25286400000005, 331.35001600000004, 333.44716800000003, 335.54432, 337.641472, 339.738624, 341.835776, 343.932928, 346.03008, 348.12723200000005, 350.22438400000004], "y": [0.2501466572284698, 0.2503989636898041, 0.2554822564125061, 0.267362505197525, 0.2635625898838043, 0.2717761397361755, 0.2754477560520172, 0.275278776884079, 0.2799545228481293, 0.2766266465187073, 0.27903613448143, 0.2817002832889557, 0.2798756062984466, 0.2839230597019195, 0.2838127315044403, 0.2853756248950958, 0.2841375172138214, 0.2884852290153503, 0.2859430313110351, 0.2888604700565338, 0.2842839062213897, 0.2834508419036865, 0.2905769944190979, 0.291401594877243, 0.2923042178153991, 0.2906481623649597, 0.2864490151405334, 0.2894583344459533, 0.2936400771141052, 0.289957731962204, 0.2898236215114593, 0.2837969064712524, 0.2944568693637848, 0.287180632352829, 0.2923372685909271, 0.2921179831027984, 0.2892000675201416, 0.2896223366260528, 0.2912705242633819, 0.2905389666557312, 0.2875383198261261, 0.2915911972522735, 0.2923913896083832, 0.2919397950172424, 0.2927466332912445, 0.2955999374389648, 0.2927011847496032, 0.2947700321674347, 0.2957072854042053, 0.2927916347980499, 0.2963354587554931, 0.2973482012748718, 0.2991726994514465, 0.2962333858013153, 0.2936276495456695, 0.2937322854995727, 0.3001607954502105, 0.2961930930614471, 0.2966057658195495, 0.2966968417167663, 0.3002983927726745, 0.2983409464359283, 0.2959610521793365, 0.2962177395820617, 0.2951928675174713, 0.2985051274299621, 0.30136439204216, 0.2984142005443573, 0.3012503385543823, 0.299721896648407, 0.3021658658981323, 0.2981589436531067, 0.2985765635967254, 0.2982990145683288, 0.3036385178565979, 0.3027603924274444, 0.3010715246200561, 0.3001661598682403, 0.298428326845169, 0.3019610047340393, 0.3065414726734161, 0.29936483502388, 0.3038617968559265, 0.3041279911994934, 0.3058141767978668, 0.3057383298873901, 0.3019986450672149, 0.3034681677818298, 0.3048995435237884, 0.3043853640556335, 0.3066711127758026, 0.30138099193573, 0.3047040700912475, 0.3035959601402282, 0.3064981997013092, 0.3050976097583771, 0.3082782626152038, 0.3046838641166687, 0.3041907548904419, 0.3060542941093445, 0.3100601136684418, 0.3060735464096069, 0.3048785924911499, 0.3074990510940552, 0.3062954843044281, 0.3097685873508453, 0.3098846077919006, 0.3045764863491058, 0.3065372407436371, 0.308793306350708, 0.3075888156890869, 0.3107441067695617, 0.3069410920143127, 0.3096908032894134, 0.3064695000648498, 0.3067855834960937, 0.3074861168861389, 0.3091084659099579, 0.3072076141834259, 0.3065063059329986, 0.3104434311389923, 0.3067492246627807, 0.3116618990898132, 0.3115111291408539, 0.3112091422080993, 0.3067144453525543, 0.3121508359909057, 0.3089344203472137, 0.3137882351875305, 0.3108713626861572, 0.312741070985794, 0.3097975254058838, 0.3122666180133819, 0.3133568465709686, 0.3127341866493225, 0.3080432415008545, 0.3120637834072113, 0.3122869431972503, 0.3120250403881073, 0.3117712736129761, 0.3109983205795288, 0.3125808835029602, 0.3116428554058075, 0.3129400014877319, 0.3116071224212646, 0.3128395676612854, 0.3080540597438812, 0.3104397654533386, 0.3116388320922851, 0.3122957944869995, 0.3113322257995605, 0.313622385263443, 0.313764363527298, 0.3136481344699859, 0.3141548335552215, 0.3139144778251648, 0.3123406767845154, 0.314969539642334, 0.3138300478458404, 0.3155047297477722, 0.3115970492362976, 0.3151944875717163, 0.3144146800041199, 0.3156995475292206, 0.3192606270313263, 0.3183640241622925, 0.3151223957538605, 0.3162081837654114], "label": "C4"}}, "layout": {"title": {"text": "Dataset ablations"}}} |